Merge branch 'next' of github.com:Budibase/budibase into datasource-refactor

2024-06-29 11:31:06 +12:00 · 2021-03-29 15:40:35 +01:00 · 2021-03-29 15:40:35 +01:00 · 9012df0b90
parent 6b6debf6f9 6efea66505
commit 9012df0b90
100 changed files with 2811 additions and 1403 deletions
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -52,8 +52,8 @@ jobs:

          mac_certs: ${{ secrets.mac_certs }}
          mac_certs_password: ${{ secrets.mac_certs_password }}
-          # windows_certs: ${{ secrets.windows_certs }}
-          # windows_certs_password: ${{ secrets.windows_certs_password }}
+          windows_certs: ${{ secrets.windows_certs }}
+          windows_certs_password: ${{ secrets.windows_certs_password }}

          # release the app after building
          release: ${{ startsWith(github.ref, 'refs/tags/v') }}
--- a/.gitignore
+++ b/.gitignore
@ -83,4 +83,7 @@ typings/
 .DS_Store

 # Nova Editor
-.nova
+.nova
+
+# swap files (linux)
+*.swp
--- a/hosting/build/docker-compose.yaml
+++ b/hosting/build/docker-compose.yaml
@ -1,16 +0,0 @@
-version: "3"
-
-services:
-  app-service:
-    build: ./server
-    volumes: 
-      - ./server:/app
-    environment:
-      SELF_HOSTED: 1 
-      PORT: 4002
-
-  worker-service:
-    build: ./worker
-    environment:
-      SELF_HOSTED: 1,
-      PORT: 4003
--- a/hosting/build/server
+++ b/hosting/build/server
@ -1 +0,0 @@
-../../packages/server/
--- a/hosting/build/worker
+++ b/hosting/build/worker
@ -1 +0,0 @@
-../../packages/worker/
--- a/hosting/docker-compose.dev.yaml
+++ b/hosting/docker-compose.dev.yaml
@ -0,0 +1,73 @@
+version: "3"
+
+# optional ports are specified throughout for more advanced use cases.
+
+services:
+  minio-service:
+    container_name: budi-minio-dev
+    restart: always
+    image: minio/minio
+    volumes:
+      - minio_data:/data
+    ports:
+      - "${MINIO_PORT}:9000"
+    environment:
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      MINIO_BROWSER: "off"
+    command: server /data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  proxy-service:
+    container_name: budi-envoy-dev
+    restart: always
+    image: envoyproxy/envoy:v1.16-latest
+    volumes:
+      - ./envoy.dev.yaml:/etc/envoy/envoy.yaml
+    ports:
+      - "${MAIN_PORT}:10000"
+    depends_on:
+      - minio-service
+      - couchdb-service
+
+  couchdb-service:
+    container_name: budi-couchdb-dev
+    restart: always
+    image: ibmcom/couchdb3
+    environment:
+      - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
+      - COUCHDB_USER=${COUCH_DB_USER}
+    ports:
+      - "${COUCH_DB_PORT}:5984"
+    volumes:
+      - couchdb3_data:/opt/couchdb/data
+  
+  couch-init:
+    container_name: budi-couchdb-init-dev
+    image: curlimages/curl
+    environment:
+      PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984"
+    depends_on:
+      - couchdb-service
+    command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"]
+
+  redis-service:
+    container_name: budi-redis-dev
+    restart: always
+    image: redis
+    ports:
+      - "${REDIS_PORT}:6379"
+    volumes:
+      - redis_data:/data
+
+volumes:
+  couchdb3_data:
+    driver: local
+  minio_data:
+    driver: local
+  redis_data:
+    driver: local
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@ -11,13 +11,18 @@ services:
      - "${APP_PORT}:4002"
    environment:
      SELF_HOSTED: 1
-      CLOUD: 1
      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
      WORKER_URL: http://worker-service:4003
+      MINIO_URL: http://minio-service:9000
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
      HOSTING_KEY: ${HOSTING_KEY}
      BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
      PORT: 4002
      JWT_SECRET: ${JWT_SECRET}
+      LOG_LEVEL: info
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      ENABLE_ANALYTICS: true
    depends_on:
      - worker-service

@ -28,7 +33,7 @@ services:
    ports:
      - "${WORKER_PORT}:4003"
    environment:
-      SELF_HOSTED: 1,
+      SELF_HOSTED: 1
      PORT: 4003
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
@ -66,7 +71,6 @@ services:
      - ./envoy.yaml:/etc/envoy/envoy.yaml
    ports:
      - "${MAIN_PORT}:10000"
-      #- "9901:9901"
    depends_on:
      - minio-service
      - worker-service
@ -75,16 +79,14 @@ services:

  couchdb-service:
    restart: always
-    image: apache/couchdb:3.0
+    image: ibmcom/couchdb3
    environment:
      - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
      - COUCHDB_USER=${COUCH_DB_USER}
    ports:
      - "${COUCH_DB_PORT}:5984"
-      #- "4369:4369"
-      #- "9100:9100"
    volumes:
-      - couchdb_data:/opt/couchdb/data
+      - couchdb3_data:/opt/couchdb/data

  couch-init:
    image: curlimages/curl
@ -93,9 +95,19 @@ services:
    depends_on:
      - couchdb-service
    command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"]
+  
+  redis-service:
+    restart: always
+    image: redis
+    ports:
+      - "${REDIS_PORT}:6379"
+    volumes:
+      - redis_data:/data

 volumes:
-  couchdb_data:
+  couchdb3_data:
    driver: local
  minio_data:
    driver: local
+  redis_data:
+    driver: local
--- a/hosting/envoy.dev.yaml
+++ b/hosting/envoy.dev.yaml
@ -0,0 +1,79 @@
+static_resources:
+  listeners:
+  - name: main_listener
+    address:
+      socket_address: { address: 0.0.0.0, port_value: 10000 }
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress
+          codec_type: auto
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_services
+              domains: ["*"]
+              routes:
+              - match: { prefix: "/db/" }
+                route:
+                  cluster: couchdb-service
+                  prefix_rewrite: "/"
+
+              - match: { prefix: "/cache/" }
+                route:
+                  cluster: redis-service
+                  prefix_rewrite: "/"
+
+              # minio is on the default route because this works
+              # best, minio + AWS SDK doesn't handle path proxy
+              - match: { prefix: "/" }
+                route:
+                  cluster: minio-service
+
+          http_filters:
+          - name: envoy.filters.http.router
+
+  clusters:
+  - name: minio-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: minio-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: minio-service
+                port_value: 9000
+
+  - name: couchdb-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: couchdb-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: couchdb-service
+                port_value: 5984
+
+  - name: redis-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: redis-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: redis-service
+                port_value: 6379
--- a/hosting/envoy.yaml
+++ b/hosting/envoy.yaml
@ -36,6 +36,11 @@ static_resources:
                  cluster: worker-service
                  prefix_rewrite: "/"

+              - match: { prefix: "/cache/" }
+                route:
+                  cluster: redis-service
+                  prefix_rewrite: "/"
+
              - match: { prefix: "/db/" }
                route:
                  cluster: couchdb-service
@ -107,3 +112,18 @@ static_resources:
                address: couchdb-service
                port_value: 5984

+  - name: redis-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: redis-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: redis-service
+                port_value: 6379
+
+
--- a/hosting/hosting.properties
+++ b/hosting/hosting.properties
@ -18,4 +18,5 @@ APP_PORT=4002
 WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
+REDIS_PORT=6379
 BUDIBASE_ENVIRONMENT=PRODUCTION
--- a/packages/builder/cypress.json
+++ b/packages/builder/cypress.json
@ -3,6 +3,7 @@
  "video": true,
  "projectId": "bmbemn",
  "env": {
-    "PORT": "4001"
+    "PORT": "4001",
+    "JWT_SECRET": "test"
  }
 }
--- a/packages/builder/cypress/setup.js
+++ b/packages/builder/cypress/setup.js
@ -1,25 +1,26 @@
-// What this script does:
-// 1. Removes the old test folder if it exists (.budibase)
-// 2. Initialises using `.budibase`
-// 3. Runs the server using said folder
-
-const { join, resolve } = require("path")
-const initialiseBudibase = require("../../server/src/utilities/initialiseBudibase")
 const cypressConfig = require("../cypress.json")
+const path = require("path")

-const homedir = join(require("os").homedir(), ".budibase")
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")

 process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
 process.env.NODE_ENV = "cypress"
 process.env.ENABLE_ANALYTICS = "false"
 process.env.PORT = cypressConfig.env.PORT
+process.env.JWT_SECRET = cypressConfig.env.JWT_SECRET
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = 1
+process.env.MINIO_URL = "http://localhost:10000/"
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"

 // Stop info logs polluting test outputs
 process.env.LOG_LEVEL = "error"

-async function run(dir) {
-  process.env.BUDIBASE_DIR = resolve(dir)
-  require("dotenv").config({ path: resolve(dir, ".env") })
+async function run() {
+  // require("dotenv").config({ path: resolve(dir, ".env") })

  // dont make this a variable or top level require
  // it will cause environment module to be loaded prematurely
@ -27,12 +28,15 @@ async function run(dir) {
  server.on("close", () => console.log("Server Closed"))
 }

-initialiseBudibase({ dir: homedir, clientId: "cypress-test" })
-  .then(() => {
-    delete require.cache[require.resolve("../../server/src/environment")]
-    const xPlatHomeDir = homedir.startsWith("~")
-      ? join(homedir(), homedir.substring(1))
-      : homedir
-    run(xPlatHomeDir)
-  })
-  .catch(e => console.error(e))
+run()
+
+// TODO: ensure that this still works
+// initialiseBudibase({ dir: homedir, clientId: "cypress-test" })
+// .then(() => {
+//   delete require.cache[require.resolve("../../server/src/environment")]
+//   const xPlatHomeDir = homedir.startsWith("~")
+//     ? join(homedir(), homedir.substring(1))
+//     : homedir
+//   run(xPlatHomeDir)
+// })
+// .catch(e => console.error(e))
--- a/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte
@ -27,7 +27,7 @@
    notifier.success("Datasource deleted")
    // navigate to first index page if the source you are deleting is selected
    if (wasSelectedSource === datasource._id) {
-      $goto('./datasource')
+      $goto("./datasource")
    }
    hideEditor()
  }
--- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
@ -37,13 +37,13 @@
  }

  async function deleteTable() {
-    const wasSelectedTable = $backendUiStore.selectedTable 
+    const wasSelectedTable = $backendUiStore.selectedTable
    await backendUiStore.actions.tables.delete(table)
    store.actions.screens.delete(templateScreens)
    await backendUiStore.actions.tables.fetch()
    notifier.success("Table deleted")
    if (wasSelectedTable._id === table._id) {
-      $goto('./table')
+      $goto("./table")
    }
    hideEditor()
  }
--- a/packages/cli/src/hosting/makeEnv.js
+++ b/packages/cli/src/hosting/makeEnv.js
@ -26,6 +26,7 @@ APP_PORT=4002
 WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
+REDIS_PORT=6379
 BUDIBASE_ENVIRONMENT=PRODUCTION`
 }

--- a/packages/client/src/api/tables.js
+++ b/packages/client/src/api/tables.js
@ -21,14 +21,16 @@ export const fetchTableData = async tableId => {
 * Perform a mango query against an internal table
 * @param {String} tableId - id of the table to search
 * @param {Object} search - Mango Compliant search object
+ * @param {Object} pagination - the pagination controls
 */
 export const searchTableData = async ({ tableId, search, pagination }) => {
-  const rows = await API.post({
+  const output = await API.post({
    url: `/api/${tableId}/rows/search`,
    body: {
      query: search,
      pagination,
    },
  })
-  return await enrichRows(rows, tableId)
+  output.rows = await enrichRows(output.rows, tableId)
+  return output
 }
--- a/packages/server/.env.template
+++ b/packages/server/.env.template
@ -1,17 +0,0 @@
-# url of couch db, including username and password
-# http://admin:password@localhost:5984
-COUCH_DB_URL={{couchDbUrl}}
-
-# identifies a client database - i.e. group of apps
-CLIENT_ID={{clientId}}
-
-# used to create cookie hashes
-JWT_SECRET={{cookieKey1}}
-
-# error level for koa-pino
-LOG_LEVEL=info
-
-DEPLOYMENT_CREDENTIALS_URL="https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/"
-DEPLOYMENT_DB_URL="https://couchdb.budi.live:5984"
-SENTRY_DSN=https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
-ENABLE_ANALYTICS="true"
--- a/packages/server/Dockerfile
+++ b/packages/server/Dockerfile
@ -2,7 +2,6 @@ FROM node:12-alpine

 WORKDIR /app

-ENV CLOUD=1
 ENV PORT=4001
 ENV COUCH_DB_URL=https://couchdb.budi.live:5984
 ENV BUDIBASE_ENVIRONMENT=PRODUCTION
--- a/packages/server/mocks/node-fetch.js
+++ b/packages/server/mocks/node-fetch.js
@ -30,6 +30,17 @@ module.exports = async (url, opts) => {
      },
      404
    )
+  } else if (url.includes("_search")) {
+    return json({
+      rows: [
+        {
+          doc: {
+            _id: "test",
+          },
+        },
+      ],
+      bookmark: "test",
+    })
  }
  return fetch(url, opts)
 }
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -36,7 +36,10 @@
    "test:integration": "jest --coverage --detectOpenHandles",
    "test:watch": "jest --watch",
    "run:docker": "node src/index",
-    "dev:builder": "cross-env PORT=4001 nodemon src/index.js",
+    "dev:stack:up": "node scripts/dev/manage.js up",
+    "dev:stack:down": "node scripts/dev/manage.js down",
+    "dev:stack:nuke": "node scripts/dev/manage.js nuke",
+    "dev:builder": "npm run dev:stack:up && nodemon src/index.js",
    "electron": "electron src/electron.js",
    "build:electron": "electron-builder --dir",
    "publish:electron": "electron-builder -mwl --publish always",
@ -63,7 +66,7 @@
      "!src/tests/**/*",
      "!src/automations/tests/**/*",
      "!src/utilities/fileProcessor.js",
-      "!src/utilities/initialiseBudibase.js"
+      "!src/utilities/fileSystem/**/*"
    ],
    "coverageReporters": [
      "lcov",
@ -134,7 +137,7 @@
  "devDependencies": {
    "@budibase/standard-components": "^0.8.9",
    "@jest/test-sequencer": "^24.8.0",
-    "cross-env": "^7.0.3",
+    "docker-compose": "^0.23.6",
    "electron": "10.1.3",
    "electron-builder": "^22.9.1",
    "electron-builder-notarize": "^1.1.2",
--- a/packages/server/scripts/dev/manage.js
+++ b/packages/server/scripts/dev/manage.js
@ -0,0 +1,96 @@
+#!/usr/bin/env node
+const compose = require("docker-compose")
+const path = require("path")
+const fs = require("fs")
+
+// This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
+const CONFIG = {
+  cwd: path.resolve(process.cwd(), "../../hosting"),
+  config: "docker-compose.dev.yaml",
+  log: true,
+}
+
+const Commands = {
+  Up: "up",
+  Down: "down",
+  Nuke: "nuke",
+}
+
+async function init() {
+  const envFilePath = path.join(process.cwd(), ".env")
+  if (fs.existsSync(envFilePath)) {
+    return
+  }
+  const envFileJson = {
+    PORT: 4001,
+    MINIO_URL: "http://localhost:10000/",
+    COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
+    WORKER_URL: "http://localhost:4002",
+    JWT_SECRET: "testsecret",
+    MINIO_ACCESS_KEY: "budibase",
+    MINIO_SECRET_KEY: "budibase",
+    COUCH_DB_PASSWORD: "budibase",
+    COUCH_DB_USER: "budibase",
+    SELF_HOSTED: 1,
+  }
+  let envFile = ""
+  Object.keys(envFileJson).forEach(key => {
+    envFile += `${key}=${envFileJson[key]}\n`
+  })
+  fs.writeFileSync(envFilePath, envFile)
+}
+
+async function up() {
+  console.log("Spinning up your budibase dev environment... 🔧✨")
+  await init()
+  await compose.upAll(CONFIG)
+}
+
+async function down() {
+  console.log("Spinning down your budibase dev environment... 🌇")
+  await compose.stop(CONFIG)
+}
+
+async function nuke() {
+  console.log(
+    "Clearing down your budibase dev environment, including all containers and volumes... 💥"
+  )
+  await compose.down(CONFIG)
+}
+
+const managementCommand = process.argv.slice(2)[0]
+
+if (
+  !managementCommand ||
+  !Object.values(Commands).some(command => managementCommand === command)
+) {
+  throw new Error(
+    "You must supply either an 'up', 'down' or 'nuke' commmand to manage the budibase development environment."
+  )
+}
+
+let command
+switch (managementCommand) {
+  case Commands.Up:
+    command = up
+    break
+  case Commands.Down:
+    command = down
+    break
+  case Commands.Nuke:
+    command = nuke
+    break
+  default:
+    command = up
+}
+
+command()
+  .then(() => {
+    console.log("Done! 🎉")
+  })
+  .catch(err => {
+    console.error(
+      "Something went wrong while managing budibase dev environment:",
+      err.message
+    )
+  })
--- a/packages/server/scripts/exportAppTemplate.js
+++ b/packages/server/scripts/exportAppTemplate.js
@ -1,6 +1,10 @@
 #!/usr/bin/env node
-const { exportTemplateFromApp } = require("../src/utilities/templates")
 const yargs = require("yargs")
+const fs = require("fs")
+const { join } = require("path")
+const CouchDB = require("../src/db")
+// load environment
+const env = require("../src/environment")

 // Script to export a chosen budibase app into a package
 // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId
@ -22,18 +26,26 @@ yargs
      },
    },
    async args => {
+      if (!env.isDev()) {
+        throw "Only works in dev"
+      }
+      const name = args.name,
+        appId = args.appId
      console.log("Exporting app..")
-      if (args.name == null || args.appId == null) {
+      if (name == null || appId == null) {
        console.error(
          "Unable to export without a name and app ID being specified, check help for more info."
        )
        return
      }
-      const exportPath = await exportTemplateFromApp({
-        templateName: args.name,
-        appId: args.appId,
-      })
-      console.log(`Template ${args.name} exported to ${exportPath}`)
+      const exportPath = join(process.cwd(), name, "db")
+      fs.ensureDirSync(exportPath)
+      const writeStream = fs.createWriteStream(join(exportPath, "dump.text"))
+      // perform couch dump
+
+      const instanceDb = new CouchDB(appId)
+      await instanceDb.dump(writeStream, {})
+      console.log(`Template ${name} exported to ${exportPath}`)
    }
  )
  .help()
--- a/packages/server/scripts/initialise.js
+++ b/packages/server/scripts/initialise.js
@ -1,17 +0,0 @@
-const { join } = require("path")
-const { homedir } = require("os")
-
-const initialiseBudibase = require("../src/utilities/initialiseBudibase")
-const DIRECTORY = "~/.budibase"
-
-function run() {
-  let opts = {}
-  let dir = DIRECTORY
-  opts.quiet = true
-  opts.dir = dir.startsWith("~") ? join(homedir(), dir.substring(1)) : dir
-  return initialiseBudibase(opts)
-}
-
-run().then(() => {
-  console.log("Init complete.")
-})
--- a/packages/server/src/api/controllers/apikeys.js
+++ b/packages/server/src/api/controllers/apikeys.js
@ -1,56 +1,32 @@
-const fs = require("fs")
-const { join } = require("../../utilities/centralPath")
-const readline = require("readline")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
-const env = require("../../environment")
-const ENV_FILE_PATH = "/.env"
+const builderDB = require("../../db/builder")

 exports.fetch = async function(ctx) {
-  ctx.status = 200
-  ctx.body = {
-    budibase: env.BUDIBASE_API_KEY,
-    userId: env.USERID_API_KEY,
+  try {
+    const mainDoc = await builderDB.getBuilderMainDoc()
+    ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {}
+  } catch (err) {
+    /* istanbul ignore next */
+    ctx.throw(400, err)
  }
 }

 exports.update = async function(ctx) {
-  const key = `${ctx.params.key.toUpperCase()}_API_KEY`
+  const key = ctx.params.key
  const value = ctx.request.body.value

-  // set environment variables
-  env._set(key, value)
-
-  // Write to file
-  await updateValues([key, value])
-
-  ctx.status = 200
-  ctx.message = `Updated ${ctx.params.key} API key succesfully.`
-  ctx.body = { [ctx.params.key]: ctx.request.body.value }
-}
-
-async function updateValues([key, value]) {
-  let newContent = ""
-  let keyExists = false
-  let envPath = join(budibaseAppsDir(), ENV_FILE_PATH)
-  const readInterface = readline.createInterface({
-    input: fs.createReadStream(envPath),
-    output: process.stdout,
-    console: false,
-  })
-  readInterface.on("line", function(line) {
-    // Mutate lines and change API Key
-    if (line.startsWith(key)) {
-      line = `${key}=${value}`
-      keyExists = true
+  try {
+    const mainDoc = await builderDB.getBuilderMainDoc()
+    if (mainDoc.apiKeys == null) {
+      mainDoc.apiKeys = {}
    }
-    newContent = `${newContent}\n${line}`
-  })
-  readInterface.on("close", function() {
-    // Write file here
-    if (!keyExists) {
-      // Add API Key if it doesn't exist in the file at all
-      newContent = `${newContent}\n${key}=${value}`
+    mainDoc.apiKeys[key] = value
+    const resp = await builderDB.setBuilderMainDoc(mainDoc)
+    ctx.body = {
+      _id: resp.id,
+      _rev: resp.rev,
    }
-    fs.writeFileSync(envPath, newContent)
-  })
+  } catch (err) {
+    /* istanbul ignore next */
+    ctx.throw(400, err)
+  }
 }
--- a/packages/server/src/api/controllers/application.js
+++ b/packages/server/src/api/controllers/application.js
@ -1,15 +1,17 @@
 const CouchDB = require("../../db")
-const compileStaticAssets = require("../../utilities/builder/compileStaticAssets")
 const env = require("../../environment")
-const { existsSync } = require("fs-extra")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
 const setBuilderToken = require("../../utilities/builder/setBuilderToken")
-const fs = require("fs-extra")
-const { join, resolve } = require("../../utilities/centralPath")
 const packageJson = require("../../../package.json")
-const { createLinkView } = require("../../db/linkedRows")
-const { createRoutingView } = require("../../utilities/routing")
-const { downloadTemplate } = require("../../utilities/templates")
+const {
+  createLinkView,
+  createRoutingView,
+  createAllSearchIndex,
+} = require("../../db/views/staticViews")
+const {
+  getTemplateStream,
+  createApp,
+  deleteApp,
+} = require("../../utilities/fileSystem")
 const {
  generateAppID,
  getLayoutParams,
@ -20,9 +22,6 @@ const {
  BUILTIN_ROLE_IDS,
  AccessController,
 } = require("../../utilities/security/roles")
-const {
-  downloadExtractComponentLibraries,
-} = require("../../utilities/createAppPackage")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const {
  createHomeScreen,
@ -32,11 +31,7 @@ const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
 const { getAllApps } = require("../../utilities")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
-const {
-  getDeployedApps,
-  getHostingInfo,
-  HostingTypes,
-} = require("../../utilities/builder/hosting")
+const { getDeployedApps } = require("../../utilities/builder/hosting")

 const URL_REGEX_SLASH = /\/|\\/g

@ -75,8 +70,7 @@ async function getAppUrlIfNotInUse(ctx) {
    url = encodeURI(`${ctx.request.body.name}`)
  }
  url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
-  const hostingInfo = await getHostingInfo()
-  if (hostingInfo.type === HostingTypes.CLOUD) {
+  if (!env.SELF_HOSTED) {
    return url
  }
  const deployedApps = await getDeployedApps()
@ -101,21 +95,13 @@ async function createInstance(template) {
  // add view for linked rows
  await createLinkView(appId)
  await createRoutingView(appId)
+  await createAllSearchIndex(appId)

  // replicate the template data to the instance DB
  // this is currently very hard to test, downloading and importing template files
  /* istanbul ignore next */
  if (template && template.useTemplate === "true") {
-    let dbDumpReadStream
-    if (template.file) {
-      dbDumpReadStream = fs.createReadStream(template.file.path)
-    } else {
-      const templatePath = await downloadTemplate(...template.key.split("/"))
-      dbDumpReadStream = fs.createReadStream(
-        join(templatePath, "db", "dump.txt")
-      )
-    }
-    const { ok } = await db.load(dbDumpReadStream)
+    const { ok } = await db.load(await getTemplateStream(template))
    if (!ok) {
      throw "Error loading database dump from template."
    }
@ -190,10 +176,10 @@ exports.create = async function(ctx) {
  const instanceDb = new CouchDB(appId)
  await instanceDb.put(newApplication)

-  const newAppFolder = await createEmptyAppPackage(ctx, newApplication)
+  await createEmptyAppPackage(ctx, newApplication)
  /* istanbul ignore next */
-  if (env.NODE_ENV !== "jest") {
-    await downloadExtractComponentLibraries(newAppFolder)
+  if (!env.isTest()) {
+    await createApp(appId)
  }

  await setBuilderToken(ctx, appId, version)
@ -222,11 +208,10 @@ exports.delete = async function(ctx) {
  const db = new CouchDB(ctx.params.appId)
  const app = await db.get(ctx.params.appId)
  const result = await db.destroy()
-
-  // remove top level directory
-  await fs.rmdir(join(budibaseAppsDir(), ctx.params.appId), {
-    recursive: true,
-  })
+  /* istanbul ignore next */
+  if (!env.isTest()) {
+    await deleteApp(ctx.params.appId)
+  }

  ctx.status = 200
  ctx.message = `Application ${app.name} deleted successfully.`
@ -234,17 +219,8 @@ exports.delete = async function(ctx) {
 }

 const createEmptyAppPackage = async (ctx, app) => {
-  const appsFolder = budibaseAppsDir()
-  const newAppFolder = resolve(appsFolder, app._id)
-
  const db = new CouchDB(app._id)

-  if (existsSync(newAppFolder)) {
-    ctx.throw(400, "App folder already exists for this application")
-  }
-
-  fs.mkdirpSync(newAppFolder)
-
  let screensAndLayouts = []
  for (let layout of BASE_LAYOUTS) {
    const cloned = cloneDeep(layout)
@ -260,6 +236,4 @@ const createEmptyAppPackage = async (ctx, app) => {
  screensAndLayouts.push(loginScreen)

  await db.bulkDocs(screensAndLayouts)
-  await compileStaticAssets(app._id)
-  return newAppFolder
 }
--- a/packages/server/src/api/controllers/auth.js
+++ b/packages/server/src/api/controllers/auth.js
@ -45,9 +45,9 @@ exports.authenticate = async ctx => {
      roleId: dbUser.roleId,
      version: app.version,
    }
-    // if in cloud add the user api key, unless self hosted
+    // if in prod add the user api key, unless self hosted
    /* istanbul ignore next */
-    if (env.CLOUD && !env.SELF_HOSTED) {
+    if (env.isProd() && !env.SELF_HOSTED) {
      const { apiKey } = await getAPIKey(ctx.user.appId)
      payload.apiKey = apiKey
    }
--- a/packages/server/src/api/controllers/backup.js
+++ b/packages/server/src/api/controllers/backup.js
@ -1,28 +1,10 @@
-const { performDump } = require("../../utilities/templates")
-const path = require("path")
-const os = require("os")
-const fs = require("fs-extra")
+const { performBackup } = require("../../utilities/fileSystem")

 exports.exportAppDump = async function(ctx) {
  const { appId } = ctx.query
-
  const appname = decodeURI(ctx.query.appname)
-
-  const backupsDir = path.join(os.homedir(), ".budibase", "backups")
-  fs.ensureDirSync(backupsDir)
-
  const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt`

-  await performDump({
-    dir: backupsDir,
-    appId,
-    name: backupIdentifier,
-  })
-
-  ctx.status = 200
-
-  const backupFile = path.join(backupsDir, backupIdentifier)
-
  ctx.attachment(backupIdentifier)
-  ctx.body = fs.createReadStream(backupFile)
+  ctx.body = await performBackup(appId, backupIdentifier)
 }
--- a/packages/server/src/api/controllers/component.js
+++ b/packages/server/src/api/controllers/component.js
@ -1,44 +1,30 @@
 const CouchDB = require("../../db")
-const { resolve, join } = require("../../utilities/centralPath")
-const {
-  budibaseTempDir,
-  budibaseAppsDir,
-} = require("../../utilities/budibaseDir")
+const { getComponentLibraryManifest } = require("../../utilities/fileSystem")

 exports.fetchAppComponentDefinitions = async function(ctx) {
  const appId = ctx.params.appId || ctx.appId
  const db = new CouchDB(appId)
  const app = await db.get(appId)

-  ctx.body = app.componentLibraries.reduce((acc, componentLibrary) => {
-    let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules")
+  let componentManifests = await Promise.all(
+    app.componentLibraries.map(async library => {
+      let manifest = await getComponentLibraryManifest(appId, library)

-    if (ctx.isDev) {
-      appDirectory = budibaseTempDir()
-    }
-
-    const componentJson = require(join(
-      appDirectory,
-      componentLibrary,
-      ctx.isDev ? "" : "package",
-      "manifest.json"
-    ))
-
-    const result = {}
-
-    // map over the components.json and add the library identifier as a key
-    // button -> @budibase/standard-components/button
-    for (let key of Object.keys(componentJson)) {
-      const fullComponentName = `${componentLibrary}/${key}`.toLowerCase()
-      result[fullComponentName] = {
+      return {
+        manifest,
+        library,
+      }
+    })
+  )
+  const definitions = {}
+  for (let { manifest, library } of componentManifests) {
+    for (let key of Object.keys(manifest)) {
+      const fullComponentName = `${library}/${key}`.toLowerCase()
+      definitions[fullComponentName] = {
        component: fullComponentName,
-        ...componentJson[key],
+        ...manifest[key],
      }
    }
-
-    return {
-      ...acc,
-      ...result,
-    }
-  }, {})
+  }
+  ctx.body = definitions
 }
--- a/packages/server/src/api/controllers/deploy/awsDeploy.js
+++ b/packages/server/src/api/controllers/deploy/awsDeploy.js
@ -66,12 +66,7 @@ exports.deploy = async function(deployment) {
  const appId = deployment.getAppId()
  const { bucket, accountId } = deployment.getVerification()
  const metadata = { accountId }
-  const s3Client = new AWS.S3({
-    params: {
-      Bucket: bucket,
-    },
-  })
-  await deployToObjectStore(appId, s3Client, metadata)
+  await deployToObjectStore(appId, bucket, metadata)
 }

 exports.replicateDb = async function(deployment) {
--- a/packages/server/src/api/controllers/deploy/selfDeploy.js
+++ b/packages/server/src/api/controllers/deploy/selfDeploy.js
@ -7,7 +7,6 @@ const {
 const {
  getWorkerUrl,
  getCouchUrl,
-  getMinioUrl,
  getSelfHostKey,
 } = require("../../../utilities/builder/hosting")

@ -45,17 +44,9 @@ exports.postDeployment = async function() {
 exports.deploy = async function(deployment) {
  const appId = deployment.getAppId()
  const verification = deployment.getVerification()
-  const objClient = new AWS.S3({
-    endpoint: await getMinioUrl(),
-    s3ForcePathStyle: true, // needed with minio?
-    signatureVersion: "v4",
-    params: {
-      Bucket: verification.bucket,
-    },
-  })
  // no metadata, aws has account ID in metadata
  const metadata = {}
-  await deployToObjectStore(appId, objClient, metadata)
+  await deployToObjectStore(appId, verification.bucket, metadata)
 }

 exports.replicateDb = async function(deployment) {
--- a/packages/server/src/api/controllers/deploy/utils.js
+++ b/packages/server/src/api/controllers/deploy/utils.js
@ -1,16 +1,24 @@
-const fs = require("fs")
-const sanitize = require("sanitize-s3-objectkey")
-const { walkDir } = require("../../../utilities")
 const { join } = require("../../../utilities/centralPath")
+const fs = require("fs")
 const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
 const fetch = require("node-fetch")
 const PouchDB = require("../../../db")
 const CouchDB = require("pouchdb")
+const { upload } = require("../../../utilities/fileSystem")

-const CONTENT_TYPE_MAP = {
-  html: "text/html",
-  css: "text/css",
-  js: "application/javascript",
+// TODO: everything in this file is to be removed
+
+function walkDir(dirPath, callback) {
+  for (let filename of fs.readdirSync(dirPath)) {
+    const filePath = `${dirPath}/${filename}`
+    const stat = fs.lstatSync(filePath)
+
+    if (stat.isFile()) {
+      callback(filePath)
+    } else {
+      walkDir(filePath, callback)
+    }
+  }
 }

 exports.fetchCredentials = async function(url, body) {
@ -34,30 +42,25 @@ exports.fetchCredentials = async function(url, body) {
  return json
 }

-exports.prepareUpload = async function({ s3Key, metadata, client, file }) {
-  const extension = [...file.name.split(".")].pop()
-  const fileBytes = fs.readFileSync(file.path)
-
-  const upload = await client
-    .upload({
-      // windows file paths need to be converted to forward slashes for s3
-      Key: sanitize(s3Key).replace(/\\/g, "/"),
-      Body: fileBytes,
-      ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
-      Metadata: metadata,
-    })
-    .promise()
+exports.prepareUpload = async function({ s3Key, bucket, metadata, file }) {
+  const response = await upload({
+    bucket,
+    metadata,
+    filename: s3Key,
+    path: file.path,
+    type: file.type,
+  })

  return {
    size: file.size,
    name: file.name,
-    extension,
-    url: upload.Location,
-    key: upload.Key,
+    extension: [...file.name.split(".")].pop(),
+    url: response.Location,
+    key: response.Key,
  }
 }

-exports.deployToObjectStore = async function(appId, objectClient, metadata) {
+exports.deployToObjectStore = async function(appId, bucket, metadata) {
  const appAssetsPath = join(budibaseAppsDir(), appId, "public")

  let uploads = []
@ -66,12 +69,12 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) {
  walkDir(appAssetsPath, function(filePath) {
    const filePathParts = filePath.split("/")
    const appAssetUpload = exports.prepareUpload({
+      bucket,
      file: {
        path: filePath,
        name: filePathParts.pop(),
      },
      s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
-      client: objectClient,
      metadata,
    })
    uploads.push(appAssetUpload)
@ -92,7 +95,7 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) {
    const attachmentUpload = exports.prepareUpload({
      file,
      s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
-      client: objectClient,
+      bucket,
      metadata,
    })

--- a/packages/server/src/api/controllers/hosting.js
+++ b/packages/server/src/api/controllers/hosting.js
@ -1,11 +1,11 @@
 const CouchDB = require("../../db")
-const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
 const {
  getHostingInfo,
  getDeployedApps,
  HostingTypes,
  getAppUrl,
 } = require("../../utilities/builder/hosting")
+const { StaticDatabases } = require("../../db/utils")

 exports.fetchInfo = async ctx => {
  ctx.body = {
@ -14,17 +14,17 @@ exports.fetchInfo = async ctx => {
 }

 exports.save = async ctx => {
-  const db = new CouchDB(BUILDER_CONFIG_DB)
+  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
  const { type } = ctx.request.body
  if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
    ctx.body = await db.remove({
      ...ctx.request.body,
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
    })
  } else {
    ctx.body = await db.put({
      ...ctx.request.body,
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
    })
  }
 }
--- a/packages/server/src/api/controllers/query.js
+++ b/packages/server/src/api/controllers/query.js
@ -93,7 +93,7 @@ exports.find = async function(ctx) {
  const db = new CouchDB(ctx.user.appId)
  const query = enrichQueries(await db.get(ctx.params.queryId))
  // remove properties that could be dangerous in real app
-  if (env.CLOUD) {
+  if (env.isProd()) {
    delete query.fields
    delete query.parameters
    delete query.schema
--- a/packages/server/src/api/controllers/row.js
+++ b/packages/server/src/api/controllers/row.js
@ -17,6 +17,7 @@ const {
 const { FieldTypes } = require("../../constants")
 const { isEqual } = require("lodash")
 const { cloneDeep } = require("lodash/fp")
+const { QueryBuilder, search } = require("./search/utils")

 const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}`

@ -259,39 +260,46 @@ exports.search = async function(ctx) {
  const db = new CouchDB(appId)
  const {
    query,
-    pagination: { pageSize = 10, page },
+    pagination: { pageSize = 10, bookmark },
  } = ctx.request.body
+  const tableId = ctx.params.tableId

-  // make all strings a starts with operation rather than pure equality
-  for (const [key, queryVal] of Object.entries(query)) {
-    if (typeof queryVal === "string") {
-      query[key] = {
-        $gt: queryVal,
-        $lt: `${queryVal}\uffff`,
-      }
-    }
+  const queryBuilder = new QueryBuilder(appId)
+    .setLimit(pageSize)
+    .addTable(tableId)
+  if (bookmark) {
+    queryBuilder.setBookmark(bookmark)
  }

-  // pure equality for table
-  query.tableId = ctx.params.tableId
-  const response = await db.find({
-    selector: query,
-    limit: pageSize,
-    skip: pageSize * page,
-  })
+  let searchString
+  if (ctx.query && ctx.query.raw && ctx.query.raw !== "") {
+    searchString = queryBuilder.complete(query["RAW"])
+  } else {
+    // make all strings a starts with operation rather than pure equality
+    for (const [key, queryVal] of Object.entries(query)) {
+      if (typeof queryVal === "string") {
+        queryBuilder.addString(key, queryVal)
+      } else {
+        queryBuilder.addEqual(key, queryVal)
+      }
+    }
+    searchString = queryBuilder.complete()
+  }

-  const rows = response.docs
+  const response = await search(searchString)

  // delete passwords from users
-  if (query.tableId === ViewNames.USERS) {
-    for (let row of rows) {
+  if (tableId === ViewNames.USERS) {
+    for (let row of response.rows) {
      delete row.password
    }
  }

-  const table = await db.get(ctx.params.tableId)
-
-  ctx.body = await outputProcessing(appId, table, rows)
+  const table = await db.get(tableId)
+  ctx.body = {
+    rows: await outputProcessing(appId, table, response.rows),
+    bookmark: response.bookmark,
+  }
 }

 exports.fetchTableRows = async function(ctx) {
--- a/packages/server/src/api/controllers/search/index.js
+++ b/packages/server/src/api/controllers/search/index.js
@ -0,0 +1,18 @@
+const { QueryBuilder, buildSearchUrl, search } = require("./utils")
+
+exports.rowSearch = async ctx => {
+  // this can't be done through pouch, have to reach for trusty node-fetch
+  const appId = ctx.user.appId
+  const bookmark = ctx.params.bookmark
+  let url
+  if (ctx.params.query) {
+    url = new QueryBuilder(appId, ctx.params.query, bookmark).complete()
+  } else if (ctx.params.raw) {
+    url = buildSearchUrl({
+      appId,
+      query: ctx.params.raw,
+      bookmark,
+    })
+  }
+  ctx.body = await search(url)
+}
--- a/packages/server/src/api/controllers/search/utils.js
+++ b/packages/server/src/api/controllers/search/utils.js
@ -0,0 +1,137 @@
+const { SearchIndexes } = require("../../../db/utils")
+const { checkSlashesInUrl } = require("../../../utilities")
+const env = require("../../../environment")
+const fetch = require("node-fetch")
+
+/**
+ * Given a set of inputs this will generate the URL which is to be sent to the search proxy in CouchDB.
+ * @param {string} appId The ID of the app which we will be searching within.
+ * @param {string} query The lucene query string which is to be used for searching.
+ * @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was
+ * returned with query for next set of search results.
+ * @param {number} limit The number of entries to return per query.
+ * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled.
+ * @return {string} The URL which a GET can be performed on to receive results.
+ */
+function buildSearchUrl({ appId, query, bookmark, excludeDocs, limit = 50 }) {
+  let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search`
+  url += `/${SearchIndexes.ROWS}?q=${query}`
+  url += `&limit=${limit}`
+  if (!excludeDocs) {
+    url += "&include_docs=true"
+  }
+  if (bookmark) {
+    url += `&bookmark=${bookmark}`
+  }
+  return checkSlashesInUrl(url)
+}
+
+class QueryBuilder {
+  constructor(appId, base) {
+    this.appId = appId
+    this.query = {
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      ...base,
+    }
+    this.limit = 50
+    this.bookmark = null
+  }
+
+  setLimit(limit) {
+    this.limit = limit
+    return this
+  }
+
+  setBookmark(bookmark) {
+    this.bookmark = bookmark
+    return this
+  }
+
+  addString(key, partial) {
+    this.query.string[key] = partial
+    return this
+  }
+
+  addFuzzy(key, fuzzy) {
+    this.query.fuzzy[key] = fuzzy
+    return this
+  }
+
+  addRange(key, low, high) {
+    this.query.range = {
+      low,
+      high,
+    }
+    return this
+  }
+
+  addEqual(key, value) {
+    this.query.equal[key] = value
+    return this
+  }
+
+  addTable(tableId) {
+    this.query.equal.tableId = tableId
+    return this
+  }
+
+  complete(rawQuery = null) {
+    let output = ""
+    function build(structure, queryFn) {
+      for (let [key, value] of Object.entries(structure)) {
+        if (output.length !== 0) {
+          output += " AND "
+        }
+        output += queryFn(key, value)
+      }
+    }
+
+    if (this.query.string) {
+      build(this.query.string, (key, value) => `${key}:${value}*`)
+    }
+    if (this.query.range) {
+      build(
+        this.query.range,
+        (key, value) => `${key}:[${value.low} TO ${value.high}]`
+      )
+    }
+    if (this.query.fuzzy) {
+      build(this.query.fuzzy, (key, value) => `${key}:${value}~`)
+    }
+    if (this.query.equal) {
+      build(this.query.equal, (key, value) => `${key}:${value}`)
+    }
+    if (rawQuery) {
+      output = output.length === 0 ? rawQuery : `&${rawQuery}`
+    }
+    return buildSearchUrl({
+      appId: this.appId,
+      query: output,
+      bookmark: this.bookmark,
+      limit: this.limit,
+    })
+  }
+}
+
+exports.search = async query => {
+  const response = await fetch(query, {
+    method: "GET",
+  })
+  const json = await response.json()
+  let output = {
+    rows: [],
+  }
+  if (json.rows != null && json.rows.length > 0) {
+    output.rows = json.rows.map(row => row.doc)
+  }
+  if (json.bookmark) {
+    output.bookmark = json.bookmark
+  }
+  return output
+}
+
+exports.QueryBuilder = QueryBuilder
+exports.buildSearchUrl = buildSearchUrl
--- a/packages/server/src/api/controllers/static/index.js
+++ b/packages/server/src/api/controllers/static/index.js
@ -2,32 +2,47 @@ require("svelte/register")

 const send = require("koa-send")
 const { resolve, join } = require("../../../utilities/centralPath")
+const { checkSlashesInUrl } = require("../../../utilities")
 const fetch = require("node-fetch")
-const fs = require("fs-extra")
 const uuid = require("uuid")
-const AWS = require("aws-sdk")
 const { prepareUpload } = require("../deploy/utils")
 const { processString } = require("@budibase/string-templates")
-const {
-  budibaseAppsDir,
-  budibaseTempDir,
-} = require("../../../utilities/budibaseDir")
+const { budibaseTempDir } = require("../../../utilities/budibaseDir")
 const { getDeployedApps } = require("../../../utilities/builder/hosting")
 const CouchDB = require("../../../db")
 const setBuilderToken = require("../../../utilities/builder/setBuilderToken")
-const fileProcessor = require("../../../utilities/fileProcessor")
+const { loadHandlebarsFile } = require("../../../utilities/fileSystem")
 const env = require("../../../environment")
 const { OBJ_STORE_DIRECTORY } = require("../../../constants")
+const fileProcessor = require("../../../utilities/fileSystem/processor")
+
+const BB_CDN = "https://cdn.app.budi.live/assets"

 function objectStoreUrl() {
  if (env.SELF_HOSTED) {
    // can use a relative url for this as all goes through the proxy (this is hosted in minio)
    return OBJ_STORE_DIRECTORY
  } else {
-    return "https://cdn.app.budi.live/assets"
+    return BB_CDN
  }
 }

+function internalObjectStoreUrl() {
+  if (env.SELF_HOSTED) {
+    return checkSlashesInUrl(env.MINIO_URL + OBJ_STORE_DIRECTORY)
+  } else {
+    return BB_CDN
+  }
+}
+
+async function returnObjectStoreFile(ctx, path) {
+  const S3_URL = `${internalObjectStoreUrl()}/${path}`
+  const response = await fetch(S3_URL)
+  const body = await response.text()
+  ctx.set("Content-Type", response.headers.get("Content-Type"))
+  ctx.body = body
+}
+
 async function checkForSelfHostedURL(ctx) {
  // the "appId" component of the URL may actually be a specific self hosted URL
  let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
@ -50,106 +65,31 @@ exports.serveBuilder = async function(ctx) {
  await send(ctx, ctx.file, { root: ctx.devPath || builderPath })
 }

-exports.serveSelfHostPage = async function(ctx) {
-  const logo = fs.readFileSync(resolve(__dirname, "selfhost/logo.svg"), "utf8")
-  const hostingHbs = fs.readFileSync(
-    resolve(__dirname, "selfhost/index.hbs"),
-    "utf8"
-  )
-  ctx.body = await processString(hostingHbs, {
-    logo,
-  })
-}
-
 exports.uploadFile = async function(ctx) {
-  let files
-  files =
+  let files =
    ctx.request.files.file.length > 1
      ? Array.from(ctx.request.files.file)
      : [ctx.request.files.file]

-  const attachmentsPath = resolve(
-    budibaseAppsDir(),
-    ctx.user.appId,
-    "attachments"
-  )
-
-  if (env.CLOUD) {
-    // remote upload
-    const s3 = new AWS.S3({
-      params: {
-        Bucket: "prod-budi-app-assets",
-      },
-    })
-
-    const uploads = files.map(file => {
-      const fileExtension = [...file.name.split(".")].pop()
-      const processedFileName = `${uuid.v4()}.${fileExtension}`
-
-      return prepareUpload({
-        file,
-        s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
-        s3,
-      })
-    })
-
-    ctx.body = await Promise.all(uploads)
-    return
-  }
-
-  ctx.body = await processLocalFileUploads({
-    files,
-    outputPath: attachmentsPath,
-    appId: ctx.user.appId,
-  })
-}
-
-async function processLocalFileUploads({ files, outputPath, appId }) {
-  // create attachments dir if it doesnt exist
-  !fs.existsSync(outputPath) && fs.mkdirSync(outputPath, { recursive: true })
-
-  const filesToProcess = files.map(file => {
+  const uploads = files.map(async file => {
    const fileExtension = [...file.name.split(".")].pop()
    // filenames converted to UUIDs so they are unique
    const processedFileName = `${uuid.v4()}.${fileExtension}`

-    return {
-      name: file.name,
-      path: file.path,
-      size: file.size,
-      type: file.type,
-      processedFileName,
+    // need to handle image processing
+    await fileProcessor.process({
+      ...file,
      extension: fileExtension,
-      outputPath: join(outputPath, processedFileName),
-      url: join("/attachments", processedFileName),
-    }
+    })
+
+    return prepareUpload({
+      file,
+      s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
+      bucket: "prod-budi-app-assets",
+    })
  })

-  const fileProcessOperations = filesToProcess.map(fileProcessor.process)
-
-  const processedFiles = await Promise.all(fileProcessOperations)
-
-  let pendingFileUploads
-  // local document used to track which files need to be uploaded
-  // db.get throws an error if the document doesn't exist
-  // need to use a promise to default
-  const db = new CouchDB(appId)
-  await db
-    .get("_local/fileuploads")
-    .then(data => {
-      pendingFileUploads = data
-    })
-    .catch(() => {
-      pendingFileUploads = { _id: "_local/fileuploads", uploads: [] }
-    })
-
-  pendingFileUploads.uploads = [
-    ...processedFiles,
-    ...pendingFileUploads.uploads,
-  ]
-  await db.put(pendingFileUploads)
-
-  return processedFiles
+  ctx.body = await Promise.all(uploads)
 }

 exports.serveApp = async function(ctx) {
@ -163,12 +103,12 @@ exports.serveApp = async function(ctx) {

  const { head, html, css } = App.render({
    title: appInfo.name,
-    production: env.CLOUD,
+    production: env.isProd(),
    appId,
    objectStoreUrl: objectStoreUrl(),
  })

-  const appHbs = fs.readFileSync(`${__dirname}/templates/app.hbs`, "utf8")
+  const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
  ctx.body = await processString(appHbs, {
    head,
    body: html,
@ -178,71 +118,47 @@ exports.serveApp = async function(ctx) {
 }

 exports.serveAttachment = async function(ctx) {
-  const appId = ctx.user.appId
-  const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments")
-
-  // Serve from object store
-  if (env.CLOUD) {
-    const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file)
-    const response = await fetch(S3_URL)
-    const body = await response.text()
-    ctx.set("Content-Type", response.headers.get("Content-Type"))
-    ctx.body = body
-    return
-  }
-
-  await send(ctx, ctx.file, { root: attachmentsPath })
+  await returnObjectStoreFile(
+    ctx,
+    join(ctx.user.appId, "attachments", ctx.file)
+  )
 }

 exports.serveAppAsset = async function(ctx) {
-  // default to homedir
-  const appPath = resolve(budibaseAppsDir(), ctx.user.appId, "public")
-
-  await send(ctx, ctx.file, { root: ctx.devPath || appPath })
+  if (env.isDev() || env.isTest()) {
+    return send(ctx, ctx.file, { root: budibaseTempDir() })
+  }
+  await returnObjectStoreFile(ctx, join(ctx.user.appId, "public", ctx.file))
 }

 exports.serveComponentLibrary = async function(ctx) {
  const appId = ctx.query.appId || ctx.appId
-  // default to homedir
-  let componentLibraryPath = resolve(
-    budibaseAppsDir(),
-    appId,
-    "node_modules",
-    decodeURI(ctx.query.library),
-    "package",
-    "dist"
-  )

-  if (ctx.isDev) {
-    componentLibraryPath = join(
+  if (env.isDev() || env.isTest()) {
+    const componentLibraryPath = join(
      budibaseTempDir(),
      decodeURI(ctx.query.library),
      "dist"
    )
+    return send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
  }
-
-  if (env.CLOUD) {
-    let componentLib = "componentlibrary"
-    if (ctx.user.version) {
-      componentLib += `-${ctx.user.version}`
-    } else {
-      componentLib += `-${COMP_LIB_BASE_APP_VERSION}`
-    }
-    const S3_URL = encodeURI(
-      join(
-        objectStoreUrl(appId),
-        componentLib,
-        ctx.query.library,
-        "dist",
-        "index.js"
-      )
+  let componentLib = "componentlibrary"
+  if (ctx.user.version) {
+    componentLib += `-${ctx.user.version}`
+  } else {
+    componentLib += `-${COMP_LIB_BASE_APP_VERSION}`
+  }
+  const S3_URL = encodeURI(
+    join(
+      objectStoreUrl(appId),
+      componentLib,
+      ctx.query.library,
+      "dist",
+      "index.js"
    )
-    const response = await fetch(S3_URL)
-    const body = await response.text()
-    ctx.type = "application/javascript"
-    ctx.body = body
-    return
-  }
-
-  await send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
+  )
+  const response = await fetch(S3_URL)
+  const body = await response.text()
+  ctx.type = "application/javascript"
+  ctx.body = body
 }
--- a/packages/server/src/api/controllers/static/selfhost/index.hbs
+++ b/packages/server/src/api/controllers/static/selfhost/index.hbs
@ -1,173 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <title>Budibase self hosting️</title>
-  <style>
-    body {
-      font-family: Inter, -apple-system, BlinkMacSystemFont, Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
-      height: 100%;
-      width: 100%;
-      margin: 0;
-      padding: 0;
-      background: #fafafa;
-    }
-
-    .main {
-      padding: 0 20px;
-      margin: 30px auto;
-      width: 60%;
-    }
-
-    h2 {
-      font-size: clamp(24px, 1.5vw, 30px);
-      text-align: center;
-      line-height: 1.3;
-      font-weight: bold;
-    }
-
-    .card-grid {
-      display: grid;
-      grid-template-columns: 1fr 1fr;
-      gap: 3rem;
-    }
-
-    .card {
-      display: grid;
-      background-color: #222222;
-      grid-template-columns: 1fr;
-      align-items: center;
-      padding: 2.5rem 1.75rem;
-      border-radius: 12px;
-      color: white;
-    }
-
-    .card h3 {
-      margin: 0;
-      font-size: 24px;
-      font-family: sans-serif;
-      font-weight: 600;
-    }
-
-    .card h3 b {
-      text-wrap: normal;
-      font-size: 36px;
-      padding-right: 14px;
-    }
-
-    .card p {
-      color: #ffffff;
-      opacity: 0.8;
-      font-size: 18px;
-      text-align: left;
-      line-height: 1.3;
-      margin-top: 1rem;
-    }
-
-    .logo {
-      width: 60px;
-      height: 60px;
-      margin: auto;
-    }
-
-    .top-text {
-      text-align: center;
-      color: #707070;
-      margin: 0 0 1.5rem 0;
-    }
-
-    .button {
-      cursor: pointer;
-      display: block;
-      background: #4285f4;
-      color: white;
-      padding: 12px 16px;
-      font-size: 16px;
-      font-weight: 600;
-      border-radius: 6px;
-      max-width: 120px;
-      text-align: center;
-      transition: 200ms background ease;
-      text-decoration: none;
-    }
-
-    .info {
-      background: #f5f5f5;
-      padding: 1rem 1rem 1rem 1rem;
-      border: #ccc 1px solid;
-      border-radius: 6px;
-      margin-top: 40px;
-      display: flex;
-      align-items: center;
-    }
-
-    .info p {
-      margin-left: 20px;
-      color: #222222;
-      font-family: sans-serif;
-    }
-
-    .info p {
-      margin-right: 20px;
-    }
-
-    .info svg {
-      margin-left: 20px;
-    }
-
-    .info a {
-      color: #4285f4;
-    }
-  </style>
-</head>
-<body>
-  <div class="main">
-    <div class="logo">
-      {{logo}}
-    </div>
-    <h2>Get started with Budibase Self Hosting</h2>
-    <p class="top-text">Use the address <b id="url"></b> in your Builder</p>
-    <div class="card-grid">
-      <div class="card">
-        <h3><b>📚</b>Documentation</h3>
-        <p>
-          Find out more about your self hosted platform.
-        </p>
-        <a class="button"
-           href="https://docs.budibase.com/self-hosting/introduction-to-self-hosting">
-          Documentation
-        </a>
-      </div>
-      <div class="card">
-        <h3><b>💻</b>Next steps</h3>
-        <p>
-          Find out how to make use of your self hosted Budibase platform.
-        </p>
-        <a class="button"
-           href="https://docs.budibase.com/self-hosting/builder-settings">
-          Next steps
-        </a>
-      </div>
-    </div>
-    <div class="info">
-      <svg preserveAspectRatio="xMidYMid meet" height="28px" width="28px" fill="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"
-           xmlns:xlink="http://www.w3.org/1999/xlink" stroke="none" class="icon-7f6730be--text-3f89f380">
-        <g>
-          <path d="M12.2 8.98c.06-.01.12-.03.18-.06.06-.02.12-.05.18-.09l.15-.12c.18-.19.29-.45.29-.71 0-.06-.01-.13-.02-.19a.603.603 0 0 0-.06-.19.757.757 0 0 0-.09-.18c-.03-.05-.08-.1-.12-.15-.28-.27-.72-.37-1.09-.21-.13.05-.23.12-.33.21-.04.05-.09.1-.12.15-.04.06-.07.12-.09.18-.03.06-.05.12-.06.19-.01.06-.02.13-.02.19 0 .26.11.52.29.71.1.09.2.16.33.21.12.05.25.08.38.08.06 0 .13-.01.2-.02M13 16v-4a1 1 0 1 0-2 0v4a1 1 0 1 0 2 0M12 3c-4.962 0-9 4.038-9 9 0 4.963 4.038 9 9 9 4.963 0 9-4.037 9-9 0-4.962-4.037-9-9-9m0 20C5.935 23 1 18.065 1 12S5.935 1 12 1c6.066 0 11 4.935 11 11s-4.934 11-11 11" fill-rule="evenodd">
-          </path>
-        </g>
-      </svg>
-      <p>A <b>Hosting Key</b> will also be required, this can be found in your hosting properties, info found <a href="https://docs.budibase.com/self-hosting/hosting-settings">here</a>.</p>
-    </div>
-  </div>
-  <script>
-    window.addEventListener("load", () => {
-      let url = document.URL.split("//")[1]
-      if (url.substring(url.length - 1) === "/") {
-        url = url.substring(0, url.length - 1)
-      }
-      document.getElementById("url").innerHTML = url
-    })
-  </script>
-</body>
-</html>
--- a/packages/server/src/api/controllers/static/selfhost/logo.svg
+++ b/packages/server/src/api/controllers/static/selfhost/logo.svg
@ -1,17 +0,0 @@
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" x="0" y="0" viewBox="0 0 48 48" xml:space="preserve">
-  <style>
-    .st0{fill:#393c44}.st1{fill:#fff}
-  </style>
-    <path class="st0" d="M-152.17-24.17H-56V72h-96.17z"/>
-    <path class="st1" d="M-83.19 48h-41.79c-1.76 0-3.19-1.43-3.19-3.19V3.02c0-1.76 1.43-3.19 3.19-3.19h41.79c1.76 0 3.19 1.43 3.19 3.19v41.79c0 1.76-1.43 3.19-3.19 3.19z"/>
-    <path class="st0" d="M-99.62 12.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V35h-4.89V12.57h4.9zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .17-.39.26-.79.26-1.2z"/>
-    <path class="st0" d="M-114.76 12.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V35h-4.89V12.57h4.9zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .18-.39.26-.79.26-1.2z"/>
-    <path d="M44.81 159H3.02c-1.76 0-3.19-1.43-3.19-3.19v-41.79c0-1.76 1.43-3.19 3.19-3.19h41.79c1.76 0 3.19 1.43 3.19 3.19v41.79c0 1.76-1.43 3.19-3.19 3.19z" fill="#4285f4"/>
-    <path class="st1" d="M28.38 123.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V146h-4.89v-22.43h4.9zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .17-.39.26-.79.26-1.2z"/>
-    <path class="st1" d="M13.24 123.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V146H8.35v-22.43h4.89zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .18-.39.26-.79.26-1.2z"/>
-    <g>
-    <path class="st0" d="M44 48H4c-2.21 0-4-1.79-4-4V4c0-2.21 1.79-4 4-4h40c2.21 0 4 1.79 4 4v40c0 2.21-1.79 4-4 4z"/>
-        <path class="st1" d="M28.48 12v10.44c1.18-1.27 2.65-1.9 4.42-1.9 1.05 0 2.01.2 2.89.61.87.41 1.62.96 2.24 1.65.62.69 1.1 1.5 1.45 2.44.35.94.52 1.93.52 2.99 0 1.08-.18 2.09-.54 3.04-.36.95-.86 1.77-1.51 2.47-.64.7-1.4 1.25-2.28 1.66-.87.4-1.81.6-2.83.6-1.84 0-3.3-.69-4.37-2.07v1.62h-5V12h5.01zm6.3 16.31c0-.45-.08-.88-.25-1.29-.17-.41-.4-.76-.69-1.06-.3-.3-.64-.54-1.02-.72-.39-.18-.81-.27-1.27-.27-.44 0-.86.09-1.24.26-.39.17-.72.41-1.01.71-.29.3-.52.66-.69 1.06-.18.41-.26.84-.26 1.29s.08.88.25 1.28c.17.4.4.74.69 1.04.29.29.64.53 1.04.71.4.18.82.27 1.26.27.44 0 .86-.09 1.24-.26.39-.17.72-.41 1.01-.71.29-.3.52-.65.69-1.05.16-.41.25-.82.25-1.26z"/>
-        <path class="st1" d="M13 12v10.44c1.18-1.27 2.65-1.9 4.42-1.9 1.05 0 2.01.2 2.89.61.87.41 1.62.96 2.24 1.65.62.69 1.1 1.5 1.45 2.44.35.94.52 1.93.52 2.99 0 1.08-.18 2.09-.54 3.04-.36.95-.86 1.77-1.51 2.47-.64.7-1.4 1.25-2.28 1.66-.87.4-1.81.6-2.82.6-1.84 0-3.3-.69-4.37-2.07v1.62H8V12h5zm6.3 16.31c0-.45-.08-.88-.25-1.29-.17-.41-.4-.76-.69-1.06-.3-.3-.64-.54-1.02-.72-.39-.18-.81-.27-1.27-.27-.44 0-.86.09-1.24.26-.39.17-.72.41-1.01.71-.29.3-.52.66-.69 1.06-.18.41-.26.84-.26 1.29s.08.88.25 1.28c.17.4.4.74.69 1.04.29.29.64.53 1.04.71.4.18.82.27 1.26.27.44 0 .86-.09 1.24-.26.39-.17.72-.41 1.01-.71.29-.3.52-.65.69-1.05.16-.41.25-.82.25-1.26z"/>
-  </g>
-</svg>
--- a/packages/server/src/api/controllers/templates.js
+++ b/packages/server/src/api/controllers/templates.js
@ -1,10 +1,5 @@
 const fetch = require("node-fetch")
-const {
-  downloadTemplate,
-  exportTemplateFromApp,
-  getLocalTemplates,
-} = require("../../utilities/templates")
-const env = require("../../environment")
+const { downloadTemplate } = require("../../utilities/fileSystem")

 // development flag, can be used to test against templates exported locally
 const DEFAULT_TEMPLATES_BUCKET =
@ -12,16 +7,11 @@ const DEFAULT_TEMPLATES_BUCKET =

 exports.fetch = async function(ctx) {
  const { type = "app" } = ctx.query
-
-  if (env.LOCAL_TEMPLATES) {
-    ctx.body = Object.values(getLocalTemplates()[type])
-  } else {
-    const response = await fetch(
-      `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
-    )
-    const json = await response.json()
-    ctx.body = Object.values(json.templates[type])
-  }
+  const response = await fetch(
+    `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
+  )
+  const json = await response.json()
+  ctx.body = Object.values(json.templates[type])
 }

 // can't currently test this, have to ignore from coverage
@ -29,26 +19,9 @@ exports.fetch = async function(ctx) {
 exports.downloadTemplate = async function(ctx) {
  const { type, name } = ctx.params

-  if (!env.LOCAL_TEMPLATES) {
-    await downloadTemplate(type, name)
-  }
+  await downloadTemplate(type, name)

  ctx.body = {
    message: `template ${type}:${name} downloaded successfully.`,
  }
 }
-
-exports.exportTemplateFromApp = async function(ctx) {
-  const { appId } = ctx.user
-  const { templateName } = ctx.request.body
-
-  await exportTemplateFromApp({
-    appId,
-    templateName,
-  })
-
-  ctx.status = 200
-  ctx.body = {
-    message: `Created template: ${templateName}`,
-  }
-}
--- a/packages/server/src/api/controllers/view/index.js
+++ b/packages/server/src/api/controllers/view/index.js
@ -1,8 +1,6 @@
 const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
-const fs = require("fs")
-const { join } = require("../../../utilities/centralPath")
-const os = require("os")
+const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
 const { fetchView } = require("../row")
 const { ViewNames } = require("../../../db/utils")
@ -120,12 +118,10 @@ const controller = {
    // Export part
    let headers = Object.keys(schema)
    const exporter = exporters[format]
-    const exportedFile = exporter(headers, ctx.body)
    const filename = `${viewName}.${format}`
-    fs.writeFileSync(join(os.tmpdir(), filename), exportedFile)
-
+    // send down the file
    ctx.attachment(filename)
-    ctx.body = fs.createReadStream(join(os.tmpdir(), filename))
+    ctx.body = apiFileReturn(exporter(headers, ctx.body))
  },
 }

--- a/packages/server/src/api/index.js
+++ b/packages/server/src/api/index.js
@ -2,8 +2,6 @@ const Router = require("@koa/router")
 const authenticated = require("../middleware/authenticated")
 const compress = require("koa-compress")
 const zlib = require("zlib")
-const { budibaseAppsDir } = require("../utilities/budibaseDir")
-const { isDev } = require("../utilities")
 const { mainRoutes, authRoutes, staticRoutes } = require("./routes")
 const pkg = require("../../package.json")

@ -25,11 +23,9 @@ router
  )
  .use(async (ctx, next) => {
    ctx.config = {
-      latestPackagesFolder: budibaseAppsDir(),
      jwtSecret: env.JWT_SECRET,
      useAppRootPath: true,
    }
-    ctx.isDev = isDev()
    await next()
  })
  .use("/health", ctx => (ctx.status = 200))
@ -68,8 +64,6 @@ for (let route of mainRoutes) {
 router.use(staticRoutes.routes())
 router.use(staticRoutes.allowedMethods())

-if (!env.SELF_HOSTED && !env.CLOUD) {
-  router.redirect("/", "/_builder")
-}
+router.redirect("/", "/_builder")

 module.exports = router
--- a/packages/server/src/api/routes/search.js
+++ b/packages/server/src/api/routes/search.js
@ -0,0 +1,8 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/search")
+
+const router = Router()
+
+router.get("/api/search/rows", controller.rowSearch)
+
+module.exports = router
--- a/packages/server/src/api/routes/static.js
+++ b/packages/server/src/api/routes/static.js
@ -1,34 +1,32 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/static")
 const { budibaseTempDir } = require("../../utilities/budibaseDir")
-const env = require("../../environment")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("../../utilities/security/permissions")
 const usage = require("../../middleware/usageQuota")
+const env = require("../../environment")

 const router = Router()

 /* istanbul ignore next */
 router.param("file", async (file, ctx, next) => {
  ctx.file = file && file.includes(".") ? file : "index.html"
-
-  // Serving the client library from your local dir in dev
-  if (ctx.isDev && ctx.file.startsWith("budibase-client")) {
+  if (!ctx.file.startsWith("budibase-client")) {
+    return next()
+  }
+  // test serves from require
+  if (env.isTest()) {
+    ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0]
+  } else if (env.isDev()) {
+    // Serving the client library from your local dir in dev
    ctx.devPath = budibaseTempDir()
  }
-
-  await next()
+  return next()
 })

-if (env.NODE_ENV !== "production") {
-  router.get("/_builder/:file*", controller.serveBuilder)
-}
-
-if (env.SELF_HOSTED) {
-  router.get("/", controller.serveSelfHostPage)
-}
-
 router
+  // TODO: for now this _builder endpoint is not authorized/secured, will need to be
+  .get("/_builder/:file*", controller.serveBuilder)
  .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
  .post("/api/attachments/upload", usage, controller.uploadFile)
  .get("/componentlibrary", controller.serveComponentLibrary)
--- a/packages/server/src/api/routes/templates.js
+++ b/packages/server/src/api/routes/templates.js
@ -12,6 +12,5 @@ router
    authorized(BUILDER),
    controller.downloadTemplate
  )
-  .post("/api/templates", authorized(BUILDER), controller.exportTemplateFromApp)

 module.exports = router
--- a/packages/server/src/api/routes/tests/apikeys.spec.js
+++ b/packages/server/src/api/routes/tests/apikeys.spec.js
@ -1,8 +1,5 @@
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const fs = require("fs")
-const path = require("path")

 describe("/api/keys", () => {
  let request = setup.getRequest()
@ -16,12 +13,14 @@ describe("/api/keys", () => {

  describe("fetch", () => {
    it("should allow fetching", async () => {
-      const res = await request
-        .get(`/api/keys`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body).toBeDefined()
+      await setup.switchToSelfHosted(async () => {
+        const res = await request
+          .get(`/api/keys`)
+          .set(config.defaultHeaders())
+          .expect("Content-Type", /json/)
+          .expect(200)
+        expect(res.body).toBeDefined()
+      })
    })

    it("should check authorization for builder", async () => {
@ -35,17 +34,18 @@ describe("/api/keys", () => {

  describe("update", () => {
    it("should allow updating a value", async () => {
-      fs.writeFileSync(path.join(budibaseAppsDir(), ".env"), "TEST_API_KEY=thing")
-      const res = await request
-        .put(`/api/keys/TEST`)
-        .send({
-          value: "test"
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body["TEST"]).toEqual("test")
-      expect(process.env.TEST_API_KEY).toEqual("test")
+      await setup.switchToSelfHosted(async () => {
+        const res = await request
+          .put(`/api/keys/TEST`)
+          .send({
+            value: "test"
+          })
+          .set(config.defaultHeaders())
+          .expect("Content-Type", /json/)
+          .expect(200)
+        expect(res.body._id).toBeDefined()
+        expect(res.body._rev).toBeDefined()
+      })
    })

    it("should check authorization for builder", async () => {
--- a/packages/server/src/api/routes/tests/auth.spec.js
+++ b/packages/server/src/api/routes/tests/auth.spec.js
@ -1,4 +1,3 @@
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")

 describe("/authenticate", () => {
--- a/packages/server/src/api/routes/tests/backup.spec.js
+++ b/packages/server/src/api/routes/tests/backup.spec.js
@ -1,6 +1,8 @@
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")

+jest.mock("../../../utilities/fileSystem/utilities")
+
 describe("/backups", () => {
  let request = setup.getRequest()
  let config = setup.getConfig()
@ -14,7 +16,7 @@ describe("/backups", () => {
  describe("exportAppDump", () => {
    it("should be able to export app", async () => {
      const res = await request
-        .get(`/api/backups/export?appId=${config.getAppId()}`)
+        .get(`/api/backups/export?appId=${config.getAppId()}&appname=test`)
        .set(config.defaultHeaders())
        .expect(200)
      expect(res.text).toBeDefined()
--- a/packages/server/src/api/routes/tests/cloud.spec.js
+++ b/packages/server/src/api/routes/tests/cloud.spec.js
@ -1,16 +0,0 @@
-const setup = require("./utilities")
-
-describe("test things in the Cloud/Self hosted", () => {
-  describe("test self hosted static page", () => {
-    it("should be able to load the static page", async () => {
-      await setup.switchToCloudForFunction(async () => {
-        let request = setup.getRequest()
-        let config = setup.getConfig()
-        await config.init()
-        const res = await request.get(`/`).expect(200)
-        expect(res.text.includes("<title>Budibase self hosting️</title>")).toEqual(true)
-        setup.afterAll()
-      })
-    })
-  })
-})
--- a/packages/server/src/api/routes/tests/component.spec.js
+++ b/packages/server/src/api/routes/tests/component.spec.js
@ -1,8 +1,5 @@
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
-const fs = require("fs")
-const { resolve, join } = require("path")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")

 describe("/component", () => {
  let request = setup.getRequest()
@ -14,23 +11,8 @@ describe("/component", () => {
    await config.init()
  })

-  function mock() {
-    const manifestFile = "manifest.json"
-    const appId = config.getAppId()
-    const libraries = [join("@budibase", "standard-components")]
-    for (let library of libraries) {
-      let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules", library, "package")
-      fs.mkdirSync(appDirectory, { recursive: true })
-
-      const file = require.resolve(library).split(join("dist", "index.js"))[0] + manifestFile
-      fs.copyFileSync(file, join(appDirectory, manifestFile))
-    }
-  }
-
  describe("fetch definitions", () => {
    it("should be able to fetch definitions", async () => {
-      // have to "mock" the files required
-      mock()
      const res = await request
        .get(`/${config.getAppId()}/components/definitions`)
        .set(config.defaultHeaders())
--- a/packages/server/src/api/routes/tests/hosting.spec.js
+++ b/packages/server/src/api/routes/tests/hosting.spec.js
@ -107,17 +107,16 @@ describe("/hosting", () => {
  })

  describe("getDeployedApps", () => {
-    it("should get apps when in builder", async () => {
-      const res = await request
+    it("should fail when not self hosted", async () => {
+      await request
        .get(`/api/hosting/apps`)
        .set(config.defaultHeaders())
        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.app1).toEqual({url: "/app1"})
+        .expect(400)
    })

    it("should get apps when in cloud", async () => {
-      await setup.switchToCloudForFunction(async () => {
+      await setup.switchToSelfHosted(async () => {
        const res = await request
          .get(`/api/hosting/apps`)
          .set(config.defaultHeaders())
--- a/packages/server/src/api/routes/tests/query.spec.js
+++ b/packages/server/src/api/routes/tests/query.spec.js
@ -89,7 +89,7 @@ describe("/queries", () => {
    })

    it("should find a query in cloud", async () => {
-      await setup.switchToCloudForFunction(async () => {
+      await setup.switchToSelfHosted(async () => {
        const query = await config.createQuery()
        const res = await request
          .get(`/api/queries/${query._id}`)
--- a/packages/server/src/api/routes/tests/row.spec.js
+++ b/packages/server/src/api/routes/tests/row.spec.js
@ -2,6 +2,9 @@ const { outputProcessing } = require("../../../utilities/rowProcessor")
 const setup = require("./utilities")
 const { basicRow } = setup.structures

+// mock the fetch for the search system
+jest.mock("node-fetch")
+
 describe("/rows", () => {
  let request = setup.getRequest()
  let config = setup.getConfig()
@ -303,25 +306,19 @@ describe("/rows", () => {

  describe("search", () => {
    it("should run a search on the table", async () => {
-      const row = await config.createRow()
-      // add another row that shouldn't be found
-      await config.createRow({
-        ...basicRow(),
-        name: "Other Contact",
-      })
      const res = await request
        .post(`/api/${table._id}/rows/search`)
        .send({
          query: {
            name: "Test",
          },
-          pagination: { pageSize: 25, page: 0 }
+          pagination: { pageSize: 25 }
        })
        .set(config.defaultHeaders())
        .expect('Content-Type', /json/)
        .expect(200)
-      expect(res.body.length).toEqual(1)
-      expect(res.body[0]._id).toEqual(row._id)
+      expect(res.body.rows.length).toEqual(1)
+      expect(res.body.bookmark).toBeDefined()
    })
  })

@ -410,7 +407,7 @@ describe("/rows", () => {
        tableId: table._id,
      })
      // the environment needs configured for this
-      await setup.switchToCloudForFunction(async () => {
+      await setup.switchToSelfHosted(async () => {
        const enriched = await outputProcessing(config.getAppId(), table, [row])
        expect(enriched[0].attachment[0].url).toBe(`/app-assets/assets/${config.getAppId()}/test/thing`)
      })
--- a/packages/server/src/api/routes/tests/templates.spec.js
+++ b/packages/server/src/api/routes/tests/templates.spec.js
@ -1,7 +1,4 @@
 const setup = require("./utilities")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const fs = require("fs")
-const { join } = require("path")

 describe("/templates", () => {
  let request = setup.getRequest()
@ -24,26 +21,4 @@ describe("/templates", () => {
      expect(Array.isArray(res.body)).toEqual(true)
    })
  })
-
-  describe("export", () => {
-    it("should be able to export the basic app", async () => {
-      const res = await request
-        .post(`/api/templates`)
-        .send({
-          templateName: "test",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.message).toEqual("Created template: test")
-      const dir = join(
-        budibaseAppsDir(),
-        "templates",
-        "app",
-        "test",
-        "db"
-      )
-      expect(fs.existsSync(dir)).toEqual(true)
-    })
-  })
 })
--- a/packages/server/src/api/routes/tests/utilities/index.js
+++ b/packages/server/src/api/routes/tests/utilities/index.js
@ -35,18 +35,18 @@ exports.getConfig = () => {
  return config
 }

-exports.switchToCloudForFunction = async func => {
+exports.switchToSelfHosted = async func => {
  // self hosted stops any attempts to Dynamo
-  env.CLOUD = true
-  env.SELF_HOSTED = true
+  env._set("NODE_ENV", "production")
+  env._set("SELF_HOSTED", true)
  let error
  try {
    await func()
  } catch (err) {
    error = err
  }
-  env.CLOUD = false
-  env.SELF_HOSTED = false
+  env._set("NODE_ENV", "jest")
+  env._set("SELF_HOSTED", false)
  // don't throw error until after reset
  if (error) {
    throw error
--- a/packages/server/src/app.js
+++ b/packages/server/src/app.js
@ -9,6 +9,7 @@ const env = require("./environment")
 const eventEmitter = require("./events")
 const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
+const fileSystem = require("./utilities/fileSystem")

 const app = new Koa()

@ -65,6 +66,7 @@ module.exports = server.listen(env.PORT || 0, async () => {
  console.log(`Budibase running on ${JSON.stringify(server.address())}`)
  env._set("PORT", server.address().port)
  eventEmitter.emitPort(env.PORT)
+  fileSystem.init()
  await automations.init()
 })

--- a/packages/server/src/automations/actions.js
+++ b/packages/server/src/automations/actions.js
@ -5,17 +5,12 @@ const deleteRow = require("./steps/deleteRow")
 const createUser = require("./steps/createUser")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const env = require("../environment")
-const download = require("download")
-const fetch = require("node-fetch")
-const { join } = require("../utilities/centralPath")
-const os = require("os")
-const fs = require("fs")
 const Sentry = require("@sentry/node")
+const {
+  automationInit,
+  getExternalAutomationStep,
+} = require("../utilities/fileSystem")

-const DEFAULT_BUCKET =
-  "https://prod-budi-automations.s3-eu-west-1.amazonaws.com"
-const DEFAULT_DIRECTORY = ".budibase-automations"
-const AUTOMATION_MANIFEST = "manifest.json"
 const BUILTIN_ACTIONS = {
  SEND_EMAIL: sendEmail.run,
  CREATE_ROW: createRow.run,
@ -33,8 +28,6 @@ const BUILTIN_DEFINITIONS = {
  OUTGOING_WEBHOOK: outgoingWebhook.definition,
 }

-let AUTOMATION_BUCKET = env.AUTOMATION_BUCKET
-let AUTOMATION_DIRECTORY = env.AUTOMATION_DIRECTORY
 let MANIFEST = null

 /* istanbul ignore next */
@ -42,22 +35,13 @@ function buildBundleName(pkgName, version) {
  return `${pkgName}@${version}.min.js`
 }

-/* istanbul ignore next */
-async function downloadPackage(name, version, bundleName) {
-  await download(
-    `${AUTOMATION_BUCKET}/${name}/${version}/${bundleName}`,
-    AUTOMATION_DIRECTORY
-  )
-  return require(join(AUTOMATION_DIRECTORY, bundleName))
-}
-
 /* istanbul ignore next */
 module.exports.getAction = async function(actionName) {
  if (BUILTIN_ACTIONS[actionName] != null) {
    return BUILTIN_ACTIONS[actionName]
  }
  // worker pools means that a worker may not have manifest
-  if (env.CLOUD && MANIFEST == null) {
+  if (env.isProd() && MANIFEST == null) {
    MANIFEST = await module.exports.init()
  }
  // env setup to get async packages
@ -66,28 +50,12 @@ module.exports.getAction = async function(actionName) {
  }
  const pkg = MANIFEST.packages[actionName]
  const bundleName = buildBundleName(pkg.stepId, pkg.version)
-  try {
-    return require(join(AUTOMATION_DIRECTORY, bundleName))
-  } catch (err) {
-    return downloadPackage(pkg.stepId, pkg.version, bundleName)
-  }
+  return getExternalAutomationStep(pkg.stepId, pkg.version, bundleName)
 }

 module.exports.init = async function() {
-  // set defaults
-  if (!AUTOMATION_DIRECTORY) {
-    AUTOMATION_DIRECTORY = join(os.homedir(), DEFAULT_DIRECTORY)
-  }
-  if (!AUTOMATION_BUCKET) {
-    AUTOMATION_BUCKET = DEFAULT_BUCKET
-  }
-  if (!fs.existsSync(AUTOMATION_DIRECTORY)) {
-    fs.mkdirSync(AUTOMATION_DIRECTORY, { recursive: true })
-  }
-  // env setup to get async packages
  try {
-    let response = await fetch(`${AUTOMATION_BUCKET}/${AUTOMATION_MANIFEST}`)
-    MANIFEST = await response.json()
+    MANIFEST = await automationInit()
    module.exports.DEFINITIONS =
      MANIFEST && MANIFEST.packages
        ? Object.assign(MANIFEST.packages, BUILTIN_DEFINITIONS)
--- a/packages/server/src/automations/index.js
+++ b/packages/server/src/automations/index.js
@ -34,10 +34,10 @@ module.exports.init = async function() {
  await actions.init()
  triggers.automationQueue.process(async job => {
    try {
-      if (env.CLOUD && job.data.automation && !env.SELF_HOSTED) {
+      if (env.USE_QUOTAS) {
        job.data.automation.apiKey = await updateQuota(job.data.automation)
      }
-      if (env.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
+      if (env.isProd()) {
        await runWorker(job)
      } else {
        await singleThread(job)
--- a/packages/server/src/automations/steps/createRow.js
+++ b/packages/server/src/automations/steps/createRow.js
@ -85,7 +85,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) {
      inputs.row.tableId,
      inputs.row
    )
-    if (env.CLOUD) {
+    if (env.isProd()) {
      await usage.update(apiKey, usage.Properties.ROW, 1)
    }
    await rowController.save(ctx)
--- a/packages/server/src/automations/steps/createUser.js
+++ b/packages/server/src/automations/steps/createUser.js
@ -72,7 +72,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) {
  }

  try {
-    if (env.CLOUD) {
+    if (env.isProd()) {
      await usage.update(apiKey, usage.Properties.USER, 1)
    }
    await userController.create(ctx)
--- a/packages/server/src/automations/steps/deleteRow.js
+++ b/packages/server/src/automations/steps/deleteRow.js
@ -70,7 +70,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) {
  }

  try {
-    if (env.CLOUD) {
+    if (env.isProd()) {
      await usage.update(apiKey, usage.Properties.ROW, -1)
    }
    await rowController.destroy(ctx)
--- a/packages/server/src/automations/tests/automation.spec.js
+++ b/packages/server/src/automations/tests/automation.spec.js
@ -47,27 +47,23 @@ describe("Run through some parts of the automations system", () => {
    expect(thread).toHaveBeenCalled()
  })

-  it("should be able to init in cloud", async () => {
-    env.CLOUD = true
-    env.BUDIBASE_ENVIRONMENT = "PRODUCTION"
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
-    await wait(100)
-    // haven't added a mock implementation so getAPIKey of usageQuota just returns undefined
-    expect(usageQuota.update).toHaveBeenCalledWith("test", "automationRuns", 1)
-    expect(workerJob).toBeDefined()
-    env.BUDIBASE_ENVIRONMENT = "JEST"
-    env.CLOUD = false
+  it("should be able to init in prod", async () => {
+    await setup.runInProd(async () => {
+      await triggers.externalTrigger(basicAutomation(), { a: 1 })
+      await wait(100)
+      // haven't added a mock implementation so getAPIKey of usageQuota just returns undefined
+      expect(usageQuota.update).toHaveBeenCalledWith("test", "automationRuns", 1)
+      expect(workerJob).toBeDefined()
+    })
  })

  it("try error scenario", async () => {
-    env.CLOUD = true
-    env.BUDIBASE_ENVIRONMENT = "PRODUCTION"
-    // the second call will throw an error
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
-    await wait(100)
-    expect(console.error).toHaveBeenCalled()
-    env.BUDIBASE_ENVIRONMENT = "JEST"
-    env.CLOUD = false
+    await setup.runInProd(async () => {
+      // the second call will throw an error
+      await triggers.externalTrigger(basicAutomation(), { a: 1 })
+      await wait(100)
+      expect(console.error).toHaveBeenCalled()
+    })
  })

  it("should be able to check triggering row filling", async () => {
--- a/packages/server/src/automations/tests/createRow.spec.js
+++ b/packages/server/src/automations/tests/createRow.spec.js
@ -42,12 +42,12 @@ describe("test the create row action", () => {
  })

  it("check usage quota attempts", async () => {
-    env.CLOUD = true
-    await setup.runStep(setup.actions.CREATE_ROW.stepId, {
-      row
+    await setup.runInProd(async () => {
+      await setup.runStep(setup.actions.CREATE_ROW.stepId, {
+        row
+      })
+      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1)
    })
-    expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1)
-    env.CLOUD = false
  })

  it("should check invalid inputs return an error", async () => {
--- a/packages/server/src/automations/tests/createUser.spec.js
+++ b/packages/server/src/automations/tests/createUser.spec.js
@ -35,9 +35,9 @@ describe("test the create user action", () => {
  })

  it("check usage quota attempts", async () => {
-    env.CLOUD = true
-    await setup.runStep(setup.actions.CREATE_USER.stepId, user)
-    expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "users", 1)
-    env.CLOUD = false
+    await setup.runInProd(async () => {
+      await setup.runStep(setup.actions.CREATE_USER.stepId, user)
+      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "users", 1)
+    })
  })
 })
--- a/packages/server/src/automations/tests/deleteRow.spec.js
+++ b/packages/server/src/automations/tests/deleteRow.spec.js
@ -36,10 +36,10 @@ describe("test the delete row action", () => {
  })

  it("check usage quota attempts", async () => {
-    env.CLOUD = true
-    await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
-    expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1)
-    env.CLOUD = false
+    await setup.runInProd(async () => {
+      await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
+      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1)
+    })
  })

  it("should check invalid inputs return an error", async () => {
--- a/packages/server/src/automations/tests/utilities/index.js
+++ b/packages/server/src/automations/tests/utilities/index.js
@ -2,6 +2,7 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const actions = require("../../actions")
 const logic = require("../../logic")
 const emitter = require("../../../events/index")
+const env = require("../../../environment")

 let config

@ -16,6 +17,22 @@ exports.afterAll = () => {
  config.end()
 }

+exports.runInProd = async fn => {
+  env._set("NODE_ENV", "production")
+  env._set("USE_QUOTAS", 1)
+  let error
+  try {
+    await fn()
+  } catch (err) {
+    error = err
+  }
+  env._set("NODE_ENV", "jest")
+  env._set("USE_QUOTAS", null)
+  if (error) {
+    throw error
+  }
+}
+
 exports.runStep = async function runStep(stepId, inputs) {
  let step
  if (
--- a/packages/server/src/constants/index.js
+++ b/packages/server/src/constants/index.js
@ -80,8 +80,6 @@ exports.AutoFieldSubTypes = {
  AUTO_ID: "autoID",
 }

-exports.BUILDER_CONFIG_DB = "builder-config-db"
-exports.HOSTING_DOC = "hosting-doc"
 exports.OBJ_STORE_DIRECTORY = "/app-assets/assets"
 exports.BaseQueryVerbs = {
  CREATE: "create",
@ -89,3 +87,9 @@ exports.BaseQueryVerbs = {
  UPDATE: "update",
  DELETE: "delete",
 }
+
+exports.ObjectStoreBuckets = {
+  BACKUPS: "backups",
+  APPS: "prod-budi-app-assets",
+  TEMPLATES: "templates",
+}
--- a/packages/server/src/db/builder.js
+++ b/packages/server/src/db/builder.js
@ -0,0 +1,38 @@
+const CouchDB = require("./index")
+const { StaticDatabases } = require("./utils")
+const env = require("../environment")
+
+const SELF_HOST_ERR = "Unable to access builder DB/doc - not self hosted."
+const BUILDER_DB = StaticDatabases.BUILDER
+
+/**
+ * This is the builder database, right now this is a single, static database
+ * that is present across the whole system and determines some core functionality
+ * for the builder (e.g. storage of API keys). This has been limited to self hosting
+ * as it doesn't make as much sense against the currently design Cloud system.
+ */
+
+exports.getBuilderMainDoc = async () => {
+  if (!env.SELF_HOSTED) {
+    throw SELF_HOST_ERR
+  }
+  const db = new CouchDB(BUILDER_DB.name)
+  try {
+    return await db.get(BUILDER_DB.baseDoc)
+  } catch (err) {
+    // doesn't exist yet, nothing to get
+    return {
+      _id: BUILDER_DB.baseDoc,
+    }
+  }
+}
+
+exports.setBuilderMainDoc = async doc => {
+  if (!env.SELF_HOSTED) {
+    throw SELF_HOST_ERR
+  }
+  // make sure to override the ID
+  doc._id = BUILDER_DB.baseDoc
+  const db = new CouchDB(BUILDER_DB.name)
+  return db.put(doc)
+}
--- a/packages/server/src/db/client.js
+++ b/packages/server/src/db/client.js
@ -1,12 +1,10 @@
 const PouchDB = require("pouchdb")
 const replicationStream = require("pouchdb-replication-stream")
 const allDbs = require("pouchdb-all-dbs")
-const { budibaseAppsDir } = require("../utilities/budibaseDir")
 const find = require("pouchdb-find")
 const env = require("../environment")

-const COUCH_DB_URL = env.COUCH_DB_URL || `leveldb://${budibaseAppsDir()}/.data/`
-const isInMemory = env.NODE_ENV === "jest"
+const COUCH_DB_URL = env.COUCH_DB_URL || "http://localhost:10000/db/"

 PouchDB.plugin(replicationStream.plugin)
 PouchDB.plugin(find)
@ -14,10 +12,10 @@ PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)

 let POUCH_DB_DEFAULTS = {
  prefix: COUCH_DB_URL,
-  skip_setup: !!env.CLOUD,
+  skip_setup: env.isProd(),
 }

-if (isInMemory) {
+if (env.isTest()) {
  PouchDB.plugin(require("pouchdb-adapter-memory"))
  POUCH_DB_DEFAULTS = {
    prefix: undefined,
--- a/packages/server/src/db/dynamoClient.js
+++ b/packages/server/src/db/dynamoClient.js
@ -1,4 +1,4 @@
-let _ = require("lodash")
+let { merge } = require("lodash")
 let env = require("../environment")

 const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
@ -38,7 +38,7 @@ class Table {
      params.Key[this._sort] = sort
    }
    if (otherProps) {
-      params = _.merge(params, otherProps)
+      params = merge(params, otherProps)
    }
    let response = await docClient.get(params).promise()
    return response.Item
@ -77,7 +77,7 @@ class Table {
      params.ConditionExpression += "attribute_exists(#PRIMARY)"
    }
    if (otherProps) {
-      params = _.merge(params, otherProps)
+      params = merge(params, otherProps)
    }
    return docClient.update(params).promise()
  }
@ -94,7 +94,7 @@ class Table {
      Item: item,
    }
    if (otherProps) {
-      params = _.merge(params, otherProps)
+      params = merge(params, otherProps)
    }
    return docClient.put(params).promise()
  }
@ -119,7 +119,7 @@ exports.init = endpoint => {
 exports.apiKeyTable = new Table(TableInfo.API_KEYS)
 exports.userTable = new Table(TableInfo.USERS)

-if (env.CLOUD) {
+if (env.isProd()) {
  exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`)
 } else {
  env._set("AWS_ACCESS_KEY_ID", "KEY_ID")
--- a/packages/server/src/db/linkedRows/index.js
+++ b/packages/server/src/db/linkedRows/index.js
@ -27,7 +27,7 @@ const EventType = {
 }

 exports.EventType = EventType
-// re-export utils here for ease of use
+// re-export search here for ease of use
 exports.IncludeDocs = IncludeDocs
 exports.getLinkDocuments = getLinkDocuments
 exports.createLinkView = createLinkView
--- a/packages/server/src/db/utils.js
+++ b/packages/server/src/db/utils.js
@ -3,6 +3,18 @@ const newid = require("./newid")
 const UNICODE_MAX = "\ufff0"
 const SEPARATOR = "_"

+const StaticDatabases = {
+  BUILDER: {
+    name: "builder-db",
+    baseDoc: "builder-doc",
+  },
+  // TODO: needs removed
+  BUILDER_HOSTING: {
+    name: "builder-config-db",
+    baseDoc: "hosting-doc",
+  },
+}
+
 const DocumentTypes = {
  TABLE: "ta",
  ROW: "ro",
@ -25,10 +37,16 @@ const ViewNames = {
  USERS: "ta_users",
 }

+const SearchIndexes = {
+  ROWS: "rows",
+}
+
+exports.StaticDatabases = StaticDatabases
 exports.ViewNames = ViewNames
 exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR
 exports.UNICODE_MAX = UNICODE_MAX
+exports.SearchIndexes = SearchIndexes

 exports.getQueryIndex = viewName => {
  return `database/${viewName}`
--- a/packages/server/src/db/views/staticViews.js
+++ b/packages/server/src/db/views/staticViews.js
@ -1,5 +1,10 @@
 const CouchDB = require("../index")
-const { DocumentTypes, SEPARATOR, ViewNames } = require("../utils")
+const {
+  DocumentTypes,
+  SEPARATOR,
+  ViewNames,
+  SearchIndexes,
+} = require("../utils")
 const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR

 /**************************************************
@ -72,3 +77,42 @@ exports.createRoutingView = async appId => {
  }
  await db.put(designDoc)
 }
+
+async function searchIndex(appId, indexName, fnString) {
+  const db = new CouchDB(appId)
+  const designDoc = await db.get("_design/database")
+  designDoc.indexes = {
+    [indexName]: {
+      index: fnString,
+    },
+  }
+  await db.put(designDoc)
+}
+
+exports.createAllSearchIndex = async appId => {
+  await searchIndex(
+    appId,
+    SearchIndexes.ROWS,
+    function(doc) {
+      function idx(input, prev) {
+        for (let key of Object.keys(input)) {
+          const idxKey = prev != null ? `${prev}.${key}` : key
+          if (key === "_id" || key === "_rev") {
+            continue
+          }
+          if (typeof input[key] !== "object") {
+            // eslint-disable-next-line no-undef
+            index(idxKey, input[key], { store: true })
+          } else {
+            idx(input[key], idxKey)
+          }
+        }
+      }
+      if (doc._id.startsWith("ro_")) {
+        // eslint-disable-next-line no-undef
+        index("default", doc._id)
+        idx(doc)
+      }
+    }.toString()
+  )
+}
--- a/packages/server/src/electron.js
+++ b/packages/server/src/electron.js
@ -1,112 +1,112 @@
-const { app, BrowserWindow, shell, dialog } = require("electron")
-const { join } = require("./utilities/centralPath")
-const isDev = require("electron-is-dev")
-const { autoUpdater } = require("electron-updater")
-const unhandled = require("electron-unhandled")
-const { existsSync } = require("fs-extra")
-const initialiseBudibase = require("./utilities/initialiseBudibase")
-const { budibaseAppsDir } = require("./utilities/budibaseDir")
-const { openNewGitHubIssue, debugInfo } = require("electron-util")
-const eventEmitter = require("./events")
+// const { app, BrowserWindow, shell, dialog } = require("electron")
+// const { join } = require("./utilities/centralPath")
+// const isDev = require("electron-is-dev")
+// const { autoUpdater } = require("electron-updater")
+// const unhandled = require("electron-unhandled")
+// const { existsSync } = require("fs-extra")
+// const initialiseBudibase = require("./utilities/initialiseBudibase")
+// const { budibaseAppsDir } = require("./utilities/budibaseDir")
+// const { openNewGitHubIssue, debugInfo } = require("electron-util")
+// const eventEmitter = require("./events")

-const budibaseDir = budibaseAppsDir()
-const envFile = join(budibaseDir, ".env")
+// const budibaseDir = budibaseAppsDir()
+// const envFile = join(budibaseDir, ".env")

-async function startApp() {
-  if (!existsSync(envFile)) {
-    await initialiseBudibase({ dir: budibaseDir })
-  }
-  // evict environment from cache, so it reloads when next asked
-  delete require.cache[require.resolve("./environment")]
-  // store the port incase its going to get overridden
-  const port = process.env.PORT
-  require("dotenv").config({ path: envFile })
-  // overwrite the port - don't want to use dotenv for the port
-  require("./environment")._set("PORT", port)
+// async function startApp() {
+//   if (!existsSync(envFile)) {
+//     await initialiseBudibase({ dir: budibaseDir })
+//   }
+//   // evict environment from cache, so it reloads when next asked
+//   delete require.cache[require.resolve("./environment")]
+//   // store the port incase its going to get overridden
+//   const port = process.env.PORT
+//   require("dotenv").config({ path: envFile })
+//   // overwrite the port - don't want to use dotenv for the port
+//   require("./environment")._set("PORT", port)

-  unhandled({
-    showDialog: true,
-    reportButton: error => {
-      openNewGitHubIssue({
-        title: error.message,
-        user: "Budibase",
-        labels: ["error-report"],
-        repo: "budibase",
-        body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${
-          error.stack
-        }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`,
-      })
-    },
-  })
+//   unhandled({
+//     showDialog: true,
+//     reportButton: error => {
+//       openNewGitHubIssue({
+//         title: error.message,
+//         user: "Budibase",
+//         labels: ["error-report"],
+//         repo: "budibase",
+//         body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${
+//           error.stack
+//         }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`,
+//       })
+//     },
+//   })

-  let win
+//   let win

-  function handleRedirect(e, url) {
-    e.preventDefault()
-    shell.openExternal(url)
-  }
+//   function handleRedirect(e, url) {
+//     e.preventDefault()
+//     shell.openExternal(url)
+//   }

-  async function createWindow() {
-    app.server = require("./app")
-    eventEmitter.on("internal:port", port => {
-      const APP_URL = `http://localhost:${port}/_builder`
-      const APP_TITLE = "Budibase Builder"
-      win = new BrowserWindow({
-        width: 1920,
-        height: 1080,
-        icon: join(__dirname, "..", "build", "icons", "512x512.png"),
-      })
-      win.setTitle(APP_TITLE)
-      win.loadURL(APP_URL)
-      if (isDev) {
-        win.webContents.openDevTools()
-      } else {
-        autoUpdater.checkForUpdatesAndNotify()
-      }
+//   async function createWindow() {
+//     app.server = require("./app")
+//     eventEmitter.on("internal:port", port => {
+//       const APP_URL = `http://localhost:${port}/_builder`
+//       const APP_TITLE = "Budibase Builder"
+//       win = new BrowserWindow({
+//         width: 1920,
+//         height: 1080,
+//         icon: join(__dirname, "..", "build", "icons", "512x512.png"),
+//       })
+//       win.setTitle(APP_TITLE)
+//       win.loadURL(APP_URL)
+//       if (isDev) {
+//         win.webContents.openDevTools()
+//       } else {
+//         autoUpdater.checkForUpdatesAndNotify()
+//       }

-      // open _blank in default browser
-      win.webContents.on("new-window", handleRedirect)
-      win.webContents.on("will-navigate", handleRedirect)
-    })
-  }
+//       // open _blank in default browser
+//       win.webContents.on("new-window", handleRedirect)
+//       win.webContents.on("will-navigate", handleRedirect)
+//     })
+//   }

-  app.whenReady().then(createWindow)
+//   app.whenReady().then(createWindow)

-  // Quit when all windows are closed.
-  app.on("window-all-closed", () => {
-    // On macOS it is common for applications and their menu bar
-    // to stay active until the user quits explicitly with Cmd + Q
-    if (process.platform !== "darwin") {
-      app.server.close()
-      app.quit()
-    }
-  })
+//   // Quit when all windows are closed.
+//   app.on("window-all-closed", () => {
+//     // On macOS it is common for applications and their menu bar
+//     // to stay active until the user quits explicitly with Cmd + Q
+//     if (process.platform !== "darwin") {
+//       app.server.close()
+//       app.quit()
+//     }
+//   })

-  app.on("activate", () => {
-    // On macOS it's common to re-create a window in the app when the
-    // dock icon is clicked and there are no other windows open.
-    if (win === null) createWindow()
-  })
-}
+//   app.on("activate", () => {
+//     // On macOS it's common to re-create a window in the app when the
+//     // dock icon is clicked and there are no other windows open.
+//     if (win === null) createWindow()
+//   })
+// }

-autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => {
-  const dialogOpts = {
-    type: "info",
-    buttons: ["Restart", "Later"],
-    title: "Budibase Update Available",
-    message: process.platform === "win32" ? releaseNotes : releaseName,
-    detail:
-      "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.",
-  }
+// autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => {
+//   const dialogOpts = {
+//     type: "info",
+//     buttons: ["Restart", "Later"],
+//     title: "Budibase Update Available",
+//     message: process.platform === "win32" ? releaseNotes : releaseName,
+//     detail:
+//       "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.",
+//   }

-  dialog.showMessageBox(dialogOpts).then(returnValue => {
-    if (returnValue.response === 0) autoUpdater.quitAndInstall()
-  })
-})
+//   dialog.showMessageBox(dialogOpts).then(returnValue => {
+//     if (returnValue.response === 0) autoUpdater.quitAndInstall()
+//   })
+// })

-autoUpdater.on("error", message => {
-  console.error("There was a problem updating the application")
-  console.error(message)
-})
+// autoUpdater.on("error", message => {
+//   console.error("There was a problem updating the application")
+//   console.error(message)
+// })

-startApp()
+// startApp()
--- a/packages/server/src/environment.js
+++ b/packages/server/src/environment.js
@ -1,45 +1,64 @@
-const { resolve, join } = require("./utilities/centralPath")
-const { homedir } = require("os")
-const { app } = require("electron")
+function isTest() {
+  return (
+    process.env.NODE_ENV === "jest" ||
+    process.env.NODE_ENV === "cypress" ||
+    process.env.JEST_WORKER_ID != null
+  )
+}
+
+function isDev() {
+  return (
+    process.env.NODE_ENV !== "production" &&
+    process.env.BUDIBASE_ENVIRONMENT !== "production"
+  )
+}

 let LOADED = false
-
-if (!LOADED) {
-  const homeDir = app ? app.getPath("home") : homedir()
-  const budibaseDir = join(homeDir, ".budibase")
-  process.env.BUDIBASE_DIR = budibaseDir
-  require("dotenv").config({ path: resolve(budibaseDir, ".env") })
+if (!LOADED && isDev() && !isTest()) {
+  require("dotenv").config()
  LOADED = true
 }

 module.exports = {
-  CLIENT_ID: process.env.CLIENT_ID,
-  NODE_ENV: process.env.NODE_ENV,
-  JWT_SECRET: process.env.JWT_SECRET,
-  BUDIBASE_DIR: process.env.BUDIBASE_DIR,
+  // important
  PORT: process.env.PORT,
+  JWT_SECRET: process.env.JWT_SECRET,
  COUCH_DB_URL: process.env.COUCH_DB_URL,
+  MINIO_URL: process.env.MINIO_URL,
+  WORKER_URL: process.env.WORKER_URL,
+  SELF_HOSTED: process.env.SELF_HOSTED,
+  AWS_REGION: process.env.AWS_REGION,
+  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
+  MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  USE_QUOTAS: process.env.USE_QUOTAS,
+  // environment
+  NODE_ENV: process.env.NODE_ENV,
+  JEST_WORKER_ID: process.env.JEST_WORKER_ID,
+  BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
+  // minor
  SALT_ROUNDS: process.env.SALT_ROUNDS,
  LOGGER: process.env.LOGGER,
  LOG_LEVEL: process.env.LOG_LEVEL,
  AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
  AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
-  BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
  SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
-  CLOUD: process.env.CLOUD,
-  SELF_HOSTED: process.env.SELF_HOSTED,
-  WORKER_URL: process.env.WORKER_URL,
-  HOSTING_KEY: process.env.HOSTING_KEY,
  DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
-  AWS_REGION: process.env.AWS_REGION,
-  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
+  // old - to remove
+  CLIENT_ID: process.env.CLIENT_ID,
+  BUDIBASE_DIR: process.env.BUDIBASE_DIR,
+  DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
  BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY,
  USERID_API_KEY: process.env.USERID_API_KEY,
-  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
-  DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
-  LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES,
+  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
+  HOSTING_KEY: process.env.HOSTING_KEY,
  _set(key, value) {
    process.env[key] = value
    module.exports[key] = value
  },
+  isTest,
+  isDev,
+  isProd: () => {
+    return !isDev()
+  },
 }
--- a/packages/server/src/index.js
+++ b/packages/server/src/index.js
@ -1,17 +1,10 @@
-const { budibaseTempDir } = require("./utilities/budibaseDir")
-const { isDev } = require("./utilities")
-
 const fixPath = require("fix-path")
-const fs = require("fs")
-
-async function runServer() {
-  if (isDev() && !fs.existsSync(budibaseTempDir())) {
-    console.error(
-      "Please run a build before attempting to run server independently to fill 'tmp' directory."
-    )
-    process.exit(-1)
-  }
+const { checkDevelopmentEnvironment } = require("./utilities/fileSystem")

+function runServer() {
+  // this will shutdown the system if development environment not ready
+  // will print an error explaining what to do
+  checkDevelopmentEnvironment()
  fixPath()
  require("./app")
 }
--- a/packages/server/src/middleware/authorized.js
+++ b/packages/server/src/middleware/authorized.js
@ -13,19 +13,12 @@ const { AuthTypes } = require("../constants")

 const ADMIN_ROLES = [BUILTIN_ROLE_IDS.ADMIN, BUILTIN_ROLE_IDS.BUILDER]

-const LOCAL_PASS = new RegExp(["webhooks/trigger"].join("|"))
-
 function hasResource(ctx) {
  return ctx.resourceId != null
 }

 module.exports = (permType, permLevel = null) => async (ctx, next) => {
-  // webhooks can pass locally
-  if (!env.CLOUD && LOCAL_PASS.test(ctx.request.url)) {
-    return next()
-  }
-
-  if (env.CLOUD && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) {
+  if (env.isProd() && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) {
    // api key header passed by external webhook
    if (await isAPIKeyValid(ctx.headers["x-api-key"])) {
      ctx.auth = {
@ -41,20 +34,23 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
    return ctx.throw(403, "API key invalid")
  }

-  // don't expose builder endpoints in the cloud
-  if (env.CLOUD && permType === PermissionTypes.BUILDER) return
-
  if (!ctx.user) {
    return ctx.throw(403, "No user info found")
  }

  const role = ctx.user.role
+  const isBuilder = role._id === BUILTIN_ROLE_IDS.BUILDER
+  const isAdmin = ADMIN_ROLES.includes(role._id)
+  const isAuthed = ctx.auth.authenticated
+
+  if (permType === PermissionTypes.BUILDER && isBuilder) {
+    return next()
+  }
+
  const { basePermissions, permissions } = await getUserPermissions(
    ctx.appId,
    role._id
  )
-  const isAdmin = ADMIN_ROLES.includes(role._id)
-  const isAuthed = ctx.auth.authenticated

  // this may need to change in the future, right now only admins
  // can have access to builder features, this is hard coded into
--- a/packages/server/src/middleware/selfhost.js
+++ b/packages/server/src/middleware/selfhost.js
@ -1,14 +1,8 @@
 const env = require("../environment")
-const hosting = require("../utilities/builder/hosting")
 // if added as a middleware will stop requests unless builder is in self host mode
 // or cloud is in self host
 module.exports = async (ctx, next) => {
-  if (env.CLOUD && env.SELF_HOSTED) {
-    await next()
-    return
-  }
-  const hostingInfo = await hosting.getHostingInfo()
-  if (hostingInfo.type === hosting.HostingTypes.SELF) {
+  if (env.SELF_HOSTED) {
    await next()
    return
  }
--- a/packages/server/src/middleware/tests/authorized.spec.js
+++ b/packages/server/src/middleware/tests/authorized.spec.js
@ -3,8 +3,15 @@ const env = require("../../environment")
 const apiKey = require("../../utilities/security/apikey")
 const { AuthTypes } = require("../../constants")
 const { PermissionTypes, PermissionLevels } = require("../../utilities/security/permissions")
-const { Test } = require("supertest")
-jest.mock("../../environment")
+jest.mock("../../environment", () => ({
+    prod: false,
+    isTest: () => true,
+    isProd: () => this.prod,
+    _set: (key, value) => {
+      this.prod = value === "production"
+    }
+  })
+)
 jest.mock("../../utilities/security/apikey")

 class TestConfiguration {
@ -47,8 +54,8 @@ class TestConfiguration {
    this.ctx.request.url = url
  }

-  setCloudEnv(isCloud) {
-    env.CLOUD = isCloud
+  setEnvironment(isProd) {
+    env._set("NODE_ENV", isProd ? "production" : "jest")
  }

  setRequestHeaders(headers) {
@ -71,12 +78,6 @@ describe("Authorization middleware", () => {

  beforeEach(() => {
    config = new TestConfiguration()
-  }) 
-
-  it("passes the middleware for local webhooks", async () => {
-    config.setRequestUrl("https://something/webhooks/trigger")
-    await config.executeMiddleware()
-    expect(config.next).toHaveBeenCalled()
  })

  describe("external web hook call", () => {
@ -85,7 +86,7 @@ describe("Authorization middleware", () => {

    beforeEach(() => {
      config = new TestConfiguration()
-      config.setCloudEnv(true)
+      config.setEnvironment(true)
      config.setRequestHeaders({
        "x-api-key": "abc123",
        "x-instanceid": "instance123",
@ -121,7 +122,7 @@ describe("Authorization middleware", () => {

    beforeEach(() => {
      config = new TestConfiguration()
-      config.setCloudEnv(true)
+      config.setEnvironment(true)
      config.setAuthenticated(true)
    })

@ -144,7 +145,7 @@ describe("Authorization middleware", () => {
    })

    it("throws if the user has only builder permissions", async () => {
-      config.setCloudEnv(false)
+      config.setEnvironment(false)
      config.setMiddlewareRequiredPermission(PermissionTypes.BUILDER)
      config.setUser({
        role: {
--- a/packages/server/src/middleware/tests/selfhost.spec.js
+++ b/packages/server/src/middleware/tests/selfhost.spec.js
@ -1,6 +1,5 @@
-const selfHostMiddleware = require("../selfhost");
+const selfHostMiddleware = require("../selfhost")
 const env = require("../../environment")
-const hosting = require("../../utilities/builder/hosting");
 jest.mock("../../environment") 
 jest.mock("../../utilities/builder/hosting") 

@ -20,16 +19,6 @@ class TestConfiguration {
    return this.middleware(this.ctx, this.next)
  }

-  setCloudHosted() {
-    env.CLOUD = 1 
-    env.SELF_HOSTED = 0
-  }
-
-  setSelfHosted() {
-    env.CLOUD = 0
-    env.SELF_HOSTED = 1
-  }
-
  afterEach() {
    jest.clearAllMocks()
  }
@ -46,30 +35,10 @@ describe("Self host middleware", () => {
    config.afterEach()
  })

-  it("calls next() when CLOUD and SELF_HOSTED env vars are set", async () => {
-    env.CLOUD = 1 
+  it("calls next() when SELF_HOSTED env var is set", async () => {
    env.SELF_HOSTED = 1 

    await config.executeMiddleware()
    expect(config.next).toHaveBeenCalled()
  })
-
-  it("throws when hostingInfo type is cloud", async () => {
-    config.setSelfHosted()
-
-    hosting.getHostingInfo.mockImplementationOnce(() => ({ type: hosting.HostingTypes.CLOUD }))
-
-    await config.executeMiddleware()
-    expect(config.throw).toHaveBeenCalledWith(400, "Endpoint unavailable in cloud hosting.")
-    expect(config.next).not.toHaveBeenCalled()
-  })
-
-  it("calls the self hosting middleware to pass through to next() when the hostingInfo type is self", async () => {
-    config.setSelfHosted()
-
-    hosting.getHostingInfo.mockImplementationOnce(() => ({ type: hosting.HostingTypes.SELF }))
-
-    await config.executeMiddleware()
-    expect(config.next).toHaveBeenCalled()
-  })
 })
--- a/packages/server/src/middleware/tests/usageQuota.spec.js
+++ b/packages/server/src/middleware/tests/usageQuota.spec.js
@ -5,7 +5,12 @@ const env = require("../../environment")

 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
-jest.mock("../../environment")
+jest.mock("../../environment", () => ({
+  isTest: () => true,
+  isProd: () => false,
+  isDev: () => true,
+  _set: () => {},
+}))

 class TestConfiguration {
  constructor() {
@ -32,12 +37,14 @@ class TestConfiguration {
    return this.middleware(this.ctx, this.next)
  }

-  cloudHosted(bool) {
+  setProd(bool) {
    if (bool) {
-      env.CLOUD = 1
+      env.isDev = () => false
+      env.isProd = () => true
      this.ctx.auth = { apiKey: "test" }
    } else {
-      env.CLOUD = 0
+      env.isDev = () => true
+      env.isProd = () => false
    }
  }

@ -102,7 +109,7 @@ describe("usageQuota middleware", () => {

  it("calculates and persists the correct usage quota for the relevant action", async () => {
    config.setUrl("/rows")
-    config.cloudHosted(true)
+    config.setProd(true)

    await config.executeMiddleware()

@ -112,7 +119,7 @@ describe("usageQuota middleware", () => {

  it("calculates the correct file size from a file upload call and adds it to quota", async () => {
    config.setUrl("/upload")
-    config.cloudHosted(true)
+    config.setProd(true)
    config.setFiles([
      {
        size: 100
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@ -44,8 +44,8 @@ module.exports = async (ctx, next) => {
    }
  }

-  // if running in builder or a self hosted cloud usage quotas should not be executed
-  if (!env.CLOUD || env.SELF_HOSTED) {
+  // if in development or a self hosted cloud usage quotas should not be executed
+  if (env.isDev() || env.SELF_HOSTED) {
    return next()
  }
  // update usage for uploads to be the total size
--- a/packages/server/src/tests/utilities/TestConfiguration.js
+++ b/packages/server/src/tests/utilities/TestConfiguration.js
@ -14,9 +14,7 @@ const {
 } = require("./structures")
 const controllers = require("./controllers")
 const supertest = require("supertest")
-const fs = require("fs")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
-const { join } = require("path")
+const { cleanup } = require("../../utilities/fileSystem")

 const EMAIL = "babs@babs.com"
 const PASSWORD = "babs_password"
@ -66,13 +64,7 @@ class TestConfiguration {
    if (this.server) {
      this.server.close()
    }
-    const appDir = budibaseAppsDir()
-    const files = fs.readdirSync(appDir)
-    for (let file of files) {
-      if (this.allApps.some(app => file.includes(app._id))) {
-        fs.rmdirSync(join(appDir, file), { recursive: true })
-      }
-    }
+    cleanup(this.allApps.map(app => app._id))
  }

  defaultHeaders() {
@ -81,9 +73,11 @@ class TestConfiguration {
      roleId: BUILTIN_ROLE_IDS.BUILDER,
    }
    const builderToken = jwt.sign(builderUser, env.JWT_SECRET)
+    // can be "production" for test case
+    const type = env.isProd() ? "cloud" : "local"
    const headers = {
      Accept: "application/json",
-      Cookie: [`budibase:builder:local=${builderToken}`],
+      Cookie: [`budibase:builder:${type}=${builderToken}`],
    }
    if (this.appId) {
      headers["x-budibase-app-id"] = this.appId
--- a/packages/server/src/utilities/builder/compileStaticAssets.js
+++ b/packages/server/src/utilities/builder/compileStaticAssets.js
@ -1,35 +0,0 @@
-const { ensureDir, constants, copyFile } = require("fs-extra")
-const { join } = require("../centralPath")
-const { budibaseAppsDir } = require("../budibaseDir")
-
-/**
- * Compile all the non-db static web assets that are required for the running of
- * a budibase application. This includes the JSON structure of the DOM and
- * the client library, a script responsible for reading the JSON structure
- * and rendering the application.
- * @param {string} appId id of the application we want to compile static assets for
- */
-module.exports = async appId => {
-  const publicPath = join(budibaseAppsDir(), appId, "public")
-  await ensureDir(publicPath)
-  await copyClientLib(publicPath)
-}
-
-/**
- * Copy the budibase client library and sourcemap from NPM to <appId>/public/.
- * The client library is then served as a static asset when the budibase application
- * is running in preview or prod
- * @param {String} publicPath - path to write the client library to
- */
-const copyClientLib = async publicPath => {
-  const sourcepath = require.resolve("@budibase/client")
-  const destPath = join(publicPath, "budibase-client.js")
-
-  await copyFile(sourcepath, destPath, constants.COPYFILE_FICLONE)
-
-  await copyFile(
-    sourcepath + ".map",
-    destPath + ".map",
-    constants.COPYFILE_FICLONE
-  )
-}
--- a/packages/server/src/utilities/builder/hosting.js
+++ b/packages/server/src/utilities/builder/hosting.js
@ -1,5 +1,5 @@
 const CouchDB = require("../../db")
-const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
+const { StaticDatabases } = require("../../db/utils")
 const fetch = require("node-fetch")
 const env = require("../../environment")

@ -23,16 +23,16 @@ exports.HostingTypes = {
 }

 exports.getHostingInfo = async () => {
-  const db = new CouchDB(BUILDER_CONFIG_DB)
+  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
  let doc
  try {
-    doc = await db.get(HOSTING_DOC)
+    doc = await db.get(StaticDatabases.BUILDER_HOSTING.baseDoc)
  } catch (err) {
    // don't write this doc, want to be able to update these default props
    // for our servers with a new release without needing to worry about state of
    // PouchDB in peoples installations
    doc = {
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
      type: exports.HostingTypes.CLOUD,
      hostingUrl: PROD_HOSTING_URL,
      selfHostKey: "",
@ -85,15 +85,11 @@ exports.getTemplatesUrl = async (appId, type, name) => {
 }

 exports.getDeployedApps = async () => {
-  const hostingInfo = await exports.getHostingInfo()
-  if (
-    (!env.CLOUD && hostingInfo.type === exports.HostingTypes.CLOUD) ||
-    (env.CLOUD && !env.SELF_HOSTED)
-  ) {
+  if (!env.SELF_HOSTED) {
    throw "Can only check apps for self hosted environments"
  }
-  const workerUrl = !env.CLOUD ? await exports.getWorkerUrl() : env.WORKER_URL
-  const hostingKey = !env.CLOUD ? hostingInfo.selfHostKey : env.HOSTING_KEY
+  const workerUrl = env.WORKER_URL
+  const hostingKey = env.HOSTING_KEY
  try {
    const response = await fetch(`${workerUrl}/api/apps`, {
      method: "GET",
@ -102,12 +98,14 @@ exports.getDeployedApps = async () => {
      },
    })
    const json = await response.json()
-    for (let value of Object.values(json)) {
+    const apps = {}
+    for (let [key, value] of Object.entries(json)) {
      if (value.url) {
        value.url = value.url.toLowerCase()
+        apps[key] = value
      }
    }
-    return json
+    return apps
  } catch (err) {
    // error, cannot determine deployed apps, don't stop app creation - sort this later
    return {}
--- a/packages/server/src/utilities/createAppPackage.js
+++ b/packages/server/src/utilities/createAppPackage.js
@ -1,29 +0,0 @@
-const stream = require("stream")
-const fetch = require("node-fetch")
-const tar = require("tar-fs")
-const zlib = require("zlib")
-const { promisify } = require("util")
-const packageJson = require("../../package.json")
-
-const streamPipeline = promisify(stream.pipeline)
-
-// can't really test this due to the downloading nature of it, wouldn't be a great test case
-/* istanbul ignore next */
-exports.downloadExtractComponentLibraries = async appFolder => {
-  const LIBRARIES = ["standard-components"]
-
-  // Need to download tarballs directly from NPM as our users may not have node on their machine
-  for (let lib of LIBRARIES) {
-    // download tarball
-    const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz`
-    const response = await fetch(registryUrl)
-    if (!response.ok)
-      throw new Error(`unexpected response ${response.statusText}`)
-
-    await streamPipeline(
-      response.body,
-      zlib.Unzip(),
-      tar.extract(`${appFolder}/node_modules/@budibase/${lib}`)
-    )
-  }
-}
--- a/packages/server/src/utilities/fileSystem/index.js
+++ b/packages/server/src/utilities/fileSystem/index.js
@ -0,0 +1,232 @@
+const { budibaseTempDir } = require("../budibaseDir")
+const { isDev } = require("../index")
+const fs = require("fs")
+const { join } = require("path")
+const uuid = require("uuid/v4")
+const CouchDB = require("../../db")
+const { ObjectStoreBuckets } = require("../../constants")
+const {
+  upload,
+  retrieve,
+  retrieveToTmp,
+  streamUpload,
+  deleteFolder,
+  downloadTarball,
+} = require("./utilities")
+const { downloadLibraries, newAppPublicPath } = require("./newApp")
+const download = require("download")
+const env = require("../../environment")
+const { homedir } = require("os")
+const fetch = require("node-fetch")
+
+const DEFAULT_AUTOMATION_BUCKET =
+  "https://prod-budi-automations.s3-eu-west-1.amazonaws.com"
+const DEFAULT_AUTOMATION_DIRECTORY = ".budibase-automations"
+
+/**
+ * The single stack system (Cloud and Builder) should not make use of the file system where possible,
+ * this file handles all of the file access for the system with the intention of limiting it all to one
+ * place. Keeping all of this logic in one place means that when we need to do file system access (like
+ * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
+ * be done through an object store instead.
+ */
+
+/**
+ * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
+ */
+exports.init = () => {
+  const tempDir = budibaseTempDir()
+  if (!fs.existsSync(tempDir)) {
+    fs.mkdirSync(tempDir)
+  }
+  const clientLibPath = join(budibaseTempDir(), "budibase-client.js")
+  if (env.isTest() && !fs.existsSync(clientLibPath)) {
+    fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath)
+  }
+}
+
+/**
+ * Checks if the system is currently in development mode and if it is makes sure
+ * everything required to function is ready.
+ */
+exports.checkDevelopmentEnvironment = () => {
+  if (!isDev()) {
+    return
+  }
+  let error
+  if (!fs.existsSync(budibaseTempDir())) {
+    error =
+      "Please run a build before attempting to run server independently to fill 'tmp' directory."
+  }
+  if (!fs.existsSync(join(process.cwd(), ".env"))) {
+    error = "Must run via yarn once to generate environment."
+  }
+  if (error) {
+    console.error(error)
+    process.exit(-1)
+  }
+}
+
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns an fs read stream which can be loaded into the database.
+ */
+exports.getTemplateStream = async template => {
+  if (template.file) {
+    return fs.createReadStream(template.file.path)
+  } else {
+    const tmpPath = await exports.downloadTemplate(...template.key.split("/"))
+    return fs.createReadStream(join(tmpPath, "db", "dump.txt"))
+  }
+}
+
+/**
+ * Used to retrieve a handlebars file from the system which will be used as a template.
+ * This is allowable as the template handlebars files should be static and identical across
+ * the cluster.
+ * @param {string} path The path to the handlebars file which is to be loaded.
+ * @returns {string} The loaded handlebars file as a string - loaded as utf8.
+ */
+exports.loadHandlebarsFile = path => {
+  return fs.readFileSync(path, "utf8")
+}
+
+/**
+ * When return a file from the API need to write the file to the system temporarily so we
+ * can create a read stream to send.
+ * @param {string} contents the contents of the file which is to be returned from the API.
+ * @return {Object} the read stream which can be put into the koa context body.
+ */
+exports.apiFileReturn = contents => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, contents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Takes a copy of the database state for an app to the object store.
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @param {string} backupName The name of the backup located in the object store.
+ * @return The backup has been completed when this promise completes and returns a file stream
+ * to the temporary backup file (to return via API if required).
+ */
+exports.performBackup = async (appId, backupName) => {
+  const path = join(budibaseTempDir(), backupName)
+  const writeStream = fs.createWriteStream(path)
+  // perform couch dump
+  const instanceDb = new CouchDB(appId)
+  await instanceDb.dump(writeStream, {})
+  // write the file to the object store
+  await streamUpload(
+    ObjectStoreBuckets.BACKUPS,
+    join(appId, backupName),
+    fs.createReadStream(path)
+  )
+  return fs.createReadStream(path)
+}
+
+/**
+ * Downloads required libraries and creates a new path in the object store.
+ * @param {string} appId The ID of the app which is being created.
+ * @return {Promise<void>} once promise completes app resources should be ready in object store.
+ */
+exports.createApp = async appId => {
+  await downloadLibraries(appId)
+  await newAppPublicPath(appId)
+}
+
+/**
+ * Removes all of the assets created for an app in the object store.
+ * @param {string} appId The ID of the app which is being deleted.
+ * @return {Promise<void>} once promise completes the app resources will be removed from object store.
+ */
+exports.deleteApp = async appId => {
+  await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+}
+
+/**
+ * Retrieves a template and pipes it to minio as well as making it available temporarily.
+ * @param {string} type The type of template which is to be retrieved.
+ * @param name
+ * @return {Promise<*>}
+ */
+exports.downloadTemplate = async (type, name) => {
+  const DEFAULT_TEMPLATES_BUCKET =
+    "prod-budi-templates.s3-eu-west-1.amazonaws.com"
+  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
+  return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type)
+}
+
+/**
+ * Retrieves component libraries from object store (or tmp symlink if in local)
+ */
+exports.getComponentLibraryManifest = async (appId, library) => {
+  const filename = "manifest.json"
+  /* istanbul ignore next */
+  // when testing in cypress and so on we need to get the package
+  // as the environment may not be fully fleshed out for dev or prod
+  if (env.isTest()) {
+    const lib = library.split("/")[1]
+    const path = require.resolve(library).split(lib)[0]
+    return require(join(path, lib, filename))
+  }
+  const devPath = join(budibaseTempDir(), library, filename)
+  if (env.isDev() && fs.existsSync(devPath)) {
+    return require(devPath)
+  }
+  const path = join(appId, "node_modules", library, "package", filename)
+  let resp = await retrieve(ObjectStoreBuckets.APPS, path)
+  if (typeof resp !== "string") {
+    resp = resp.toString("utf8")
+  }
+  return JSON.parse(resp)
+}
+
+exports.automationInit = async () => {
+  const directory =
+    env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY)
+  const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET
+  if (!fs.existsSync(directory)) {
+    fs.mkdirSync(directory, { recursive: true })
+  }
+  // env setup to get async packages
+  let response = await fetch(`${bucket}/manifest.json`)
+  return response.json()
+}
+
+exports.getExternalAutomationStep = async (name, version, bundleName) => {
+  const directory =
+    env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY)
+  const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET
+  try {
+    return require(join(directory, bundleName))
+  } catch (err) {
+    await download(`${bucket}/${name}/${version}/${bundleName}`, directory)
+    return require(join(directory, bundleName))
+  }
+}
+
+/**
+ * All file reads come through here just to make sure all of them make sense
+ * allows a centralised location to check logic is all good.
+ */
+exports.readFileSync = (filepath, options = "utf8") => {
+  return fs.readFileSync(filepath, options)
+}
+
+/**
+ * Given a set of app IDs makes sure file system is cleared of any of their temp info.
+ */
+exports.cleanup = appIds => {
+  for (let appId of appIds) {
+    fs.rmdirSync(join(budibaseTempDir(), appId), { recursive: true })
+  }
+}
+
+/**
+ * Full function definition for below can be found in the utilities.
+ */
+exports.upload = upload
+exports.retrieve = retrieve
+exports.retrieveToTmp = retrieveToTmp
--- a/packages/server/src/utilities/fileSystem/newApp.js
+++ b/packages/server/src/utilities/fileSystem/newApp.js
@ -0,0 +1,40 @@
+const packageJson = require("../../../package.json")
+const { join } = require("path")
+const { ObjectStoreBuckets } = require("../../constants")
+const { streamUpload, downloadTarball } = require("./utilities")
+const fs = require("fs")
+
+const BUCKET_NAME = ObjectStoreBuckets.APPS
+
+// can't really test this due to the downloading nature of it, wouldn't be a great test case
+/* istanbul ignore next */
+exports.downloadLibraries = async appId => {
+  const LIBRARIES = ["standard-components"]
+
+  const paths = {}
+  // Need to download tarballs directly from NPM as our users may not have node on their machine
+  for (let lib of LIBRARIES) {
+    // download tarball
+    const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz`
+    const path = join(appId, "node_modules", "@budibase", lib)
+    paths[`@budibase/${lib}`] = await downloadTarball(
+      registryUrl,
+      BUCKET_NAME,
+      path
+    )
+  }
+  return paths
+}
+
+exports.newAppPublicPath = async appId => {
+  const path = join(appId, "public")
+  const sourcepath = require.resolve("@budibase/client")
+  const destPath = join(path, "budibase-client.js")
+
+  await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath))
+  await streamUpload(
+    BUCKET_NAME,
+    destPath + ".map",
+    fs.createReadStream(sourcepath + ".map")
+  )
+}
--- a/packages/server/src/utilities/fileSystem/processor.js
+++ b/packages/server/src/utilities/fileSystem/processor.js
@ -1,25 +1,20 @@
-const fs = require("fs")
 const jimp = require("jimp")
-const fsPromises = fs.promises

 const FORMATS = {
  IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
 }

 function processImage(file) {
+  // this will overwrite the temp file
  return jimp.read(file.path).then(img => {
-    return img.resize(300, jimp.AUTO).write(file.outputPath)
+    return img.resize(300, jimp.AUTO).write(file.path)
  })
 }

 async function process(file) {
  if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) {
    await processImage(file)
-    return file
  }
-
-  // No processing required
-  await fsPromises.copyFile(file.path, file.outputPath)
  return file
 }

--- a/packages/server/src/utilities/fileSystem/utilities.js
+++ b/packages/server/src/utilities/fileSystem/utilities.js
@ -0,0 +1,243 @@
+const sanitize = require("sanitize-s3-objectkey")
+const AWS = require("aws-sdk")
+const stream = require("stream")
+const fetch = require("node-fetch")
+const tar = require("tar-fs")
+const zlib = require("zlib")
+const { promisify } = require("util")
+const { join } = require("path")
+const fs = require("fs")
+const { budibaseTempDir } = require("../budibaseDir")
+const env = require("../../environment")
+const { ObjectStoreBuckets } = require("../../constants")
+const uuid = require("uuid/v4")
+
+const streamPipeline = promisify(stream.pipeline)
+// use this as a temporary store of buckets that are being created
+const STATE = {
+  bucketCreationPromises: {},
+}
+
+const CONTENT_TYPE_MAP = {
+  html: "text/html",
+  css: "text/css",
+  js: "application/javascript",
+}
+const STRING_CONTENT_TYPES = [
+  CONTENT_TYPE_MAP.html,
+  CONTENT_TYPE_MAP.css,
+  CONTENT_TYPE_MAP.js,
+]
+
+function publicPolicy(bucketName) {
+  return {
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Effect: "Allow",
+        Principal: {
+          AWS: ["*"],
+        },
+        Action: "s3:GetObject",
+        Resource: [`arn:aws:s3:::${bucketName}/*`],
+      },
+    ],
+  }
+}
+
+const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS]
+
+/**
+ * Gets a connection to the object store using the S3 SDK.
+ * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from.
+ * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
+ * @constructor
+ */
+exports.ObjectStore = bucket => {
+  if (env.SELF_HOSTED) {
+    AWS.config.update({
+      accessKeyId: env.MINIO_ACCESS_KEY,
+      secretAccessKey: env.MINIO_SECRET_KEY,
+    })
+  }
+  const config = {
+    s3ForcePathStyle: true,
+    signatureVersion: "v4",
+    params: {
+      Bucket: bucket,
+    },
+  }
+  if (env.MINIO_URL) {
+    config.endpoint = env.MINIO_URL
+  }
+  return new AWS.S3(config)
+}
+
+/**
+ * Given an object store and a bucket name this will make sure the bucket exists,
+ * if it does not exist then it will create it.
+ */
+exports.makeSureBucketExists = async (client, bucketName) => {
+  try {
+    await client
+      .headBucket({
+        Bucket: bucketName,
+      })
+      .promise()
+  } catch (err) {
+    const promises = STATE.bucketCreationPromises
+    if (promises[bucketName]) {
+      await promises[bucketName]
+    } else if (err.statusCode === 404) {
+      // bucket doesn't exist create it
+      promises[bucketName] = client
+        .createBucket({
+          Bucket: bucketName,
+        })
+        .promise()
+      await promises[bucketName]
+      delete promises[bucketName]
+      // public buckets are quite hidden in the system, make sure
+      // no bucket is set accidentally
+      if (PUBLIC_BUCKETS.includes(bucketName)) {
+        await client
+          .putBucketPolicy({
+            Bucket: bucketName,
+            Policy: JSON.stringify(publicPolicy(bucketName)),
+          })
+          .promise()
+      }
+    } else {
+      throw err
+    }
+  }
+}
+
+/**
+ * Uploads the contents of a file given the required parameters, useful when
+ * temp files in use (for example file uploaded as an attachment).
+ */
+exports.upload = async ({ bucket, filename, path, type, metadata }) => {
+  const extension = [...filename.split(".")].pop()
+  const fileBytes = fs.readFileSync(path)
+
+  const objectStore = exports.ObjectStore(bucket)
+  await exports.makeSureBucketExists(objectStore, bucket)
+
+  const config = {
+    // windows file paths need to be converted to forward slashes for s3
+    Key: sanitize(filename).replace(/\\/g, "/"),
+    Body: fileBytes,
+    ContentType: type || CONTENT_TYPE_MAP[extension.toLowerCase()],
+  }
+  if (metadata) {
+    config.Metadata = metadata
+  }
+  return objectStore.upload(config).promise()
+}
+
+/**
+ * Similar to the upload function but can be used to send a file stream
+ * through to the object store.
+ */
+exports.streamUpload = async (bucket, filename, stream) => {
+  const objectStore = exports.ObjectStore(bucket)
+  await exports.makeSureBucketExists(objectStore, bucket)
+
+  const params = {
+    Bucket: bucket,
+    Key: sanitize(filename).replace(/\\/g, "/"),
+    Body: stream,
+  }
+  return objectStore.upload(params).promise()
+}
+
+/**
+ * retrieves the contents of a file from the object store, if it is a known content type it
+ * will be converted, otherwise it will be returned as a buffer stream.
+ */
+exports.retrieve = async (bucket, filepath) => {
+  const objectStore = exports.ObjectStore(bucket)
+  const params = {
+    Bucket: bucket,
+    Key: sanitize(filepath).replace(/\\/g, "/"),
+  }
+  const response = await objectStore.getObject(params).promise()
+  // currently these are all strings
+  if (STRING_CONTENT_TYPES.includes(response.ContentType)) {
+    return response.Body.toString("utf8")
+  } else {
+    return response.Body
+  }
+}
+
+/**
+ * Same as retrieval function but puts to a temporary file.
+ */
+exports.retrieveToTmp = async (bucket, filepath) => {
+  const data = await exports.retrieve(bucket, filepath)
+  const outputPath = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(outputPath, data)
+  return outputPath
+}
+
+exports.deleteFolder = async (bucket, folder) => {
+  const client = exports.ObjectStore(bucket)
+  const listParams = {
+    Bucket: bucket,
+    Prefix: folder,
+  }
+
+  let response = await client.listObjects(listParams).promise()
+  if (response.Contents.length === 0) {
+    return
+  }
+  const deleteParams = {
+    Bucket: bucket,
+    Delete: {
+      Objects: [],
+    },
+  }
+
+  response.Contents.forEach(content => {
+    deleteParams.Delete.Objects.push({ Key: content.Key })
+  })
+
+  response = await client.deleteObjects(deleteParams).promise()
+  // can only empty 1000 items at once
+  if (response.Deleted.length === 1000) {
+    return exports.deleteFolder(bucket, folder)
+  }
+}
+
+exports.uploadDirectory = async (bucket, localPath, bucketPath) => {
+  let uploads = []
+  const files = fs.readdirSync(localPath, { withFileTypes: true })
+  for (let file of files) {
+    const path = join(bucketPath, file.name)
+    const local = join(localPath, file.name)
+    if (file.isDirectory()) {
+      uploads.push(exports.uploadDirectory(bucket, local, path))
+    } else {
+      uploads.push(
+        exports.streamUpload(bucket, path, fs.createReadStream(local))
+      )
+    }
+  }
+  await Promise.all(uploads)
+}
+
+exports.downloadTarball = async (url, bucket, path) => {
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`unexpected response ${response.statusText}`)
+  }
+
+  const tmpPath = join(budibaseTempDir(), path)
+  await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
+  if (!env.isTest()) {
+    await exports.uploadDirectory(bucket, tmpPath, path)
+  }
+  // return the temporary path incase there is a use for it
+  return tmpPath
+}
--- a/packages/server/src/utilities/index.js
+++ b/packages/server/src/utilities/index.js
@ -1,6 +1,5 @@
 const env = require("../environment")
 const { DocumentTypes, SEPARATOR } = require("../db/utils")
-const fs = require("fs")
 const CouchDB = require("../db")

 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
@ -13,14 +12,7 @@ function confirmAppId(possibleAppId) {

 exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))

-exports.isDev = () => {
-  return (
-    !env.CLOUD &&
-    env.NODE_ENV !== "production" &&
-    env.NODE_ENV !== "jest" &&
-    env.NODE_ENV !== "cypress"
-  )
-}
+exports.isDev = env.isDev

 /**
 * Given a request tries to find the appId, which can be located in various places
@ -28,10 +20,18 @@ exports.isDev = () => {
 * @returns {string|undefined} If an appId was found it will be returned.
 */
 exports.getAppId = ctx => {
-  let appId = confirmAppId(ctx.headers["x-budibase-app-id"])
-  if (!appId) {
-    appId = confirmAppId(env.CLOUD ? ctx.subdomains[1] : ctx.params.appId)
+  const options = [ctx.headers["x-budibase-app-id"], ctx.params.appId]
+  if (ctx.subdomains) {
+    options.push(ctx.subdomains[1])
  }
+  let appId
+  for (let option of options) {
+    appId = confirmAppId(option)
+    if (appId) {
+      break
+    }
+  }
+
  // look in body if can't find it in subdomain
  if (!appId && ctx.request.body && ctx.request.body.appId) {
    appId = confirmAppId(ctx.request.body.appId)
@ -51,7 +51,7 @@ exports.getAppId = ctx => {
 * @returns {string} The name of the token trying to find
 */
 exports.getCookieName = (name = "builder") => {
-  let environment = env.CLOUD ? "cloud" : "local"
+  let environment = env.isProd() ? "cloud" : "local"
  return `budibase:${name}:${environment}`
 }

@ -89,24 +89,6 @@ exports.isClient = ctx => {
  return ctx.headers["x-budibase-type"] === "client"
 }

-/**
- * Recursively walk a directory tree and execute a callback on all files.
- * @param {String} dirPath - Directory to traverse
- * @param {Function} callback - callback to execute on files
- */
-exports.walkDir = (dirPath, callback) => {
-  for (let filename of fs.readdirSync(dirPath)) {
-    const filePath = `${dirPath}/${filename}`
-    const stat = fs.lstatSync(filePath)
-
-    if (stat.isFile()) {
-      callback(filePath)
-    } else {
-      exports.walkDir(filePath, callback)
-    }
-  }
-}
-
 exports.getLogoUrl = () => {
  return "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg"
 }
@ -124,3 +106,7 @@ exports.getAllApps = async () => {
      .map(({ value }) => value)
  }
 }
+
+exports.checkSlashesInUrl = url => {
+  return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
+}
--- a/packages/server/src/utilities/initialiseBudibase.js
+++ b/packages/server/src/utilities/initialiseBudibase.js
@ -1,37 +0,0 @@
-const { existsSync, readFile, writeFile, ensureDir } = require("fs-extra")
-const { join, resolve } = require("./centralPath")
-const { processString } = require("@budibase/string-templates")
-const uuid = require("uuid")
-
-module.exports = async opts => {
-  await ensureDir(opts.dir)
-  await setCouchDbUrl(opts)
-
-  // need an env file
-  await createDevEnvFile(opts)
-}
-
-const setCouchDbUrl = async opts => {
-  if (!opts.couchDbUrl) {
-    const dataDir = join(opts.dir, ".data")
-    await ensureDir(dataDir)
-    opts.couchDbUrl =
-      dataDir + (dataDir.endsWith("/") || dataDir.endsWith("\\") ? "" : "/")
-  }
-}
-
-const createDevEnvFile = async opts => {
-  const destConfigFile = join(opts.dir, "./.env")
-  let createConfig = !existsSync(destConfigFile) || opts.quiet
-  if (createConfig) {
-    const template = await readFile(
-      resolve(__dirname, "..", "..", ".env.template"),
-      {
-        encoding: "utf8",
-      }
-    )
-    opts.cookieKey1 = opts.cookieKey1 || uuid.v4()
-    const config = await processString(template, opts)
-    await writeFile(destConfigFile, config, { flag: "w+" })
-  }
-}
--- a/packages/server/src/utilities/rowProcessor.js
+++ b/packages/server/src/utilities/rowProcessor.js
@ -180,7 +180,7 @@ exports.outputProcessing = async (appId, table, rows) => {
    rows
  )
  // update the attachments URL depending on hosting
-  if (env.CLOUD && env.SELF_HOSTED) {
+  if (env.isProd() && env.SELF_HOSTED) {
    for (let [property, column] of Object.entries(table.schema)) {
      if (column.type === FieldTypes.ATTACHMENT) {
        for (let row of outputRows) {
--- a/packages/server/src/utilities/templates.js
+++ b/packages/server/src/utilities/templates.js
@ -1,81 +0,0 @@
-const fs = require("fs-extra")
-const { join } = require("./centralPath")
-const os = require("os")
-const fetch = require("node-fetch")
-const stream = require("stream")
-const tar = require("tar-fs")
-const zlib = require("zlib")
-const { promisify } = require("util")
-const streamPipeline = promisify(stream.pipeline)
-const { budibaseAppsDir } = require("./budibaseDir")
-const env = require("../environment")
-const CouchDB = require("../db")
-
-const DEFAULT_TEMPLATES_BUCKET =
-  "prod-budi-templates.s3-eu-west-1.amazonaws.com"
-
-exports.getLocalTemplates = function() {
-  const templatesDir = join(os.homedir(), ".budibase", "templates", "app")
-  const templateObj = { app: {} }
-  fs.ensureDirSync(templatesDir)
-  const templateNames = fs.readdirSync(templatesDir)
-  for (let name of templateNames) {
-    templateObj.app[name] = {
-      name,
-      category: "local",
-      description: "local template",
-      type: "app",
-      key: `app/${name}`,
-    }
-  }
-  return templateObj
-}
-
-// can't really test this, downloading is just not something we should do in a behavioural test
-/* istanbul ignore next */
-exports.downloadTemplate = async function(type, name) {
-  const dirName = join(budibaseAppsDir(), "templates", type, name)
-  if (env.LOCAL_TEMPLATES) {
-    return dirName
-  }
-  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
-  const response = await fetch(templateUrl)
-
-  if (!response.ok) {
-    throw new Error(
-      `Error downloading template ${type}:${name}: ${response.statusText}`
-    )
-  }
-
-  // stream the response, unzip and extract
-  await streamPipeline(
-    response.body,
-    zlib.Unzip(),
-    tar.extract(join(budibaseAppsDir(), "templates", type))
-  )
-
-  return dirName
-}
-
-async function performDump({ dir, appId, name = "dump.txt" }) {
-  const writeStream = fs.createWriteStream(join(dir, name))
-  // perform couch dump
-  const instanceDb = new CouchDB(appId)
-  await instanceDb.dump(writeStream, {})
-}
-
-exports.performDump = performDump
-
-exports.exportTemplateFromApp = async function({ templateName, appId }) {
-  // Copy frontend files
-  const templatesDir = join(
-    budibaseAppsDir(),
-    "templates",
-    "app",
-    templateName,
-    "db"
-  )
-  fs.ensureDirSync(templatesDir)
-  await performDump({ dir: templatesDir, appId })
-  return templatesDir
-}
--- a/packages/server/src/utilities/tests/csvParser.spec.js
+++ b/packages/server/src/utilities/tests/csvParser.spec.js
@ -1,4 +1,4 @@
-const fs = require("fs")
+const { readFileSync } = require("../fileSystem")
 const csvParser = require("../csvParser")

 const CSV_PATH = __dirname + "/test.csv"
@ -33,7 +33,7 @@ const SCHEMAS = {
 }

 describe("CSV Parser", () => {
-  const csvString = fs.readFileSync(CSV_PATH, "utf8")
+  const csvString = readFileSync(CSV_PATH, "utf8")

  describe("parsing", () => {
    it("returns status and types for a valid CSV transformation", async () => {
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@ -50,7 +50,7 @@ exports.Properties = {
 }

 exports.getAPIKey = async appId => {
-  if (env.SELF_HOSTED) {
+  if (!env.USE_QUOTAS) {
    return { apiKey: null }
  }
  return apiKeyTable.get({ primary: appId })
@ -65,8 +65,7 @@ exports.getAPIKey = async appId => {
 * also been reset after this call.
 */
 exports.update = async (apiKey, property, usage) => {
-  // don't try validate in builder
-  if (!env.CLOUD || env.SELF_HOSTED) {
+  if (!env.USE_QUOTAS) {
    return
  }
  try {
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
--- a/packages/standard-components/src/Search.svelte
+++ b/packages/standard-components/src/Search.svelte
@ -25,10 +25,11 @@
  let tableDefinition
  let schema

-  // pagination
-  let page = 0
+  let nextBookmark = null
+  let bookmark = null
+  let lastBookmark = null

-  $: fetchData(table, page)
+  $: fetchData(table, bookmark)
  // omit empty strings
  $: parsedSearch = Object.keys(search).reduce(
    (acc, next) =>
@ -38,33 +39,43 @@
  $: actions = [
    {
      type: ActionTypes.RefreshDatasource,
-      callback: () => fetchData(table, page),
+      callback: () => fetchData(table, bookmark),
      metadata: { datasource: { type: "table", tableId: table } },
    },
  ]

-  async function fetchData(table, page) {
+  async function fetchData(table, mark) {
    if (table) {
      const tableDef = await API.fetchTableDefinition(table)
      schema = tableDef.schema
-      rows = await API.searchTableData({
+      const output = await API.searchTableData({
        tableId: table,
        search: parsedSearch,
        pagination: {
          pageSize,
-          page,
+          bookmark: mark,
        },
      })
+      rows = output.rows
+      nextBookmark = output.bookmark
    }
    loaded = true
  }

  function nextPage() {
-    page += 1
+    lastBookmark = bookmark
+    bookmark = nextBookmark
  }

  function previousPage() {
-    page -= 1
+    nextBookmark = bookmark
+    if (lastBookmark !== bookmark) {
+      bookmark = lastBookmark
+    } else {
+      // special case for going back to beginning
+      bookmark = null
+      lastBookmark = null
+    }
  }
 </script>

@ -99,15 +110,15 @@
          secondary
          on:click={() => {
            search = {}
-            page = 0
+            bookmark = null
          }}>
          Reset
        </Button>
        <Button
          primary
          on:click={() => {
-            page = 0
-            fetchData(table, page)
+            bookmark = null
+            fetchData(table, bookmark)
          }}>
          Search
        </Button>
@ -129,10 +140,10 @@
      {/if}
    {/if}
    <div class="pagination">
-      {#if page > 0}
+      {#if lastBookmark != null || bookmark != null}
        <Button primary on:click={previousPage}>Back</Button>
      {/if}
-      {#if rows.length === pageSize}
+      {#if nextBookmark != null && rows.length !== 0}
        <Button primary on:click={nextPage}>Next</Button>
      {/if}
    </div>
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@ -33,6 +33,5 @@
    "pouchdb": "^7.2.2",
    "pouchdb-all-dbs": "^1.0.2",
    "server-destroy": "^1.0.1"
-  },
-  "gitHead": "1b95326b20d1352d36305910259228b96a683dc7"
+  }
 }
--- a/packages/worker/src/db/index.js
+++ b/packages/worker/src/db/index.js
@ -1,12 +1,9 @@
 const PouchDB = require("pouchdb")
 const allDbs = require("pouchdb-all-dbs")
 const env = require("../environment")
-const { join } = require("path")
-const { homedir } = require("os")

 // level option is purely for testing (development)
-const COUCH_DB_URL =
-  env.COUCH_DB_URL || `leveldb://${join(homedir(), ".budibase")}/.data/`
+const COUCH_DB_URL = env.COUCH_DB_URL || "http://localhost:10000/db/"

 const Pouch = PouchDB.defaults({
  prefix: COUCH_DB_URL,