Some further work to make sure all use of the budibaseAppsDir is removed in important places.

packages/server/scripts/exportAppTemplate.js

@@ -3,7 +3,8 @@ const yargs = require("yargs")
 const fs = require("fs")
 const { join } = require("path")
 const CouchDB = require("../src/db")
-const { budibaseAppsDir } = require("../src/utilities/budibaseDir")
+// load environment
+const env = require("../src/environment")
 
 // Script to export a chosen budibase app into a package
 // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId
@@ -25,6 +26,9 @@ yargs
       },
     },
     async args => {
+      if (!env.isDev()) {
+        throw "Only works in dev"
+      }
       const name = args.name,
         appId = args.appId
       console.log("Exporting app..")
@@ -34,10 +38,11 @@ yargs
         )
         return
       }
-      const exportPath = join(budibaseAppsDir(), "templates", "app", name, "db")
+      const exportPath = join(process.cwd(), name, "db")
       fs.ensureDirSync(exportPath)
       const writeStream = fs.createWriteStream(join(exportPath, "dump.text"))
       // perform couch dump
+
       const instanceDb = new CouchDB(appId)
       await instanceDb.dump(writeStream, {})
       console.log(`Template ${name} exported to ${exportPath}`)

packages/server/src/api/controllers/static/index.js

@@ -6,10 +6,7 @@ const fetch = require("node-fetch")
 const uuid = require("uuid")
 const { prepareUpload } = require("../deploy/utils")
 const { processString } = require("@budibase/string-templates")
-const {
-  budibaseAppsDir,
-  budibaseTempDir,
-} = require("../../../utilities/budibaseDir")
+const { budibaseTempDir } = require("../../../utilities/budibaseDir")
 const { getDeployedApps } = require("../../../utilities/builder/hosting")
 const CouchDB = require("../../../db")
 const setBuilderToken = require("../../../utilities/builder/setBuilderToken")
@@ -18,15 +15,36 @@ const env = require("../../../environment")
 const { OBJ_STORE_DIRECTORY } = require("../../../constants")
 const fileProcessor = require("../../../utilities/fileSystem/processor")
 
+const BB_CDN = "https://cdn.app.budi.live/assets"
+
 function objectStoreUrl() {
   if (env.SELF_HOSTED) {
     // can use a relative url for this as all goes through the proxy (this is hosted in minio)
     return OBJ_STORE_DIRECTORY
   } else {
-    return "https://cdn.app.budi.live/assets"
+    return BB_CDN
   }
 }
 
+function internalObjectStoreUrl() {
+  if (env.SELF_HOSTED) {
+    return (env.MINIO_URL + OBJ_STORE_DIRECTORY).replace(
+      /(https?:\/\/)|(\/)+/g,
+      "$1$2"
+    )
+  } else {
+    return BB_CDN
+  }
+}
+
+async function returnObjectStoreFile(ctx, path) {
+  const S3_URL = `${internalObjectStoreUrl()}/${path}`
+  const response = await fetch(S3_URL)
+  const body = await response.text()
+  ctx.set("Content-Type", response.headers.get("Content-Type"))
+  ctx.body = body
+}
+
 async function checkForSelfHostedURL(ctx) {
   // the "appId" component of the URL may actually be a specific self hosted URL
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
@@ -102,69 +120,47 @@ exports.serveApp = async function(ctx) {
 }
 
 exports.serveAttachment = async function(ctx) {
-  const appId = ctx.user.appId
-  const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments")
-
-  // Serve from object store
-  if (env.isProd()) {
-    const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file)
-    const response = await fetch(S3_URL)
-    const body = await response.text()
-    ctx.set("Content-Type", response.headers.get("Content-Type"))
-    ctx.body = body
-    return
-  }
-
-  await send(ctx, ctx.file, { root: attachmentsPath })
+  await returnObjectStoreFile(
+    ctx,
+    join(ctx.user.appId, "attachments", ctx.file)
+  )
 }
 
 exports.serveAppAsset = async function(ctx) {
-  // default to homedir
-  const appPath = resolve(budibaseAppsDir(), ctx.user.appId, "public")
-
-  await send(ctx, ctx.file, { root: ctx.devPath || appPath })
+  if (env.isDev() || env.isTest()) {
+    return send(ctx, ctx.file, { root: budibaseTempDir() })
+  }
+  await returnObjectStoreFile(ctx, join(ctx.user.appId, "public", ctx.file))
 }
 
 exports.serveComponentLibrary = async function(ctx) {
   const appId = ctx.query.appId || ctx.appId
-  // default to homedir
-  let componentLibraryPath = resolve(
-    budibaseAppsDir(),
-    appId,
-    "node_modules",
-    decodeURI(ctx.query.library),
-    "package",
-    "dist"
-  )
 
-  if (env.isDev()) {
-    componentLibraryPath = join(
+  if (env.isDev() || env.isTest()) {
+    const componentLibraryPath = join(
       budibaseTempDir(),
       decodeURI(ctx.query.library),
       "dist"
     )
-  } else {
-    let componentLib = "componentlibrary"
-    if (ctx.user.version) {
-      componentLib += `-${ctx.user.version}`
-    } else {
-      componentLib += `-${COMP_LIB_BASE_APP_VERSION}`
-    }
-    const S3_URL = encodeURI(
-      join(
-        objectStoreUrl(appId),
-        componentLib,
-        ctx.query.library,
-        "dist",
-        "index.js"
-      )
-    )
-    const response = await fetch(S3_URL)
-    const body = await response.text()
-    ctx.type = "application/javascript"
-    ctx.body = body
-    return
+    return send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
   }
-
-  await send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
+  let componentLib = "componentlibrary"
+  if (ctx.user.version) {
+    componentLib += `-${ctx.user.version}`
+  } else {
+    componentLib += `-${COMP_LIB_BASE_APP_VERSION}`
+  }
+  const S3_URL = encodeURI(
+    join(
+      objectStoreUrl(appId),
+      componentLib,
+      ctx.query.library,
+      "dist",
+      "index.js"
+    )
+  )
+  const response = await fetch(S3_URL)
+  const body = await response.text()
+  ctx.type = "application/javascript"
+  ctx.body = body
 }

packages/server/src/api/index.js

@@ -2,7 +2,6 @@ const Router = require("@koa/router")
 const authenticated = require("../middleware/authenticated")
 const compress = require("koa-compress")
 const zlib = require("zlib")
-const { budibaseAppsDir } = require("../utilities/budibaseDir")
 const { mainRoutes, authRoutes, staticRoutes } = require("./routes")
 const pkg = require("../../package.json")
 
@@ -24,7 +23,6 @@ router
   )
   .use(async (ctx, next) => {
     ctx.config = {
-      latestPackagesFolder: budibaseAppsDir(),
       jwtSecret: env.JWT_SECRET,
       useAppRootPath: true,
     }

packages/server/src/utilities/fileSystem/index.js

@@ -8,6 +8,7 @@ const { ObjectStoreBuckets } = require("../../constants")
 const {
   upload,
   retrieve,
+  retrieveToTmp,
   streamUpload,
   deleteFolder,
   downloadTarball,
@@ -38,6 +39,10 @@ exports.init = () => {
   if (!fs.existsSync(tempDir)) {
     fs.mkdirSync(tempDir)
   }
+  const clientLibPath = join(budibaseTempDir(), "budibase-client.js")
+  if (env.isTest() && !fs.existsSync(clientLibPath)) {
+    fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath)
+  }
 }
 
 /**
@@ -224,3 +229,4 @@ exports.cleanup = appIds => {
  */
 exports.upload = upload
 exports.retrieve = retrieve
+exports.retrieveToTmp = retrieveToTmp

packages/server/src/utilities/fileSystem/utilities.js

@@ -10,6 +10,7 @@ const fs = require("fs")
 const { budibaseTempDir } = require("../budibaseDir")
 const env = require("../../environment")
 const { ObjectStoreBuckets } = require("../../constants")
+const uuid = require("uuid/v4")
 
 const streamPipeline = promisify(stream.pipeline)
 
@@ -146,11 +147,11 @@ exports.streamUpload = async (bucket, filename, stream) => {
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-exports.retrieve = async (bucket, filename) => {
+exports.retrieve = async (bucket, filepath) => {
   const objectStore = exports.ObjectStore(bucket)
   const params = {
     Bucket: bucket,
-    Key: sanitize(filename).replace(/\\/g, "/"),
+    Key: sanitize(filepath).replace(/\\/g, "/"),
   }
   const response = await objectStore.getObject(params).promise()
   // currently these are all strings
@@ -161,6 +162,16 @@ exports.retrieve = async (bucket, filename) => {
   }
 }
 
+/**
+ * Same as retrieval function but puts to a temporary file.
+ */
+exports.retrieveToTmp = async (bucket, filepath) => {
+  const data = await exports.retrieve(bucket, filepath)
+  const outputPath = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(outputPath, data)
+  return outputPath
+}
+
 exports.deleteFolder = async (bucket, folder) => {
   const client = exports.ObjectStore(bucket)
   const listParams = {