Major update - removing the use of context for PouchDB instances, swapping knowledge of PouchDB to the PouchLike structure that replaces it.

packages/backend-core/src/cache/tests/writethrough.spec.js

@@ -1,6 +1,6 @@
 require("../../../tests/utilities/TestConfiguration")
 const { Writethrough } = require("../writethrough")
-const { dangerousGetDB } = require("../../db")
+const { getDB } = require("../../db")
 const tk = require("timekeeper")
 
 const START_DATE = Date.now()
@@ -8,8 +8,8 @@ tk.freeze(START_DATE)
 
 const DELAY = 5000
 
-const db = dangerousGetDB("test")
+const db = getDB("test")
-const db2 = dangerousGetDB("test2")
+const db2 = getDB("test2")
 const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
 
 describe("writethrough", () => {

packages/backend-core/src/cache/writethrough.ts

@@ -1,7 +1,7 @@
 import BaseCache from "./base"
 import { getWritethroughClient } from "../redis/init"
 import { logWarn } from "../logging"
-import PouchDB from "pouchdb"
+import { PouchLike } from "../couch"
 
 const DEFAULT_WRITE_RATE_MS = 10000
 let CACHE: BaseCache | null = null
@@ -19,7 +19,7 @@ async function getCache() {
   return CACHE
 }
 
-function makeCacheKey(db: PouchDB.Database, key: string) {
+function makeCacheKey(db: PouchLike, key: string) {
   return db.name + key
 }
 
@@ -28,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
 }
 
 export async function put(
-  db: PouchDB.Database,
+  db: PouchLike,
   doc: any,
   writeRateMs: number = DEFAULT_WRITE_RATE_MS
 ) {
@@ -64,7 +64,7 @@ export async function put(
   return { ok: true, id: output._id, rev: output._rev }
 }
 
-export async function get(db: PouchDB.Database, id: string): Promise<any> {
+export async function get(db: PouchLike, id: string): Promise<any> {
   const cache = await getCache()
   const cacheKey = makeCacheKey(db, id)
   let cacheItem: CacheItem = await cache.get(cacheKey)
@@ -77,7 +77,7 @@ export async function get(db: PouchDB.Database, id: string): Promise<any> {
 }
 
 export async function remove(
-  db: PouchDB.Database,
+  db: PouchLike,
   docOrId: any,
   rev?: any
 ): Promise<void> {
@@ -95,13 +95,10 @@ export async function remove(
 }
 
 export class Writethrough {
-  db: PouchDB.Database
+  db: PouchLike
   writeRateMs: number
 
-  constructor(
+  constructor(db: PouchLike, writeRateMs: number = DEFAULT_WRITE_RATE_MS) {
-    db: PouchDB.Database,
-    writeRateMs: number = DEFAULT_WRITE_RATE_MS
-  ) {
     this.db = db
     this.writeRateMs = writeRateMs
   }

packages/backend-core/src/context/constants.ts

@@ -1,17 +1,5 @@
 export enum ContextKey {
   TENANT_ID = "tenantId",
-  GLOBAL_DB = "globalDb",
   APP_ID = "appId",
   IDENTITY = "identity",
-  // whatever the request app DB was
-  CURRENT_DB = "currentDb",
-  // get the prod app DB from the request
-  PROD_DB = "prodDb",
-  // get the dev app DB from the request
-  DEV_DB = "devDb",
-  DB_OPTS = "dbOpts",
-  // check if something else is using the context, don't close DB
-  TENANCY_IN_USE = "tenancyInUse",
-  APP_IN_USE = "appInUse",
-  IDENTITY_IN_USE = "identityInUse",
 }

packages/backend-core/src/context/index.ts

@@ -1,20 +1,12 @@
 import env from "../environment"
 import { SEPARATOR, DocumentType } from "../db/constants"
 import cls from "./FunctionContext"
-import { dangerousGetDB, closeDB } from "../db"
 import { baseGlobalDBName } from "../db/tenancy"
 import { IdentityContext } from "@budibase/types"
 import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
 import { ContextKey } from "./constants"
-import PouchDB from "pouchdb"
+import { PouchLike } from "../couch"
-import {
+import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
-  updateUsing,
-  closeWithUsing,
-  setAppTenantId,
-  setIdentity,
-  closeAppDBs,
-  getContextDB,
-} from "./utils"
 
 export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
 
@@ -22,29 +14,19 @@ export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
 // store an app ID to pretend there is a context
 let TEST_APP_ID: string | null = null
 
-export const closeTenancy = async () => {
-  try {
-    if (env.USE_COUCH) {
-      const db = getGlobalDB()
-      await closeDB(db)
-    }
-  } catch (err) {
-    // no DB found - skip closing
-    return
-  }
-  // clear from context now that database is closed/task is finished
-  cls.setOnContext(ContextKey.TENANT_ID, null)
-  cls.setOnContext(ContextKey.GLOBAL_DB, null)
-}
-
-// export const isDefaultTenant = () => {
-//   return getTenantId() === DEFAULT_TENANT_ID
-// }
-
 export const isMultiTenant = () => {
   return env.MULTI_TENANCY
 }
 
+const setAppTenantId = (appId: string) => {
+  const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
+  updateTenantId(appTenantId)
+}
+
+const setIdentity = (identity: IdentityContext | null) => {
+  cls.setOnContext(ContextKey.IDENTITY, identity)
+}
+
 /**
  * Given an app ID this will attempt to retrieve the tenant ID from it.
  * @return {null|string} The tenant ID found within the app ID.
@@ -78,47 +60,28 @@ export const doInContext = async (appId: string, task: any) => {
   })
 }
 
-export const doInTenant = (tenantId: string | null, task: any) => {
+export const doInTenant = (tenantId: string | null, task: any): any => {
   // make sure default always selected in single tenancy
   if (!env.MULTI_TENANCY) {
     tenantId = tenantId || DEFAULT_TENANT_ID
   }
-  // the internal function is so that we can re-use an existing
-  // context - don't want to close DB on a parent context
-  async function internal(opts = { existing: false }) {
-    // set the tenant id + global db if this is a new context
-    if (!opts.existing) {
-      updateTenantId(tenantId)
-    }
 
-    try {
+  return cls.run(async () => {
-      // invoke the task
+    updateTenantId(tenantId)
-      return await task()
+    return await task()
-    } finally {
+  })
-      await closeWithUsing(ContextKey.TENANCY_IN_USE, () => {
-        return closeTenancy()
-      })
-    }
-  }
-
-  const existing = cls.getFromContext(ContextKey.TENANT_ID) === tenantId
-  return updateUsing(ContextKey.TENANCY_IN_USE, existing, internal)
 }
 
-export const doInAppContext = (appId: string, task: any) => {
+export const doInAppContext = (appId: string, task: any): any => {
   if (!appId) {
     throw new Error("appId is required")
   }
 
   const identity = getIdentity()
 
-  // the internal function is so that we can re-use an existing
+  return cls.run(async () => {
-  // context - don't want to close DB on a parent context
-  async function internal(opts = { existing: false }) {
     // set the app tenant id
-    if (!opts.existing) {
+    setAppTenantId(appId)
-      setAppTenantId(appId)
-    }
     // set the app ID
     cls.setOnContext(ContextKey.APP_ID, appId)
 
@@ -126,47 +89,28 @@ export const doInAppContext = (appId: string, task: any) => {
     if (identity) {
       setIdentity(identity)
     }
-    try {
+    // invoke the task
-      // invoke the task
+    return await task()
-      return await task()
+  })
-    } finally {
-      await closeWithUsing(ContextKey.APP_IN_USE, async () => {
-        await closeAppDBs()
-        await closeTenancy()
-      })
-    }
-  }
-  const existing = cls.getFromContext(ContextKey.APP_ID) === appId
-  return updateUsing(ContextKey.APP_IN_USE, existing, internal)
 }
 
-export const doInIdentityContext = (identity: IdentityContext, task: any) => {
+export const doInIdentityContext = (
+  identity: IdentityContext,
+  task: any
+): any => {
   if (!identity) {
     throw new Error("identity is required")
   }
 
-  async function internal(opts = { existing: false }) {
+  return cls.run(async () => {
-    if (!opts.existing) {
+    cls.setOnContext(ContextKey.IDENTITY, identity)
-      cls.setOnContext(ContextKey.IDENTITY, identity)
+    // set the tenant so that doInTenant will preserve identity
-      // set the tenant so that doInTenant will preserve identity
+    if (identity.tenantId) {
-      if (identity.tenantId) {
+      updateTenantId(identity.tenantId)
-        updateTenantId(identity.tenantId)
-      }
     }
-
+    // invoke the task
-    try {
+    return await task()
-      // invoke the task
+  })
-      return await task()
-    } finally {
-      await closeWithUsing(ContextKey.IDENTITY_IN_USE, async () => {
-        setIdentity(null)
-        await closeTenancy()
-      })
-    }
-  }
-
-  const existing = cls.getFromContext(ContextKey.IDENTITY)
-  return updateUsing(ContextKey.IDENTITY_IN_USE, existing, internal)
 }
 
 export const getIdentity = (): IdentityContext | undefined => {
@@ -179,15 +123,10 @@ export const getIdentity = (): IdentityContext | undefined => {
 
 export const updateTenantId = (tenantId: string | null) => {
   cls.setOnContext(ContextKey.TENANT_ID, tenantId)
-  if (env.USE_COUCH) {
-    setGlobalDB(tenantId)
-  }
 }
 
 export const updateAppId = async (appId: string) => {
   try {
-    // have to close first, before removing the databases from context
-    await closeAppDBs()
     cls.setOnContext(ContextKey.APP_ID, appId)
   } catch (err) {
     if (env.isTest()) {
@@ -198,19 +137,9 @@ export const updateAppId = async (appId: string) => {
   }
 }
 
-export const setGlobalDB = (tenantId: string | null) => {
+export const getGlobalDB = (): PouchLike => {
-  const dbName = baseGlobalDBName(tenantId)
+  const tenantId = cls.getFromContext(ContextKey.TENANT_ID)
-  const db = dangerousGetDB(dbName)
+  return new PouchLike(baseGlobalDBName(tenantId))
-  cls.setOnContext(ContextKey.GLOBAL_DB, db)
-  return db
-}
-
-export const getGlobalDB = () => {
-  const db = cls.getFromContext(ContextKey.GLOBAL_DB)
-  if (!db) {
-    throw new Error("Global DB not found")
-  }
-  return db
 }
 
 export const isTenantIdSet = () => {
@@ -246,22 +175,25 @@ export const isTenancyEnabled = () => {
  * Opens the app database based on whatever the request
  * contained, dev or prod.
  */
-export const getAppDB = (opts?: any) => {
+export const getAppDB = (opts?: any): PouchLike => {
-  return getContextDB(ContextKey.CURRENT_DB, opts)
+  const appId = getAppId()
+  return new PouchLike(appId, opts)
 }
 
 /**
  * This specifically gets the prod app ID, if the request
  * contained a development app ID, this will open the prod one.
  */
-export const getProdAppDB = (opts?: any) => {
+export const getProdAppDB = (opts?: any): PouchLike => {
-  return getContextDB(ContextKey.PROD_DB, opts)
+  const appId = getAppId()
+  return new PouchLike(getProdAppID(appId), opts)
 }
 
 /**
  * This specifically gets the dev app ID, if the request
  * contained a prod app ID, this will open the dev one.
  */
-export const getDevAppDB = (opts?: any) => {
+export const getDevAppDB = (opts?: any): PouchLike => {
-  return getContextDB(ContextKey.DEV_DB, opts)
+  const appId = getAppId()
+  return new PouchLike(getDevelopmentAppID(appId), opts)
 }

packages/backend-core/src/context/tests/index.spec.ts

@@ -5,8 +5,8 @@ import env from "../../environment"
 
 // must use require to spy index file exports due to known issue in jest
 const dbUtils = require("../../db")
-jest.spyOn(dbUtils, "closeDB")
+jest.spyOn(dbUtils, "closePouchDB")
-jest.spyOn(dbUtils, "dangerousGetDB")
+jest.spyOn(dbUtils, "getDB")
 
 describe("context", () => {
   beforeEach(() => {
@@ -25,8 +25,8 @@ describe("context", () => {
           const db = context.getGlobalDB()
           expect(db.name).toBe("global-db")
         })
-        expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
+        expect(dbUtils.getDB).toHaveBeenCalledTimes(1)
-        expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
+        expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1)
       })
     })
 
@@ -85,8 +85,8 @@ describe("context", () => {
           const db = context.getGlobalDB()
           expect(db.name).toBe("test_global-db")
         })
-        expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
+        expect(dbUtils.getDB).toHaveBeenCalledTimes(1)
-        expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
+        expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1)
       })
 
       it("sets the tenant id when nested with same tenant id", async () => {
@@ -123,8 +123,8 @@ describe("context", () => {
         })
 
         // only 1 db is opened and closed
-        expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
+        expect(dbUtils.getDB).toHaveBeenCalledTimes(1)
-        expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
+        expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1)
       })
 
       it("sets different tenant id inside another context", () => {

packages/backend-core/src/context/utils.ts

@@ -1,109 +0,0 @@
-import {
-  DEFAULT_TENANT_ID,
-  getAppId,
-  getTenantIDFromAppID,
-  updateTenantId,
-} from "./index"
-import cls from "./FunctionContext"
-import { IdentityContext } from "@budibase/types"
-import { ContextKey } from "./constants"
-import { dangerousGetDB, closeDB } from "../db"
-import { isEqual } from "lodash"
-import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
-import env from "../environment"
-
-export async function updateUsing(
-  usingKey: string,
-  existing: boolean,
-  internal: (opts: { existing: boolean }) => Promise<any>
-) {
-  const using = cls.getFromContext(usingKey)
-  if (using && existing) {
-    cls.setOnContext(usingKey, using + 1)
-    return internal({ existing: true })
-  } else {
-    return cls.run(async () => {
-      cls.setOnContext(usingKey, 1)
-      return internal({ existing: false })
-    })
-  }
-}
-
-export async function closeWithUsing(
-  usingKey: string,
-  closeFn: () => Promise<any>
-) {
-  const using = cls.getFromContext(usingKey)
-  if (!using || using <= 1) {
-    await closeFn()
-  } else {
-    cls.setOnContext(usingKey, using - 1)
-  }
-}
-
-export const setAppTenantId = (appId: string) => {
-  const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
-  updateTenantId(appTenantId)
-}
-
-export const setIdentity = (identity: IdentityContext | null) => {
-  cls.setOnContext(ContextKey.IDENTITY, identity)
-}
-
-// this function makes sure the PouchDB objects are closed and
-// fully deleted when finished - this protects against memory leaks
-export async function closeAppDBs() {
-  const dbKeys = [ContextKey.CURRENT_DB, ContextKey.PROD_DB, ContextKey.DEV_DB]
-  for (let dbKey of dbKeys) {
-    const db = cls.getFromContext(dbKey)
-    if (!db) {
-      continue
-    }
-    await closeDB(db)
-    // clear the DB from context, incase someone tries to use it again
-    cls.setOnContext(dbKey, null)
-  }
-  // clear the app ID now that the databases are closed
-  if (cls.getFromContext(ContextKey.APP_ID)) {
-    cls.setOnContext(ContextKey.APP_ID, null)
-  }
-  if (cls.getFromContext(ContextKey.DB_OPTS)) {
-    cls.setOnContext(ContextKey.DB_OPTS, null)
-  }
-}
-
-export function getContextDB(key: string, opts: any) {
-  const dbOptsKey = `${key}${ContextKey.DB_OPTS}`
-  let storedOpts = cls.getFromContext(dbOptsKey)
-  let db = cls.getFromContext(key)
-  if (db && isEqual(opts, storedOpts)) {
-    return db
-  }
-
-  const appId = getAppId()
-  let toUseAppId
-
-  switch (key) {
-    case ContextKey.CURRENT_DB:
-      toUseAppId = appId
-      break
-    case ContextKey.PROD_DB:
-      toUseAppId = getProdAppID(appId)
-      break
-    case ContextKey.DEV_DB:
-      toUseAppId = getDevelopmentAppID(appId)
-      break
-  }
-  db = dangerousGetDB(toUseAppId, opts)
-  try {
-    cls.setOnContext(key, db)
-    if (opts) {
-      cls.setOnContext(dbOptsKey, opts)
-    }
-  } catch (err) {
-    if (!env.isTest()) {
-      throw err
-    }
-  }
-  return db
-}

packages/backend-core/src/couch/index.ts

@@ -1,3 +1,4 @@
 export * from "./couch"
 export * from "./pouchLike"
 export * from "./utils"
+export { init } from "./pouchDB"

packages/backend-core/src/couch/pouchDB.ts

@@ -0,0 +1,37 @@
+import PouchDB from "pouchdb"
+import env from "../environment"
+import { PouchOptions } from "@budibase/types"
+import * as pouch from "../db/pouch"
+
+let Pouch: any
+let initialised = false
+
+export async function init(opts?: PouchOptions) {
+  Pouch = pouch.getPouch(opts)
+  initialised = true
+}
+
+const checkInitialised = () => {
+  if (!initialised) {
+    throw new Error("init has not been called")
+  }
+}
+
+export function getPouchDB(dbName: string, opts?: any): PouchDB.Database {
+  checkInitialised()
+  return new Pouch(dbName, opts)
+}
+
+// use this function if you have called getPouchDB - close
+// the databases you've opened once finished
+export async function closePouchDB(db: PouchDB.Database) {
+  if (!db || env.isTest()) {
+    return
+  }
+  try {
+    // specifically await so that if there is an error, it can be ignored
+    return await db.close()
+  } catch (err) {
+    // ignore error, already closed
+  }
+}

packages/backend-core/src/couch/pouchLike.ts

@@ -2,12 +2,16 @@ import Nano from "nano"
 import { AnyDocument } from "@budibase/types"
 import { getCouchInfo } from "./couch"
 import { directCouchCall } from "./utils"
-import { getPouchDB } from "../db"
+import { getPouchDB } from "./pouchDB"
 
 export type PouchLikeOpts = {
   skip_setup?: boolean
 }
 
+export type PutOpts = {
+  force?: boolean
+}
+
 export type QueryOpts = {
   include_docs?: boolean
   startkey?: string
@@ -19,6 +23,10 @@ export type QueryOpts = {
   keys?: string[]
 }
 
+type QueryResp<T> = Promise<{
+  rows: { doc?: T | any; value?: any }[]
+}>
+
 export class PouchLike {
   public readonly name: string
   private static nano: Nano.ServerScope
@@ -45,11 +53,15 @@ export class PouchLike {
     })
   }
 
+  async exists() {
+    let response = await directCouchCall(`/${this.name}`, "HEAD")
+    return response.status === 200
+  }
+
   async checkSetup() {
     let shouldCreate = !this.pouchOpts?.skip_setup
     // check exists in a lightweight fashion
-    let response = await directCouchCall(`/${this.name}`, "HEAD")
+    let exists = await this.exists()
-    let exists = response.status === 200
     if (!shouldCreate && !exists) {
       throw new Error("DB does not exist")
     }
@@ -70,26 +82,43 @@ export class PouchLike {
     }
   }
 
-  async info() {
+  async get<T>(id?: string): Promise<T | any> {
-    const db = PouchLike.nano.db.use(this.name)
-    return db.info()
-  }
-
-  async get(id: string) {
     const db = await this.checkSetup()
+    if (!id) {
+      throw new Error("Unable to get doc without a valid _id.")
+    }
     return this.updateOutput(() => db.get(id))
   }
 
-  async remove(id: string, rev: string) {
+  async remove(id?: string, rev?: string) {
     const db = await this.checkSetup()
+    if (!id || !rev) {
+      throw new Error("Unable to remove doc without a valid _id and _rev.")
+    }
     return this.updateOutput(() => db.destroy(id, rev))
   }
 
-  async put(document: AnyDocument) {
+  async put(document: AnyDocument, opts?: PutOpts) {
     if (!document._id) {
       throw new Error("Cannot store document without _id field.")
     }
     const db = await this.checkSetup()
+    if (!document.createdAt) {
+      document.createdAt = new Date().toISOString()
+    }
+    document.updatedAt = new Date().toISOString()
+    if (opts?.force && document._id) {
+      try {
+        const existing = await this.get(document._id)
+        if (existing) {
+          document._rev = existing._rev
+        }
+      } catch (err: any) {
+        if (err.status !== 404) {
+          throw err
+        }
+      }
+    }
     return this.updateOutput(() => db.insert(document))
   }
 
@@ -98,12 +127,12 @@ export class PouchLike {
     return this.updateOutput(() => db.bulk({ docs: documents }))
   }
 
-  async allDocs(params: QueryOpts) {
+  async allDocs<T>(params: QueryOpts): QueryResp<T> {
     const db = await this.checkSetup()
     return this.updateOutput(() => db.list(params))
   }
 
-  async query(viewName: string, params: QueryOpts) {
+  async query<T>(viewName: string, params: QueryOpts): QueryResp<T> {
     const db = await this.checkSetup()
     const [database, view] = viewName.split("/")
     return this.updateOutput(() => db.view(database, view, params))

packages/backend-core/src/db/Replication.ts

@@ -1,4 +1,4 @@
-import { dangerousGetDB, closeDB } from "."
+import { getPouchDB, closePouchDB } from "../couch/pouchDB"
 import { DocumentType } from "./constants"
 
 class Replication {
@@ -12,12 +12,12 @@ class Replication {
    * @param {String} target - the DB you want to replicate to, or rollback from
    */
   constructor({ source, target }: any) {
-    this.source = dangerousGetDB(source)
+    this.source = getPouchDB(source)
-    this.target = dangerousGetDB(target)
+    this.target = getPouchDB(target)
   }
 
   close() {
-    return Promise.all([closeDB(this.source), closeDB(this.target)])
+    return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
   }
 
   promisify(operation: any, opts = {}) {
@@ -68,7 +68,7 @@ class Replication {
   async rollback() {
     await this.target.destroy()
     // Recreate the DB again
-    this.target = dangerousGetDB(this.target.name)
+    this.target = getPouchDB(this.target.name)
     // take the opportunity to remove deleted tombstones
     await this.replicate()
   }

packages/backend-core/src/db/index.ts

@@ -1,87 +1,30 @@
-import * as pouch from "./pouch"
 import env from "../environment"
-import { PouchOptions, CouchFindOptions } from "@budibase/types"
+import { CouchFindOptions } from "@budibase/types"
-import PouchDB from "pouchdb"
 import { PouchLike } from "../couch"
 import { directCouchQuery } from "../couch"
-export { directCouchQuery } from "../couch"
+export { init, PouchLike } from "../couch"
 
-const openDbs: string[] = []
-let Pouch: any
 let initialised = false
 const dbList = new Set()
 
-if (env.MEMORY_LEAK_CHECK) {
-  setInterval(() => {
-    console.log("--- OPEN DBS ---")
-    console.log(openDbs)
-  }, 5000)
-}
-
-const put =
-  (dbPut: any) =>
-  async (doc: any, options = {}) => {
-    if (!doc.createdAt) {
-      doc.createdAt = new Date().toISOString()
-    }
-    doc.updatedAt = new Date().toISOString()
-    return dbPut(doc, options)
-  }
-
 const checkInitialised = () => {
   if (!initialised) {
     throw new Error("init has not been called")
   }
 }
 
-export async function init(opts?: PouchOptions) {
+export function getDB(dbName: string, opts?: any): PouchLike {
-  Pouch = pouch.getPouch(opts)
-  initialised = true
-}
-
-export function getPouchDB(dbName: string, opts?: any): PouchDB.Database {
-  checkInitialised()
   if (env.isTest()) {
     dbList.add(dbName)
   }
-  const db = new Pouch(dbName, opts)
-  if (env.MEMORY_LEAK_CHECK) {
-    openDbs.push(db.name)
-  }
-  const dbPut = db.put
-  db.put = put(dbPut)
-  return db
-}
-
-// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
-// this function is prone to leaks, should only be used
-// in situations that using the function doWithDB does not work
-export function dangerousGetDB(dbName: string, opts?: any): PouchLike {
   return new PouchLike(dbName, opts)
 }
 
-// use this function if you have called dangerousGetDB - close
-// the databases you've opened once finished
-export async function closeDB(db: PouchDB.Database) {
-  if (!db || env.isTest()) {
-    return
-  }
-  if (env.MEMORY_LEAK_CHECK) {
-    openDbs.splice(openDbs.indexOf(db.name), 1)
-  }
-  try {
-    // specifically await so that if there is an error, it can be ignored
-    return await db.close()
-  } catch (err) {
-    // ignore error, already closed
-  }
-}
-
 // we have to use a callback for this so that we can close
 // the DB when we're done, without this manual requests would
 // need to close the database when done with it to avoid memory leaks
 export async function doWithDB(dbName: string, cb: any, opts = {}) {
-  const db = dangerousGetDB(dbName, opts)
+  const db = getDB(dbName, opts)
   // need this to be async so that we can correctly close DB after all
   // async operations have been completed
   return await cb(db)

packages/backend-core/src/db/tests/index.spec.js

@@ -1,11 +1,11 @@
 require("../../../tests/utilities/TestConfiguration")
-const { dangerousGetDB } = require("../")
+const { getDB } = require("../")
 
 describe("db", () => { 
   
   describe("getDB", () => {
     it("returns a db", async () => {
-      const db = dangerousGetDB("test")
+      const db = getDB("test")
       expect(db).toBeDefined()
       expect(db._adapter).toBe("memory")
       expect(db.prefix).toBe("_pouch_")
@@ -13,7 +13,7 @@ describe("db", () => {
     })
 
     it("uses the custom put function", async () => {
-      const db = dangerousGetDB("test")
+      const db = getDB("test")
       let doc = { _id: "test" }
       await db.put(doc)
       doc = await db.get(doc._id)

packages/backend-core/src/db/views.ts

@@ -1,6 +1,6 @@
 import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils"
 import { getGlobalDB } from "../context"
-import PouchDB from "pouchdb"
+import { PouchLike, QueryOpts } from "../couch"
 import { StaticDatabases } from "./constants"
 import { doWithDB } from "./"
 
@@ -19,7 +19,7 @@ interface DesignDocument {
   views: any
 }
 
-async function removeDeprecated(db: PouchDB.Database, viewName: ViewName) {
+async function removeDeprecated(db: PouchLike, viewName: ViewName) {
   // @ts-ignore
   if (!DeprecatedViews[viewName]) {
     return
@@ -70,16 +70,13 @@ export const createAccountEmailView = async () => {
       emit(doc.email.toLowerCase(), doc._id)
     }
   }`
-  await doWithDB(
+  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => {
-    StaticDatabases.PLATFORM_INFO.name,
+    await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
-    async (db: PouchDB.Database) => {
+  })
-      await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
-    }
-  )
 }
 
 export const createUserAppView = async () => {
-  const db = getGlobalDB() as PouchDB.Database
+  const db = getGlobalDB()
   const viewJs = `function(doc) {
     if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) {
       for (let prodAppId of Object.keys(doc.roles)) {
@@ -117,12 +114,9 @@ export const createPlatformUserView = async () => {
       emit(doc._id.toLowerCase(), doc._id)
     }
   }`
-  await doWithDB(
+  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => {
-    StaticDatabases.PLATFORM_INFO.name,
+    await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
-    async (db: PouchDB.Database) => {
+  })
-      await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
-    }
-  )
 }
 
 export interface QueryViewOptions {
@@ -131,13 +125,13 @@ export interface QueryViewOptions {
 
 export const queryView = async <T>(
   viewName: ViewName,
-  params: PouchDB.Query.Options<T, T>,
+  params: QueryOpts,
-  db: PouchDB.Database,
+  db: PouchLike,
   createFunc: any,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   try {
-    let response = await db.query<T, T>(`database/${viewName}`, params)
+    let response = await db.query(`database/${viewName}`, params)
     const rows = response.rows
     const docs = rows.map(row => (params.include_docs ? row.doc : row.value))
 
@@ -161,7 +155,7 @@ export const queryView = async <T>(
 
 export const queryPlatformView = async <T>(
   viewName: ViewName,
-  params: PouchDB.Query.Options<T, T>,
+  params: QueryOpts,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
@@ -169,19 +163,16 @@ export const queryPlatformView = async <T>(
     [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
   }
 
-  return doWithDB(
+  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => {
-    StaticDatabases.PLATFORM_INFO.name,
+    const createFn = CreateFuncByName[viewName]
-    async (db: PouchDB.Database) => {
+    return queryView(viewName, params, db, createFn, opts)
-      const createFn = CreateFuncByName[viewName]
+  })
-      return queryView(viewName, params, db, createFn, opts)
-    }
-  )
 }
 
 export const queryGlobalView = async <T>(
   viewName: ViewName,
-  params: PouchDB.Query.Options<T, T>,
+  params: QueryOpts,
-  db?: PouchDB.Database,
+  db?: PouchLike,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
@@ -192,7 +183,7 @@ export const queryGlobalView = async <T>(
   }
   // can pass DB in if working with something specific
   if (!db) {
-    db = getGlobalDB() as PouchDB.Database
+    db = getGlobalDB()
   }
   const createFn = CreateFuncByName[viewName]
   return queryView(viewName, params, db, createFn, opts)

packages/backend-core/src/environment.ts

@@ -69,7 +69,6 @@ const env = {
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
   SERVICE: process.env.SERVICE || "budibase",
-  MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false,
   LOG_LEVEL: process.env.LOG_LEVEL,
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   DEPLOYMENT_ENVIRONMENT:

packages/backend-core/src/index.ts

@@ -21,6 +21,7 @@ import * as middleware from "./middleware"
 import plugins from "./plugin"
 import encryption from "./security/encryption"
 import * as queue from "./queue"
+import * as types from "./types"
 
 // mimic the outer package exports
 import * as db from "./pkg/db"
@@ -67,6 +68,7 @@ const core = {
   encryption,
   queue,
   permissions,
+  ...types,
 }
 
 export = core

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,6 +1,6 @@
 require("../../../tests/utilities/TestConfiguration")
 const { runMigrations, getMigrationsDoc } = require("../index")
-const { dangerousGetDB } = require("../../db")
+const { getDB } = require("../../db")
 const {
   StaticDatabases,
 } = require("../../db/utils")
@@ -18,7 +18,7 @@ describe("migrations", () => {
   }]
 
   beforeEach(() => {
-    db = dangerousGetDB(StaticDatabases.GLOBAL.name)
+    db = getDB(StaticDatabases.GLOBAL.name)
   })
 
   afterEach(async () => {

packages/backend-core/src/types.ts

@@ -0,0 +1 @@
+export { PouchLike } from "./couch"

packages/backend-core/src/users.ts

@@ -8,10 +8,9 @@ import { queryGlobalView } from "./db/views"
 import { UNICODE_MAX } from "./db/constants"
 import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
-import PouchDB from "pouchdb"
 
 export const bulkGetGlobalUsersById = async (userIds: string[]) => {
-  const db = getGlobalDB() as PouchDB.Database
+  const db = getGlobalDB()
   return (
     await db.allDocs({
       keys: userIds,
@@ -21,7 +20,7 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => {
 }
 
 export const bulkUpdateGlobalUsers = async (users: User[]) => {
-  const db = getGlobalDB() as PouchDB.Database
+  const db = getGlobalDB()
   return (await db.bulkDocs(users)) as BulkDocsResponse
 }
 

packages/server/src/api/controllers/application.ts

@@ -524,12 +524,10 @@ export const sync = async (ctx: any, next: any) => {
   // replicate prod to dev
   const prodAppId = getProdAppID(appId)
 
-  try {
+  // specific case, want to make sure setup is skipped
-    // specific case, want to make sure setup is skipped
+  const prodDb = context.getProdAppDB({ skip_setup: true })
-    const prodDb = context.getProdAppDB({ skip_setup: true })
+  const exists = await prodDb.exists()
-    const info = await prodDb.info()
+  if (!exists) {
-    if (info.error) throw info.error
-  } catch (err) {
     // the database doesn't exist. Don't replicate
     ctx.status = 200
     ctx.body = {

packages/server/src/api/controllers/row/internal.js

@@ -6,7 +6,7 @@ const {
   DocumentType,
   InternalTables,
 } = require("../../../db/utils")
-const { dangerousGetDB } = require("@budibase/backend-core/db")
+const { getDB } = require("@budibase/backend-core/db")
 const userController = require("../user")
 const {
   inputProcessing,
@@ -251,7 +251,7 @@ exports.fetch = async ctx => {
 }
 
 exports.find = async ctx => {
-  const db = dangerousGetDB(ctx.appId)
+  const db = getDB(ctx.appId)
   const table = await db.get(ctx.params.tableId)
   let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId)
   row = await outputProcessing(table, row)

packages/server/src/db/inMemoryView.js

@@ -2,7 +2,7 @@ const newid = require("./newid")
 
 // bypass the main application db config
 // use in memory pouchdb directly
-const { getPouch, closeDB } = require("@budibase/backend-core/db")
+const { getPouch, closePouchDB } = require("@budibase/backend-core/db")
 const Pouch = getPouch({ inMemory: true })
 
 exports.runView = async (view, calculation, group, data) => {
@@ -44,6 +44,6 @@ exports.runView = async (view, calculation, group, data) => {
     return response
   } finally {
     await db.destroy()
-    await closeDB(db)
+    await closePouchDB(db)
   }
 }

packages/server/src/migrations/functions/backfill/app/tables.ts

@@ -1,9 +1,8 @@
-import { events } from "@budibase/backend-core"
+import { events, PouchLike } from "@budibase/backend-core"
 import sdk from "../../../../sdk"
-import PouchDB from "pouchdb"
 
 export const backfill = async (
-  appDb: PouchDB.Database,
+  appDb: PouchLike,
   timestamp: string | number
 ) => {
   const tables = await sdk.tables.getAllInternalTables(appDb)

packages/server/src/sdk/app/backups/imports.ts

@@ -1,4 +1,4 @@
-import { db as dbCore } from "@budibase/backend-core"
+import { db as dbCore, PouchLike } from "@budibase/backend-core"
 import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
@@ -17,7 +17,6 @@ import {
   CouchFindOptions,
   RowAttachment,
 } from "@budibase/types"
-import PouchDB from "pouchdb"
 const uuid = require("uuid/v4")
 const tar = require("tar")
 
@@ -29,10 +28,7 @@ type TemplateType = {
   key?: string
 }
 
-async function updateAttachmentColumns(
+async function updateAttachmentColumns(prodAppId: string, db: PouchLike) {
-  prodAppId: string,
-  db: PouchDB.Database
-) {
   // iterate through attachment documents and update them
   const tables = await sdk.tables.getAllInternalTables(db)
   for (let table of tables) {
@@ -86,7 +82,7 @@ async function updateAttachmentColumns(
   }
 }
 
-async function updateAutomations(prodAppId: string, db: PouchDB.Database) {
+async function updateAutomations(prodAppId: string, db: PouchLike) {
   const automations = (
     await db.allDocs(
       getAutomationParams(null, {
@@ -154,7 +150,7 @@ export function getListOfAppsInMulti(tmpPath: string) {
 
 export async function importApp(
   appId: string,
-  db: PouchDB.Database,
+  db: PouchLike,
   template: TemplateType
 ) {
   let prodAppId = dbCore.getProdAppID(appId)

packages/server/src/sdk/app/backups/statistics.ts

@@ -1,13 +1,12 @@
-import { context, db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore, PouchLike } from "@budibase/backend-core"
 import {
   getDatasourceParams,
   getTableParams,
   getAutomationParams,
   getScreenParams,
 } from "../../../db/utils"
-import PouchDB from "pouchdb"
 
-async function runInContext(appId: string, cb: any, db?: PouchDB.Database) {
+async function runInContext(appId: string, cb: any, db?: PouchLike) {
   if (db) {
     return cb(db)
   } else {
@@ -19,13 +18,10 @@ async function runInContext(appId: string, cb: any, db?: PouchDB.Database) {
   }
 }
 
-export async function calculateDatasourceCount(
+export async function calculateDatasourceCount(appId: string, db?: PouchLike) {
-  appId: string,
-  db?: PouchDB.Database
-) {
   return runInContext(
     appId,
-    async (db: PouchDB.Database) => {
+    async (db: PouchLike) => {
       const datasourceList = await db.allDocs(getDatasourceParams())
       const tableList = await db.allDocs(getTableParams())
       return datasourceList.rows.length + tableList.rows.length
@@ -34,13 +30,10 @@ export async function calculateDatasourceCount(
   )
 }
 
-export async function calculateAutomationCount(
+export async function calculateAutomationCount(appId: string, db?: PouchLike) {
-  appId: string,
-  db?: PouchDB.Database
-) {
   return runInContext(
     appId,
-    async (db: PouchDB.Database) => {
+    async (db: PouchLike) => {
       const automationList = await db.allDocs(getAutomationParams())
       return automationList.rows.length
     },
@@ -48,13 +41,10 @@ export async function calculateAutomationCount(
   )
 }
 
-export async function calculateScreenCount(
+export async function calculateScreenCount(appId: string, db?: PouchLike) {
-  appId: string,
-  db?: PouchDB.Database
-) {
   return runInContext(
     appId,
-    async (db: PouchDB.Database) => {
+    async (db: PouchLike) => {
       const screenList = await db.allDocs(getScreenParams())
       return screenList.rows.length
     },
@@ -63,7 +53,7 @@ export async function calculateScreenCount(
 }
 
 export async function calculateBackupStats(appId: string) {
-  return runInContext(appId, async (db: PouchDB.Database) => {
+  return runInContext(appId, async (db: PouchLike) => {
     const promises = []
     promises.push(calculateDatasourceCount(appId, db))
     promises.push(calculateAutomationCount(appId, db))

packages/server/src/sdk/app/tables/index.ts

@@ -1,4 +1,4 @@
-import { getAppDB } from "@budibase/backend-core/context"
+import { context, PouchLike } from "@budibase/backend-core"
 import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
@@ -6,11 +6,10 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { Table } from "@budibase/types"
-import PouchDB from "pouchdb"
 
-async function getAllInternalTables(db?: PouchDB.Database): Promise<Table[]> {
+async function getAllInternalTables(db?: PouchLike): Promise<Table[]> {
   if (!db) {
-    db = getAppDB() as PouchDB.Database
+    db = context.getAppDB()
   }
   const internalTables = await db.allDocs(
     getTableParams(null, {
@@ -25,7 +24,7 @@ async function getAllInternalTables(db?: PouchDB.Database): Promise<Table[]> {
 }
 
 async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
-  const db = getAppDB()
+  const db = context.getAppDB()
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
@@ -42,7 +41,7 @@ async function getExternalTable(
 }
 
 async function getTable(tableId: any): Promise<Table> {
-  const db = getAppDB()
+  const db = context.getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
     const datasource = await db.get(datasourceId)