From c744d23832413f3df04eec7c03ccb97a2c41ac2a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 9 Nov 2022 16:53:42 +0000 Subject: [PATCH] Major update - removing the use of context for PouchDB instances, swapping knowledge of PouchDB to the PouchLike structure that replaces it. --- .../src/cache/tests/writethrough.spec.js | 6 +- .../backend-core/src/cache/writethrough.ts | 17 +- .../backend-core/src/context/constants.ts | 12 -- packages/backend-core/src/context/index.ts | 160 +++++------------- .../src/context/tests/index.spec.ts | 16 +- packages/backend-core/src/context/utils.ts | 109 ------------ packages/backend-core/src/couch/index.ts | 1 + packages/backend-core/src/couch/pouchDB.ts | 37 ++++ packages/backend-core/src/couch/pouchLike.ts | 55 ++++-- packages/backend-core/src/db/Replication.ts | 10 +- packages/backend-core/src/db/index.ts | 65 +------ .../backend-core/src/db/tests/index.spec.js | 6 +- packages/backend-core/src/db/views.ts | 49 +++--- packages/backend-core/src/environment.ts | 1 - packages/backend-core/src/index.ts | 2 + .../src/migrations/tests/index.spec.js | 4 +- packages/backend-core/src/types.ts | 1 + packages/backend-core/src/users.ts | 5 +- .../server/src/api/controllers/application.ts | 10 +- .../src/api/controllers/row/internal.js | 4 +- packages/server/src/db/inMemoryView.js | 4 +- .../functions/backfill/app/tables.ts | 5 +- .../server/src/sdk/app/backups/imports.ts | 12 +- .../server/src/sdk/app/backups/statistics.ts | 28 +-- packages/server/src/sdk/app/tables/index.ts | 11 +- 25 files changed, 211 insertions(+), 419 deletions(-) delete mode 100644 packages/backend-core/src/context/utils.ts create mode 100644 packages/backend-core/src/couch/pouchDB.ts create mode 100644 packages/backend-core/src/types.ts diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.js b/packages/backend-core/src/cache/tests/writethrough.spec.js index 68db24b325..ad29cb2aea 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.js +++ b/packages/backend-core/src/cache/tests/writethrough.spec.js @@ -1,6 +1,6 @@ require("../../../tests/utilities/TestConfiguration") const { Writethrough } = require("../writethrough") -const { dangerousGetDB } = require("../../db") +const { getDB } = require("../../db") const tk = require("timekeeper") const START_DATE = Date.now() @@ -8,8 +8,8 @@ tk.freeze(START_DATE) const DELAY = 5000 -const db = dangerousGetDB("test") -const db2 = dangerousGetDB("test2") +const db = getDB("test") +const db2 = getDB("test2") const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY) describe("writethrough", () => { diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 495ba58590..11dad81239 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,7 +1,7 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" import { logWarn } from "../logging" -import PouchDB from "pouchdb" +import { PouchLike } from "../couch" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null @@ -19,7 +19,7 @@ async function getCache() { return CACHE } -function makeCacheKey(db: PouchDB.Database, key: string) { +function makeCacheKey(db: PouchLike, key: string) { return db.name + key } @@ -28,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { } export async function put( - db: PouchDB.Database, + db: PouchLike, doc: any, writeRateMs: number = DEFAULT_WRITE_RATE_MS ) { @@ -64,7 +64,7 @@ export async function put( return { ok: true, id: output._id, rev: output._rev } } -export async function get(db: PouchDB.Database, id: string): Promise { +export async function get(db: PouchLike, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) let cacheItem: CacheItem = await cache.get(cacheKey) @@ -77,7 +77,7 @@ export async function get(db: PouchDB.Database, id: string): Promise { } export async function remove( - db: PouchDB.Database, + db: PouchLike, docOrId: any, rev?: any ): Promise { @@ -95,13 +95,10 @@ export async function remove( } export class Writethrough { - db: PouchDB.Database + db: PouchLike writeRateMs: number - constructor( - db: PouchDB.Database, - writeRateMs: number = DEFAULT_WRITE_RATE_MS - ) { + constructor(db: PouchLike, writeRateMs: number = DEFAULT_WRITE_RATE_MS) { this.db = db this.writeRateMs = writeRateMs } diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts index 937ad8f248..af30b1d241 100644 --- a/packages/backend-core/src/context/constants.ts +++ b/packages/backend-core/src/context/constants.ts @@ -1,17 +1,5 @@ export enum ContextKey { TENANT_ID = "tenantId", - GLOBAL_DB = "globalDb", APP_ID = "appId", IDENTITY = "identity", - // whatever the request app DB was - CURRENT_DB = "currentDb", - // get the prod app DB from the request - PROD_DB = "prodDb", - // get the dev app DB from the request - DEV_DB = "devDb", - DB_OPTS = "dbOpts", - // check if something else is using the context, don't close DB - TENANCY_IN_USE = "tenancyInUse", - APP_IN_USE = "appInUse", - IDENTITY_IN_USE = "identityInUse", } diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index c3955c71d9..30a53f1587 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -1,20 +1,12 @@ import env from "../environment" import { SEPARATOR, DocumentType } from "../db/constants" import cls from "./FunctionContext" -import { dangerousGetDB, closeDB } from "../db" import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" import { ContextKey } from "./constants" -import PouchDB from "pouchdb" -import { - updateUsing, - closeWithUsing, - setAppTenantId, - setIdentity, - closeAppDBs, - getContextDB, -} from "./utils" +import { PouchLike } from "../couch" +import { getDevelopmentAppID, getProdAppID } from "../db/conversions" export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID @@ -22,29 +14,19 @@ export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID // store an app ID to pretend there is a context let TEST_APP_ID: string | null = null -export const closeTenancy = async () => { - try { - if (env.USE_COUCH) { - const db = getGlobalDB() - await closeDB(db) - } - } catch (err) { - // no DB found - skip closing - return - } - // clear from context now that database is closed/task is finished - cls.setOnContext(ContextKey.TENANT_ID, null) - cls.setOnContext(ContextKey.GLOBAL_DB, null) -} - -// export const isDefaultTenant = () => { -// return getTenantId() === DEFAULT_TENANT_ID -// } - export const isMultiTenant = () => { return env.MULTI_TENANCY } +const setAppTenantId = (appId: string) => { + const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID + updateTenantId(appTenantId) +} + +const setIdentity = (identity: IdentityContext | null) => { + cls.setOnContext(ContextKey.IDENTITY, identity) +} + /** * Given an app ID this will attempt to retrieve the tenant ID from it. * @return {null|string} The tenant ID found within the app ID. @@ -78,47 +60,28 @@ export const doInContext = async (appId: string, task: any) => { }) } -export const doInTenant = (tenantId: string | null, task: any) => { +export const doInTenant = (tenantId: string | null, task: any): any => { // make sure default always selected in single tenancy if (!env.MULTI_TENANCY) { tenantId = tenantId || DEFAULT_TENANT_ID } - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { - // set the tenant id + global db if this is a new context - if (!opts.existing) { - updateTenantId(tenantId) - } - try { - // invoke the task - return await task() - } finally { - await closeWithUsing(ContextKey.TENANCY_IN_USE, () => { - return closeTenancy() - }) - } - } - - const existing = cls.getFromContext(ContextKey.TENANT_ID) === tenantId - return updateUsing(ContextKey.TENANCY_IN_USE, existing, internal) + return cls.run(async () => { + updateTenantId(tenantId) + return await task() + }) } -export const doInAppContext = (appId: string, task: any) => { +export const doInAppContext = (appId: string, task: any): any => { if (!appId) { throw new Error("appId is required") } const identity = getIdentity() - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { + return cls.run(async () => { // set the app tenant id - if (!opts.existing) { - setAppTenantId(appId) - } + setAppTenantId(appId) // set the app ID cls.setOnContext(ContextKey.APP_ID, appId) @@ -126,47 +89,28 @@ export const doInAppContext = (appId: string, task: any) => { if (identity) { setIdentity(identity) } - try { - // invoke the task - return await task() - } finally { - await closeWithUsing(ContextKey.APP_IN_USE, async () => { - await closeAppDBs() - await closeTenancy() - }) - } - } - const existing = cls.getFromContext(ContextKey.APP_ID) === appId - return updateUsing(ContextKey.APP_IN_USE, existing, internal) + // invoke the task + return await task() + }) } -export const doInIdentityContext = (identity: IdentityContext, task: any) => { +export const doInIdentityContext = ( + identity: IdentityContext, + task: any +): any => { if (!identity) { throw new Error("identity is required") } - async function internal(opts = { existing: false }) { - if (!opts.existing) { - cls.setOnContext(ContextKey.IDENTITY, identity) - // set the tenant so that doInTenant will preserve identity - if (identity.tenantId) { - updateTenantId(identity.tenantId) - } + return cls.run(async () => { + cls.setOnContext(ContextKey.IDENTITY, identity) + // set the tenant so that doInTenant will preserve identity + if (identity.tenantId) { + updateTenantId(identity.tenantId) } - - try { - // invoke the task - return await task() - } finally { - await closeWithUsing(ContextKey.IDENTITY_IN_USE, async () => { - setIdentity(null) - await closeTenancy() - }) - } - } - - const existing = cls.getFromContext(ContextKey.IDENTITY) - return updateUsing(ContextKey.IDENTITY_IN_USE, existing, internal) + // invoke the task + return await task() + }) } export const getIdentity = (): IdentityContext | undefined => { @@ -179,15 +123,10 @@ export const getIdentity = (): IdentityContext | undefined => { export const updateTenantId = (tenantId: string | null) => { cls.setOnContext(ContextKey.TENANT_ID, tenantId) - if (env.USE_COUCH) { - setGlobalDB(tenantId) - } } export const updateAppId = async (appId: string) => { try { - // have to close first, before removing the databases from context - await closeAppDBs() cls.setOnContext(ContextKey.APP_ID, appId) } catch (err) { if (env.isTest()) { @@ -198,19 +137,9 @@ export const updateAppId = async (appId: string) => { } } -export const setGlobalDB = (tenantId: string | null) => { - const dbName = baseGlobalDBName(tenantId) - const db = dangerousGetDB(dbName) - cls.setOnContext(ContextKey.GLOBAL_DB, db) - return db -} - -export const getGlobalDB = () => { - const db = cls.getFromContext(ContextKey.GLOBAL_DB) - if (!db) { - throw new Error("Global DB not found") - } - return db +export const getGlobalDB = (): PouchLike => { + const tenantId = cls.getFromContext(ContextKey.TENANT_ID) + return new PouchLike(baseGlobalDBName(tenantId)) } export const isTenantIdSet = () => { @@ -246,22 +175,25 @@ export const isTenancyEnabled = () => { * Opens the app database based on whatever the request * contained, dev or prod. */ -export const getAppDB = (opts?: any) => { - return getContextDB(ContextKey.CURRENT_DB, opts) +export const getAppDB = (opts?: any): PouchLike => { + const appId = getAppId() + return new PouchLike(appId, opts) } /** * This specifically gets the prod app ID, if the request * contained a development app ID, this will open the prod one. */ -export const getProdAppDB = (opts?: any) => { - return getContextDB(ContextKey.PROD_DB, opts) +export const getProdAppDB = (opts?: any): PouchLike => { + const appId = getAppId() + return new PouchLike(getProdAppID(appId), opts) } /** * This specifically gets the dev app ID, if the request * contained a prod app ID, this will open the dev one. */ -export const getDevAppDB = (opts?: any) => { - return getContextDB(ContextKey.DEV_DB, opts) +export const getDevAppDB = (opts?: any): PouchLike => { + const appId = getAppId() + return new PouchLike(getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.ts index 55ecd333a3..749231c3e5 100644 --- a/packages/backend-core/src/context/tests/index.spec.ts +++ b/packages/backend-core/src/context/tests/index.spec.ts @@ -5,8 +5,8 @@ import env from "../../environment" // must use require to spy index file exports due to known issue in jest const dbUtils = require("../../db") -jest.spyOn(dbUtils, "closeDB") -jest.spyOn(dbUtils, "dangerousGetDB") +jest.spyOn(dbUtils, "closePouchDB") +jest.spyOn(dbUtils, "getDB") describe("context", () => { beforeEach(() => { @@ -25,8 +25,8 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("global-db") }) - expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + expect(dbUtils.getDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) }) @@ -85,8 +85,8 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("test_global-db") }) - expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + expect(dbUtils.getDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets the tenant id when nested with same tenant id", async () => { @@ -123,8 +123,8 @@ describe("context", () => { }) // only 1 db is opened and closed - expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + expect(dbUtils.getDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets different tenant id inside another context", () => { diff --git a/packages/backend-core/src/context/utils.ts b/packages/backend-core/src/context/utils.ts deleted file mode 100644 index 6e7100b594..0000000000 --- a/packages/backend-core/src/context/utils.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { - DEFAULT_TENANT_ID, - getAppId, - getTenantIDFromAppID, - updateTenantId, -} from "./index" -import cls from "./FunctionContext" -import { IdentityContext } from "@budibase/types" -import { ContextKey } from "./constants" -import { dangerousGetDB, closeDB } from "../db" -import { isEqual } from "lodash" -import { getDevelopmentAppID, getProdAppID } from "../db/conversions" -import env from "../environment" - -export async function updateUsing( - usingKey: string, - existing: boolean, - internal: (opts: { existing: boolean }) => Promise -) { - const using = cls.getFromContext(usingKey) - if (using && existing) { - cls.setOnContext(usingKey, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(usingKey, 1) - return internal({ existing: false }) - }) - } -} - -export async function closeWithUsing( - usingKey: string, - closeFn: () => Promise -) { - const using = cls.getFromContext(usingKey) - if (!using || using <= 1) { - await closeFn() - } else { - cls.setOnContext(usingKey, using - 1) - } -} - -export const setAppTenantId = (appId: string) => { - const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID - updateTenantId(appTenantId) -} - -export const setIdentity = (identity: IdentityContext | null) => { - cls.setOnContext(ContextKey.IDENTITY, identity) -} - -// this function makes sure the PouchDB objects are closed and -// fully deleted when finished - this protects against memory leaks -export async function closeAppDBs() { - const dbKeys = [ContextKey.CURRENT_DB, ContextKey.PROD_DB, ContextKey.DEV_DB] - for (let dbKey of dbKeys) { - const db = cls.getFromContext(dbKey) - if (!db) { - continue - } - await closeDB(db) - // clear the DB from context, incase someone tries to use it again - cls.setOnContext(dbKey, null) - } - // clear the app ID now that the databases are closed - if (cls.getFromContext(ContextKey.APP_ID)) { - cls.setOnContext(ContextKey.APP_ID, null) - } - if (cls.getFromContext(ContextKey.DB_OPTS)) { - cls.setOnContext(ContextKey.DB_OPTS, null) - } -} - -export function getContextDB(key: string, opts: any) { - const dbOptsKey = `${key}${ContextKey.DB_OPTS}` - let storedOpts = cls.getFromContext(dbOptsKey) - let db = cls.getFromContext(key) - if (db && isEqual(opts, storedOpts)) { - return db - } - - const appId = getAppId() - let toUseAppId - - switch (key) { - case ContextKey.CURRENT_DB: - toUseAppId = appId - break - case ContextKey.PROD_DB: - toUseAppId = getProdAppID(appId) - break - case ContextKey.DEV_DB: - toUseAppId = getDevelopmentAppID(appId) - break - } - db = dangerousGetDB(toUseAppId, opts) - try { - cls.setOnContext(key, db) - if (opts) { - cls.setOnContext(dbOptsKey, opts) - } - } catch (err) { - if (!env.isTest()) { - throw err - } - } - return db -} diff --git a/packages/backend-core/src/couch/index.ts b/packages/backend-core/src/couch/index.ts index daba8f74a9..7a4ac20486 100644 --- a/packages/backend-core/src/couch/index.ts +++ b/packages/backend-core/src/couch/index.ts @@ -1,3 +1,4 @@ export * from "./couch" export * from "./pouchLike" export * from "./utils" +export { init } from "./pouchDB" diff --git a/packages/backend-core/src/couch/pouchDB.ts b/packages/backend-core/src/couch/pouchDB.ts new file mode 100644 index 0000000000..74ad8a3dc6 --- /dev/null +++ b/packages/backend-core/src/couch/pouchDB.ts @@ -0,0 +1,37 @@ +import PouchDB from "pouchdb" +import env from "../environment" +import { PouchOptions } from "@budibase/types" +import * as pouch from "../db/pouch" + +let Pouch: any +let initialised = false + +export async function init(opts?: PouchOptions) { + Pouch = pouch.getPouch(opts) + initialised = true +} + +const checkInitialised = () => { + if (!initialised) { + throw new Error("init has not been called") + } +} + +export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { + checkInitialised() + return new Pouch(dbName, opts) +} + +// use this function if you have called getPouchDB - close +// the databases you've opened once finished +export async function closePouchDB(db: PouchDB.Database) { + if (!db || env.isTest()) { + return + } + try { + // specifically await so that if there is an error, it can be ignored + return await db.close() + } catch (err) { + // ignore error, already closed + } +} diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts index 2591380ba0..d605ec3d04 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -2,12 +2,16 @@ import Nano from "nano" import { AnyDocument } from "@budibase/types" import { getCouchInfo } from "./couch" import { directCouchCall } from "./utils" -import { getPouchDB } from "../db" +import { getPouchDB } from "./pouchDB" export type PouchLikeOpts = { skip_setup?: boolean } +export type PutOpts = { + force?: boolean +} + export type QueryOpts = { include_docs?: boolean startkey?: string @@ -19,6 +23,10 @@ export type QueryOpts = { keys?: string[] } +type QueryResp = Promise<{ + rows: { doc?: T | any; value?: any }[] +}> + export class PouchLike { public readonly name: string private static nano: Nano.ServerScope @@ -45,11 +53,15 @@ export class PouchLike { }) } + async exists() { + let response = await directCouchCall(`/${this.name}`, "HEAD") + return response.status === 200 + } + async checkSetup() { let shouldCreate = !this.pouchOpts?.skip_setup // check exists in a lightweight fashion - let response = await directCouchCall(`/${this.name}`, "HEAD") - let exists = response.status === 200 + let exists = await this.exists() if (!shouldCreate && !exists) { throw new Error("DB does not exist") } @@ -70,26 +82,43 @@ export class PouchLike { } } - async info() { - const db = PouchLike.nano.db.use(this.name) - return db.info() - } - - async get(id: string) { + async get(id?: string): Promise { const db = await this.checkSetup() + if (!id) { + throw new Error("Unable to get doc without a valid _id.") + } return this.updateOutput(() => db.get(id)) } - async remove(id: string, rev: string) { + async remove(id?: string, rev?: string) { const db = await this.checkSetup() + if (!id || !rev) { + throw new Error("Unable to remove doc without a valid _id and _rev.") + } return this.updateOutput(() => db.destroy(id, rev)) } - async put(document: AnyDocument) { + async put(document: AnyDocument, opts?: PutOpts) { if (!document._id) { throw new Error("Cannot store document without _id field.") } const db = await this.checkSetup() + if (!document.createdAt) { + document.createdAt = new Date().toISOString() + } + document.updatedAt = new Date().toISOString() + if (opts?.force && document._id) { + try { + const existing = await this.get(document._id) + if (existing) { + document._rev = existing._rev + } + } catch (err: any) { + if (err.status !== 404) { + throw err + } + } + } return this.updateOutput(() => db.insert(document)) } @@ -98,12 +127,12 @@ export class PouchLike { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: QueryOpts) { + async allDocs(params: QueryOpts): QueryResp { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } - async query(viewName: string, params: QueryOpts) { + async query(viewName: string, params: QueryOpts): QueryResp { const db = await this.checkSetup() const [database, view] = viewName.split("/") return this.updateOutput(() => db.view(database, view, params)) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index e0bd3c7a43..c6eea8db5e 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,4 +1,4 @@ -import { dangerousGetDB, closeDB } from "." +import { getPouchDB, closePouchDB } from "../couch/pouchDB" import { DocumentType } from "./constants" class Replication { @@ -12,12 +12,12 @@ class Replication { * @param {String} target - the DB you want to replicate to, or rollback from */ constructor({ source, target }: any) { - this.source = dangerousGetDB(source) - this.target = dangerousGetDB(target) + this.source = getPouchDB(source) + this.target = getPouchDB(target) } close() { - return Promise.all([closeDB(this.source), closeDB(this.target)]) + return Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) } promisify(operation: any, opts = {}) { @@ -68,7 +68,7 @@ class Replication { async rollback() { await this.target.destroy() // Recreate the DB again - this.target = dangerousGetDB(this.target.name) + this.target = getPouchDB(this.target.name) // take the opportunity to remove deleted tombstones await this.replicate() } diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 73d1327af3..e732b3a2fb 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,87 +1,30 @@ -import * as pouch from "./pouch" import env from "../environment" -import { PouchOptions, CouchFindOptions } from "@budibase/types" -import PouchDB from "pouchdb" +import { CouchFindOptions } from "@budibase/types" import { PouchLike } from "../couch" import { directCouchQuery } from "../couch" -export { directCouchQuery } from "../couch" +export { init, PouchLike } from "../couch" -const openDbs: string[] = [] -let Pouch: any let initialised = false const dbList = new Set() -if (env.MEMORY_LEAK_CHECK) { - setInterval(() => { - console.log("--- OPEN DBS ---") - console.log(openDbs) - }, 5000) -} - -const put = - (dbPut: any) => - async (doc: any, options = {}) => { - if (!doc.createdAt) { - doc.createdAt = new Date().toISOString() - } - doc.updatedAt = new Date().toISOString() - return dbPut(doc, options) - } - const checkInitialised = () => { if (!initialised) { throw new Error("init has not been called") } } -export async function init(opts?: PouchOptions) { - Pouch = pouch.getPouch(opts) - initialised = true -} - -export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { - checkInitialised() +export function getDB(dbName: string, opts?: any): PouchLike { if (env.isTest()) { dbList.add(dbName) } - const db = new Pouch(dbName, opts) - if (env.MEMORY_LEAK_CHECK) { - openDbs.push(db.name) - } - const dbPut = db.put - db.put = put(dbPut) - return db -} - -// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION -// this function is prone to leaks, should only be used -// in situations that using the function doWithDB does not work -export function dangerousGetDB(dbName: string, opts?: any): PouchLike { return new PouchLike(dbName, opts) } -// use this function if you have called dangerousGetDB - close -// the databases you've opened once finished -export async function closeDB(db: PouchDB.Database) { - if (!db || env.isTest()) { - return - } - if (env.MEMORY_LEAK_CHECK) { - openDbs.splice(openDbs.indexOf(db.name), 1) - } - try { - // specifically await so that if there is an error, it can be ignored - return await db.close() - } catch (err) { - // ignore error, already closed - } -} - // we have to use a callback for this so that we can close // the DB when we're done, without this manual requests would // need to close the database when done with it to avoid memory leaks export async function doWithDB(dbName: string, cb: any, opts = {}) { - const db = dangerousGetDB(dbName, opts) + const db = getDB(dbName, opts) // need this to be async so that we can correctly close DB after all // async operations have been completed return await cb(db) diff --git a/packages/backend-core/src/db/tests/index.spec.js b/packages/backend-core/src/db/tests/index.spec.js index bc0c638126..8f81e6b8dc 100644 --- a/packages/backend-core/src/db/tests/index.spec.js +++ b/packages/backend-core/src/db/tests/index.spec.js @@ -1,11 +1,11 @@ require("../../../tests/utilities/TestConfiguration") -const { dangerousGetDB } = require("../") +const { getDB } = require("../") describe("db", () => { describe("getDB", () => { it("returns a db", async () => { - const db = dangerousGetDB("test") + const db = getDB("test") expect(db).toBeDefined() expect(db._adapter).toBe("memory") expect(db.prefix).toBe("_pouch_") @@ -13,7 +13,7 @@ describe("db", () => { }) it("uses the custom put function", async () => { - const db = dangerousGetDB("test") + const db = getDB("test") let doc = { _id: "test" } await db.put(doc) doc = await db.get(doc._id) diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index f0fff918fc..06919fd188 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -1,6 +1,6 @@ import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils" import { getGlobalDB } from "../context" -import PouchDB from "pouchdb" +import { PouchLike, QueryOpts } from "../couch" import { StaticDatabases } from "./constants" import { doWithDB } from "./" @@ -19,7 +19,7 @@ interface DesignDocument { views: any } -async function removeDeprecated(db: PouchDB.Database, viewName: ViewName) { +async function removeDeprecated(db: PouchLike, viewName: ViewName) { // @ts-ignore if (!DeprecatedViews[viewName]) { return @@ -70,16 +70,13 @@ export const createAccountEmailView = async () => { emit(doc.email.toLowerCase(), doc._id) } }` - await doWithDB( - StaticDatabases.PLATFORM_INFO.name, - async (db: PouchDB.Database) => { - await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL) - } - ) + await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL) + }) } export const createUserAppView = async () => { - const db = getGlobalDB() as PouchDB.Database + const db = getGlobalDB() const viewJs = `function(doc) { if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) { for (let prodAppId of Object.keys(doc.roles)) { @@ -117,12 +114,9 @@ export const createPlatformUserView = async () => { emit(doc._id.toLowerCase(), doc._id) } }` - await doWithDB( - StaticDatabases.PLATFORM_INFO.name, - async (db: PouchDB.Database) => { - await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE) - } - ) + await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE) + }) } export interface QueryViewOptions { @@ -131,13 +125,13 @@ export interface QueryViewOptions { export const queryView = async ( viewName: ViewName, - params: PouchDB.Query.Options, - db: PouchDB.Database, + params: QueryOpts, + db: PouchLike, createFunc: any, opts?: QueryViewOptions ): Promise => { try { - let response = await db.query(`database/${viewName}`, params) + let response = await db.query(`database/${viewName}`, params) const rows = response.rows const docs = rows.map(row => (params.include_docs ? row.doc : row.value)) @@ -161,7 +155,7 @@ export const queryView = async ( export const queryPlatformView = async ( viewName: ViewName, - params: PouchDB.Query.Options, + params: QueryOpts, opts?: QueryViewOptions ): Promise => { const CreateFuncByName: any = { @@ -169,19 +163,16 @@ export const queryPlatformView = async ( [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView, } - return doWithDB( - StaticDatabases.PLATFORM_INFO.name, - async (db: PouchDB.Database) => { - const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db, createFn, opts) - } - ) + return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + const createFn = CreateFuncByName[viewName] + return queryView(viewName, params, db, createFn, opts) + }) } export const queryGlobalView = async ( viewName: ViewName, - params: PouchDB.Query.Options, - db?: PouchDB.Database, + params: QueryOpts, + db?: PouchLike, opts?: QueryViewOptions ): Promise => { const CreateFuncByName: any = { @@ -192,7 +183,7 @@ export const queryGlobalView = async ( } // can pass DB in if working with something specific if (!db) { - db = getGlobalDB() as PouchDB.Database + db = getGlobalDB() } const createFn = CreateFuncByName[viewName] return queryView(viewName, params, db, createFn, opts) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 6e2ac94be9..2443287d5a 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -69,7 +69,6 @@ const env = { DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, SERVICE: process.env.SERVICE || "budibase", - MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false, LOG_LEVEL: process.env.LOG_LEVEL, SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD, DEPLOYMENT_ENVIRONMENT: diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 17393b8ac3..06997ced90 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -21,6 +21,7 @@ import * as middleware from "./middleware" import plugins from "./plugin" import encryption from "./security/encryption" import * as queue from "./queue" +import * as types from "./types" // mimic the outer package exports import * as db from "./pkg/db" @@ -67,6 +68,7 @@ const core = { encryption, queue, permissions, + ...types, } export = core diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js index c5ec143143..653ef453c5 100644 --- a/packages/backend-core/src/migrations/tests/index.spec.js +++ b/packages/backend-core/src/migrations/tests/index.spec.js @@ -1,6 +1,6 @@ require("../../../tests/utilities/TestConfiguration") const { runMigrations, getMigrationsDoc } = require("../index") -const { dangerousGetDB } = require("../../db") +const { getDB } = require("../../db") const { StaticDatabases, } = require("../../db/utils") @@ -18,7 +18,7 @@ describe("migrations", () => { }] beforeEach(() => { - db = dangerousGetDB(StaticDatabases.GLOBAL.name) + db = getDB(StaticDatabases.GLOBAL.name) }) afterEach(async () => { diff --git a/packages/backend-core/src/types.ts b/packages/backend-core/src/types.ts new file mode 100644 index 0000000000..5eb3109b27 --- /dev/null +++ b/packages/backend-core/src/types.ts @@ -0,0 +1 @@ +export { PouchLike } from "./couch" diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 44f04749c9..a38debfc19 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -8,10 +8,9 @@ import { queryGlobalView } from "./db/views" import { UNICODE_MAX } from "./db/constants" import { BulkDocsResponse, User } from "@budibase/types" import { getGlobalDB } from "./context" -import PouchDB from "pouchdb" export const bulkGetGlobalUsersById = async (userIds: string[]) => { - const db = getGlobalDB() as PouchDB.Database + const db = getGlobalDB() return ( await db.allDocs({ keys: userIds, @@ -21,7 +20,7 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => { } export const bulkUpdateGlobalUsers = async (users: User[]) => { - const db = getGlobalDB() as PouchDB.Database + const db = getGlobalDB() return (await db.bulkDocs(users)) as BulkDocsResponse } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index f3dca51f72..b97e980a5e 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -524,12 +524,10 @@ export const sync = async (ctx: any, next: any) => { // replicate prod to dev const prodAppId = getProdAppID(appId) - try { - // specific case, want to make sure setup is skipped - const prodDb = context.getProdAppDB({ skip_setup: true }) - const info = await prodDb.info() - if (info.error) throw info.error - } catch (err) { + // specific case, want to make sure setup is skipped + const prodDb = context.getProdAppDB({ skip_setup: true }) + const exists = await prodDb.exists() + if (!exists) { // the database doesn't exist. Don't replicate ctx.status = 200 ctx.body = { diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 52f8a548fe..0f1324f10e 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -6,7 +6,7 @@ const { DocumentType, InternalTables, } = require("../../../db/utils") -const { dangerousGetDB } = require("@budibase/backend-core/db") +const { getDB } = require("@budibase/backend-core/db") const userController = require("../user") const { inputProcessing, @@ -251,7 +251,7 @@ exports.fetch = async ctx => { } exports.find = async ctx => { - const db = dangerousGetDB(ctx.appId) + const db = getDB(ctx.appId) const table = await db.get(ctx.params.tableId) let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId) row = await outputProcessing(table, row) diff --git a/packages/server/src/db/inMemoryView.js b/packages/server/src/db/inMemoryView.js index ec99b4738c..57ea89071c 100644 --- a/packages/server/src/db/inMemoryView.js +++ b/packages/server/src/db/inMemoryView.js @@ -2,7 +2,7 @@ const newid = require("./newid") // bypass the main application db config // use in memory pouchdb directly -const { getPouch, closeDB } = require("@budibase/backend-core/db") +const { getPouch, closePouchDB } = require("@budibase/backend-core/db") const Pouch = getPouch({ inMemory: true }) exports.runView = async (view, calculation, group, data) => { @@ -44,6 +44,6 @@ exports.runView = async (view, calculation, group, data) => { return response } finally { await db.destroy() - await closeDB(db) + await closePouchDB(db) } } diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index 6663c3c43b..c9d1e5c794 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -1,9 +1,8 @@ -import { events } from "@budibase/backend-core" +import { events, PouchLike } from "@budibase/backend-core" import sdk from "../../../../sdk" -import PouchDB from "pouchdb" export const backfill = async ( - appDb: PouchDB.Database, + appDb: PouchLike, timestamp: string | number ) => { const tables = await sdk.tables.getAllInternalTables(appDb) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 7cc9e0b0e6..634f507220 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,4 +1,4 @@ -import { db as dbCore } from "@budibase/backend-core" +import { db as dbCore, PouchLike } from "@budibase/backend-core" import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" @@ -17,7 +17,6 @@ import { CouchFindOptions, RowAttachment, } from "@budibase/types" -import PouchDB from "pouchdb" const uuid = require("uuid/v4") const tar = require("tar") @@ -29,10 +28,7 @@ type TemplateType = { key?: string } -async function updateAttachmentColumns( - prodAppId: string, - db: PouchDB.Database -) { +async function updateAttachmentColumns(prodAppId: string, db: PouchLike) { // iterate through attachment documents and update them const tables = await sdk.tables.getAllInternalTables(db) for (let table of tables) { @@ -86,7 +82,7 @@ async function updateAttachmentColumns( } } -async function updateAutomations(prodAppId: string, db: PouchDB.Database) { +async function updateAutomations(prodAppId: string, db: PouchLike) { const automations = ( await db.allDocs( getAutomationParams(null, { @@ -154,7 +150,7 @@ export function getListOfAppsInMulti(tmpPath: string) { export async function importApp( appId: string, - db: PouchDB.Database, + db: PouchLike, template: TemplateType ) { let prodAppId = dbCore.getProdAppID(appId) diff --git a/packages/server/src/sdk/app/backups/statistics.ts b/packages/server/src/sdk/app/backups/statistics.ts index 7a8e24dc58..9fe1a04d21 100644 --- a/packages/server/src/sdk/app/backups/statistics.ts +++ b/packages/server/src/sdk/app/backups/statistics.ts @@ -1,13 +1,12 @@ -import { context, db as dbCore } from "@budibase/backend-core" +import { context, db as dbCore, PouchLike } from "@budibase/backend-core" import { getDatasourceParams, getTableParams, getAutomationParams, getScreenParams, } from "../../../db/utils" -import PouchDB from "pouchdb" -async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { +async function runInContext(appId: string, cb: any, db?: PouchLike) { if (db) { return cb(db) } else { @@ -19,13 +18,10 @@ async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { } } -export async function calculateDatasourceCount( - appId: string, - db?: PouchDB.Database -) { +export async function calculateDatasourceCount(appId: string, db?: PouchLike) { return runInContext( appId, - async (db: PouchDB.Database) => { + async (db: PouchLike) => { const datasourceList = await db.allDocs(getDatasourceParams()) const tableList = await db.allDocs(getTableParams()) return datasourceList.rows.length + tableList.rows.length @@ -34,13 +30,10 @@ export async function calculateDatasourceCount( ) } -export async function calculateAutomationCount( - appId: string, - db?: PouchDB.Database -) { +export async function calculateAutomationCount(appId: string, db?: PouchLike) { return runInContext( appId, - async (db: PouchDB.Database) => { + async (db: PouchLike) => { const automationList = await db.allDocs(getAutomationParams()) return automationList.rows.length }, @@ -48,13 +41,10 @@ export async function calculateAutomationCount( ) } -export async function calculateScreenCount( - appId: string, - db?: PouchDB.Database -) { +export async function calculateScreenCount(appId: string, db?: PouchLike) { return runInContext( appId, - async (db: PouchDB.Database) => { + async (db: PouchLike) => { const screenList = await db.allDocs(getScreenParams()) return screenList.rows.length }, @@ -63,7 +53,7 @@ export async function calculateScreenCount( } export async function calculateBackupStats(appId: string) { - return runInContext(appId, async (db: PouchDB.Database) => { + return runInContext(appId, async (db: PouchLike) => { const promises = [] promises.push(calculateDatasourceCount(appId, db)) promises.push(calculateAutomationCount(appId, db)) diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 5af92404a1..ef41630d6b 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,4 +1,4 @@ -import { getAppDB } from "@budibase/backend-core/context" +import { context, PouchLike } from "@budibase/backend-core" import { BudibaseInternalDB, getTableParams } from "../../../db/utils" import { breakExternalTableId, @@ -6,11 +6,10 @@ import { isSQL, } from "../../../integrations/utils" import { Table } from "@budibase/types" -import PouchDB from "pouchdb" -async function getAllInternalTables(db?: PouchDB.Database): Promise { +async function getAllInternalTables(db?: PouchLike): Promise { if (!db) { - db = getAppDB() as PouchDB.Database + db = context.getAppDB() } const internalTables = await db.allDocs( getTableParams(null, { @@ -25,7 +24,7 @@ async function getAllInternalTables(db?: PouchDB.Database): Promise { } async function getAllExternalTables(datasourceId: any): Promise { - const db = getAppDB() + const db = context.getAppDB() const datasource = await db.get(datasourceId) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." @@ -42,7 +41,7 @@ async function getExternalTable( } async function getTable(tableId: any): Promise { - const db = getAppDB() + const db = context.getAppDB() if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) const datasource = await db.get(datasourceId)