From b4b471e054552ef13d1ba1f4e314f566da82f088 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 11 Nov 2022 12:46:32 +0000 Subject: [PATCH] Re-jigging things a bit, hiding as much of the couch/pouch stuff away. --- packages/backend-core/db.js | 1 - packages/backend-core/package.json | 3 +- .../backend-core/src/cache/writethrough.ts | 2 +- .../backend-core/src/context/deprovision.ts | 3 +- packages/backend-core/src/context/index.ts | 2 +- packages/backend-core/src/couch/couch.ts | 35 -------- packages/backend-core/src/couch/index.ts | 4 - packages/backend-core/src/couch/pouchDB.ts | 37 --------- packages/backend-core/src/db/Replication.ts | 2 +- .../backend-core/src/db/couch/connections.ts | 77 ++++++++++++++++++ packages/backend-core/src/db/couch/index.ts | 4 + .../src/db/{pouch.ts => couch/pouchDB.ts} | 80 ++++++++----------- .../src/{ => db}/couch/pouchLike.ts | 2 +- .../backend-core/src/{ => db}/couch/utils.ts | 4 +- packages/backend-core/src/db/index.ts | 5 +- packages/backend-core/src/db/utils.ts | 2 +- packages/backend-core/src/db/views.ts | 8 +- packages/backend-core/src/pkg/db.ts | 1 - packages/backend-core/src/types.ts | 2 +- packages/server/package.json | 3 +- packages/worker/package.json | 3 +- 21 files changed, 138 insertions(+), 142 deletions(-) delete mode 100644 packages/backend-core/src/couch/couch.ts delete mode 100644 packages/backend-core/src/couch/index.ts delete mode 100644 packages/backend-core/src/couch/pouchDB.ts create mode 100644 packages/backend-core/src/db/couch/connections.ts create mode 100644 packages/backend-core/src/db/couch/index.ts rename packages/backend-core/src/db/{pouch.ts => couch/pouchDB.ts} (52%) rename packages/backend-core/src/{ => db}/couch/pouchLike.ts (99%) rename packages/backend-core/src/{ => db}/couch/utils.ts (89%) diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js index 0d2869d9f1..d2adf6c092 100644 --- a/packages/backend-core/db.js +++ b/packages/backend-core/db.js @@ -3,5 +3,4 @@ module.exports = { ...require("./src/db/constants"), ...require("./src/db"), ...require("./src/db/views"), - ...require("./src/db/pouch"), } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 42a5f2bc53..2fbc087214 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -57,7 +57,8 @@ "preset": "ts-jest", "testEnvironment": "node", "moduleNameMapper": { - "@budibase/types": "/../types/src" + "@budibase/types": "/../types/src", + "axios": "axios/dist/node/axios.cjs" }, "setupFiles": [ "./scripts/jestSetup.ts" diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 11dad81239..e13cc8cc0d 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,7 +1,7 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" import { logWarn } from "../logging" -import { PouchLike } from "../couch" +import { PouchLike } from "../db" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null diff --git a/packages/backend-core/src/context/deprovision.ts b/packages/backend-core/src/context/deprovision.ts index 179c7f407a..befcd48a15 100644 --- a/packages/backend-core/src/context/deprovision.ts +++ b/packages/backend-core/src/context/deprovision.ts @@ -1,8 +1,7 @@ import { getGlobalUserParams, getAllApps } from "../db/utils" -import { doWithDB } from "../db" +import { doWithDB, PouchLike } from "../db" import { doWithGlobalDB } from "../tenancy" import { StaticDatabases } from "../db/constants" -import { PouchLike } from "../couch" import { User } from "@budibase/types" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 63b57a1b6e..34e6603ef7 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -5,7 +5,7 @@ import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" import { ContextMap, ContextKey } from "./constants" -import { PouchLike } from "../couch" +import { PouchLike } from "../db" import { getDevelopmentAppID, getProdAppID } from "../db/conversions" export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID diff --git a/packages/backend-core/src/couch/couch.ts b/packages/backend-core/src/couch/couch.ts deleted file mode 100644 index 41fb83cb86..0000000000 --- a/packages/backend-core/src/couch/couch.ts +++ /dev/null @@ -1,35 +0,0 @@ -import env from "../environment" -import { getUrlInfo } from "../db/pouch" - -export const getCouchInfo = () => { - const urlInfo = getUrlInfo() - let username - let password - if (env.COUCH_DB_USERNAME) { - // set from env - username = env.COUCH_DB_USERNAME - } else if (urlInfo.auth.username) { - // set from url - username = urlInfo.auth.username - } else if (!env.isTest()) { - throw new Error("CouchDB username not set") - } - if (env.COUCH_DB_PASSWORD) { - // set from env - password = env.COUCH_DB_PASSWORD - } else if (urlInfo.auth.password) { - // set from url - password = urlInfo.auth.password - } else if (!env.isTest()) { - throw new Error("CouchDB password not set") - } - const authCookie = Buffer.from(`${username}:${password}`).toString("base64") - return { - url: urlInfo.url!, - auth: { - username: username, - password: password, - }, - cookie: `Basic ${authCookie}`, - } -} diff --git a/packages/backend-core/src/couch/index.ts b/packages/backend-core/src/couch/index.ts deleted file mode 100644 index 7a4ac20486..0000000000 --- a/packages/backend-core/src/couch/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from "./couch" -export * from "./pouchLike" -export * from "./utils" -export { init } from "./pouchDB" diff --git a/packages/backend-core/src/couch/pouchDB.ts b/packages/backend-core/src/couch/pouchDB.ts deleted file mode 100644 index 74ad8a3dc6..0000000000 --- a/packages/backend-core/src/couch/pouchDB.ts +++ /dev/null @@ -1,37 +0,0 @@ -import PouchDB from "pouchdb" -import env from "../environment" -import { PouchOptions } from "@budibase/types" -import * as pouch from "../db/pouch" - -let Pouch: any -let initialised = false - -export async function init(opts?: PouchOptions) { - Pouch = pouch.getPouch(opts) - initialised = true -} - -const checkInitialised = () => { - if (!initialised) { - throw new Error("init has not been called") - } -} - -export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { - checkInitialised() - return new Pouch(dbName, opts) -} - -// use this function if you have called getPouchDB - close -// the databases you've opened once finished -export async function closePouchDB(db: PouchDB.Database) { - if (!db || env.isTest()) { - return - } - try { - // specifically await so that if there is an error, it can be ignored - return await db.close() - } catch (err) { - // ignore error, already closed - } -} diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index c6eea8db5e..12f6001a70 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,4 +1,4 @@ -import { getPouchDB, closePouchDB } from "../couch/pouchDB" +import { getPouchDB, closePouchDB } from "./couch/pouchDB" import { DocumentType } from "./constants" class Replication { diff --git a/packages/backend-core/src/db/couch/connections.ts b/packages/backend-core/src/db/couch/connections.ts new file mode 100644 index 0000000000..a2206de634 --- /dev/null +++ b/packages/backend-core/src/db/couch/connections.ts @@ -0,0 +1,77 @@ +import env from "../../environment" + +export const getCouchInfo = () => { + const urlInfo = getUrlInfo() + let username + let password + if (env.COUCH_DB_USERNAME) { + // set from env + username = env.COUCH_DB_USERNAME + } else if (urlInfo.auth.username) { + // set from url + username = urlInfo.auth.username + } else if (!env.isTest()) { + throw new Error("CouchDB username not set") + } + if (env.COUCH_DB_PASSWORD) { + // set from env + password = env.COUCH_DB_PASSWORD + } else if (urlInfo.auth.password) { + // set from url + password = urlInfo.auth.password + } else if (!env.isTest()) { + throw new Error("CouchDB password not set") + } + const authCookie = Buffer.from(`${username}:${password}`).toString("base64") + return { + url: urlInfo.url!, + auth: { + username: username, + password: password, + }, + cookie: `Basic ${authCookie}`, + } +} + +export const getUrlInfo = (url = env.COUCH_DB_URL) => { + let cleanUrl, username, password, host + if (url) { + // Ensure the URL starts with a protocol + const protoRegex = /^https?:\/\//i + if (!protoRegex.test(url)) { + url = `http://${url}` + } + + // Split into protocol and remainder + const split = url.split("://") + const protocol = split[0] + const rest = split.slice(1).join("://") + + // Extract auth if specified + if (url.includes("@")) { + // Split into host and remainder + let parts = rest.split("@") + host = parts[parts.length - 1] + let auth = parts.slice(0, -1).join("@") + + // Split auth into username and password + if (auth.includes(":")) { + const authParts = auth.split(":") + username = authParts[0] + password = authParts.slice(1).join(":") + } else { + username = auth + } + } else { + host = rest + } + cleanUrl = `${protocol}://${host}` + } + return { + url: cleanUrl, + auth: { + username, + password, + }, + } +} diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts new file mode 100644 index 0000000000..3730a1e248 --- /dev/null +++ b/packages/backend-core/src/db/couch/index.ts @@ -0,0 +1,4 @@ +export * from "./connections" +export * from "./pouchLike" +export * from "./utils" +export { init, getPouch, getPouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/db/pouch.ts b/packages/backend-core/src/db/couch/pouchDB.ts similarity index 52% rename from packages/backend-core/src/db/pouch.ts rename to packages/backend-core/src/db/couch/pouchDB.ts index fdb051060a..eead0e4d6f 100644 --- a/packages/backend-core/src/db/pouch.ts +++ b/packages/backend-core/src/db/couch/pouchDB.ts @@ -1,50 +1,10 @@ import PouchDB from "pouchdb" -import env from "../environment" -import { getCouchInfo } from "../couch" -export { getCouchInfo } from "../couch" +import env from "../../environment" +import { PouchOptions } from "@budibase/types" +import { getCouchInfo } from "./connections" -export const getUrlInfo = (url = env.COUCH_DB_URL) => { - let cleanUrl, username, password, host - if (url) { - // Ensure the URL starts with a protocol - const protoRegex = /^https?:\/\//i - if (!protoRegex.test(url)) { - url = `http://${url}` - } - - // Split into protocol and remainder - const split = url.split("://") - const protocol = split[0] - const rest = split.slice(1).join("://") - - // Extract auth if specified - if (url.includes("@")) { - // Split into host and remainder - let parts = rest.split("@") - host = parts[parts.length - 1] - let auth = parts.slice(0, -1).join("@") - - // Split auth into username and password - if (auth.includes(":")) { - const authParts = auth.split(":") - username = authParts[0] - password = authParts.slice(1).join(":") - } else { - username = auth - } - } else { - host = rest - } - cleanUrl = `${protocol}://${host}` - } - return { - url: cleanUrl, - auth: { - username, - password, - }, - } -} +let Pouch: any +let initialised = false /** * Return a constructor for PouchDB. @@ -92,3 +52,33 @@ export const getPouch = (opts: any = {}) => { return PouchDB.defaults(POUCH_DB_DEFAULTS) } + +export async function init(opts?: PouchOptions) { + Pouch = getPouch(opts) + initialised = true +} + +const checkInitialised = () => { + if (!initialised) { + throw new Error("init has not been called") + } +} + +export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { + checkInitialised() + return new Pouch(dbName, opts) +} + +// use this function if you have called getPouchDB - close +// the databases you've opened once finished +export async function closePouchDB(db: PouchDB.Database) { + if (!db || env.isTest()) { + return + } + try { + // specifically await so that if there is an error, it can be ignored + return await db.close() + } catch (err) { + // ignore error, already closed + } +} diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/db/couch/pouchLike.ts similarity index 99% rename from packages/backend-core/src/couch/pouchLike.ts rename to packages/backend-core/src/db/couch/pouchLike.ts index fd5a94e1af..975bbd96c4 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/db/couch/pouchLike.ts @@ -1,6 +1,6 @@ import Nano from "nano" import { AllDocsResponse, AnyDocument } from "@budibase/types" -import { getCouchInfo } from "./couch" +import { getCouchInfo } from "./connections" import { directCouchCall } from "./utils" import { getPouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts similarity index 89% rename from packages/backend-core/src/couch/utils.ts rename to packages/backend-core/src/db/couch/utils.ts index 25a4643d54..426bf92158 100644 --- a/packages/backend-core/src/couch/utils.ts +++ b/packages/backend-core/src/db/couch/utils.ts @@ -1,6 +1,6 @@ -import { getCouchInfo } from "./couch" +import { getCouchInfo } from "./connections" import fetch from "node-fetch" -import { checkSlashesInUrl } from "../helpers" +import { checkSlashesInUrl } from "../../helpers" export async function directCouchCall( path: string, diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index e732b3a2fb..7cc49eae40 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,8 +1,7 @@ import env from "../environment" import { CouchFindOptions } from "@budibase/types" -import { PouchLike } from "../couch" -import { directCouchQuery } from "../couch" -export { init, PouchLike } from "../couch" +import { directCouchQuery, PouchLike } from "./couch" +export { init, PouchLike, getPouch, getPouchDB } from "./couch" let initialised = false const dbList = new Set() diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index a59b2ffe05..9a4ed98418 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -15,7 +15,7 @@ import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" -import { PouchLike } from "../couch" +import { PouchLike } from "./couch" export * from "./constants" export * from "./conversions" diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 8ba508050f..23771d3f2d 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -1,6 +1,6 @@ import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils" import { getGlobalDB } from "../context" -import { PouchLike, QueryOpts } from "../couch" +import { PouchLike, QueryOpts } from "./couch" import { StaticDatabases } from "./constants" import { doWithDB } from "./" @@ -133,7 +133,9 @@ export const queryView = async ( try { let response = await db.query(`database/${viewName}`, params) const rows = response.rows - const docs = rows.map(row => (params.include_docs ? row.doc : row.value)) + const docs = rows.map((row: any) => + params.include_docs ? row.doc : row.value + ) // if arrayResponse has been requested, always return array regardless of length if (opts?.arrayResponse) { @@ -186,5 +188,5 @@ export const queryGlobalView = async ( db = getGlobalDB() } const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db, createFn, opts) + return queryView(viewName, params, db!, createFn, opts) } diff --git a/packages/backend-core/src/pkg/db.ts b/packages/backend-core/src/pkg/db.ts index 0254adddd5..d2bc474786 100644 --- a/packages/backend-core/src/pkg/db.ts +++ b/packages/backend-core/src/pkg/db.ts @@ -3,5 +3,4 @@ export * from "../db" export * from "../db/utils" export * from "../db/views" -export * from "../db/pouch" export * from "../db/constants" diff --git a/packages/backend-core/src/types.ts b/packages/backend-core/src/types.ts index 5eb3109b27..eeb784b0c1 100644 --- a/packages/backend-core/src/types.ts +++ b/packages/backend-core/src/types.ts @@ -1 +1 @@ -export { PouchLike } from "./couch" +export { PouchLike } from "./db" diff --git a/packages/server/package.json b/packages/server/package.json index 2c4394fc64..6d75877c79 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -42,7 +42,8 @@ "moduleNameMapper": { "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", - "@budibase/types": "/../types/src" + "@budibase/types": "/../types/src", + "axios": "axios/dist/node/axios.cjs" }, "setupFiles": [ "./scripts/jestSetup.js" diff --git a/packages/worker/package.json b/packages/worker/package.json index c3b90811ac..844448b569 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -98,7 +98,8 @@ "moduleNameMapper": { "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", - "@budibase/types": "/../types/src" + "@budibase/types": "/../types/src", + "axios": "axios/dist/node/axios.cjs" }, "setupFiles": [ "./scripts/jestSetup.js"