Merge pull request #9550 from Budibase/fix/cli-memory-issues

CLI backup + app export memory usage improvements

packages/backend-core/package.json

@@ -48,7 +48,7 @@
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
     "pouchdb-find": "7.2.2",
-    "pouchdb-replication-stream": "1.2.9",
+    "@budibase/pouchdb-replication-stream": "1.2.10",
     "redlock": "4.2.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "7.3.7",

packages/backend-core/src/db/couch/pouchDB.ts

@@ -39,7 +39,7 @@ export const getPouch = (opts: PouchOptions = {}) => {
   }
 
   if (opts.replication) {
-    const replicationStream = require("pouchdb-replication-stream")
+    const replicationStream = require("@budibase/pouchdb-replication-stream")
     PouchDB.plugin(replicationStream.plugin)
     // @ts-ignore
     PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)

packages/backend-core/yarn.lock

@@ -487,6 +487,19 @@
     qs "^6.11.0"
     tough-cookie "^4.1.2"
 
+"@budibase/pouchdb-replication-stream@1.2.10":
+  version "1.2.10"
+  resolved "https://registry.yarnpkg.com/@budibase/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.10.tgz#4100df2effd7c823edadddcdbdc380f6827eebf5"
+  integrity sha512-1zeorOwbelZ7HF5vFB+pKE8Mnh31om8k1M6T3AZXVULYTHLsyJrMTozSv5CJ1P8ZfOIJab09HDzCXDh2icFekg==
+  dependencies:
+    argsarray "0.0.1"
+    inherits "^2.0.3"
+    lodash.pick "^4.0.0"
+    ndjson "^1.4.3"
+    pouch-stream "^0.4.0"
+    pouchdb-promise "^6.0.4"
+    through2 "^2.0.0"
+
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
@@ -5066,19 +5079,6 @@ pouchdb-promise@^6.0.4:
   dependencies:
     lie "3.1.1"
 
-pouchdb-replication-stream@1.2.9:
-  version "1.2.9"
-  resolved "https://registry.yarnpkg.com/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.9.tgz#aa4fa5d8f52df4825392f18e07c7e11acffc650a"
-  integrity sha512-hM8XRBfamTTUwRhKwLS/jSNouBhn9R/4ugdHNRD1EvJzwV8iImh6sDYbCU9PGuznjyOjXz6vpFRzKeI2KYfwnQ==
-  dependencies:
-    argsarray "0.0.1"
-    inherits "^2.0.3"
-    lodash.pick "^4.0.0"
-    ndjson "^1.4.3"
-    pouch-stream "^0.4.0"
-    pouchdb-promise "^6.0.4"
-    through2 "^2.0.0"
-
 pouchdb-selector-core@7.2.2:
   version "7.2.2"
   resolved "https://registry.yarnpkg.com/pouchdb-selector-core/-/pouchdb-selector-core-7.2.2.tgz#264d7436a8c8ac3801f39960e79875ef7f3879a0"

packages/cli/src/backups/utils.js

@@ -74,17 +74,17 @@ exports.getConfig = async (envFile = true) => {
   return config
 }
 
-exports.replication = (from, to) => {
-  return new Promise((resolve, reject) => {
-    from.replicate
-      .to(to)
-      .on("complete", () => {
-        resolve()
-      })
-      .on("error", err => {
-        reject(err)
-      })
-  })
+exports.replication = async (from, to) => {
+  const pouch = getPouch()
+  try {
+    await pouch.replicate(from, to, {
+      batch_size: 1000,
+      batch_limit: 5,
+      style: "main_only",
+    })
+  } catch (err) {
+    throw new Error(`Replication failed - ${JSON.stringify(err)}`)
+  }
 }
 
 exports.getPouches = config => {

packages/server/scripts/load/create-many-apps.js

@@ -0,0 +1,24 @@
+#!/bin/node
+const { createApp } = require("./utils")
+
+const APP_COUNT = 100
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  for (let i = 0; i < APP_COUNT; i++) {
+    const app = await createApp(process.argv[2])
+    console.log(`App created: ${app._id}`)
+  }
+}
+
+run()
+  .then(() => {
+    console.log(`Finished creating ${APP_COUNT} apps.`)
+  })
+  .catch(err => {
+    console.error(err)
+  })

packages/server/scripts/load/create-many-rows.js

@@ -0,0 +1,30 @@
+#!/bin/node
+const { createApp, getTable, createRow } = require("./utils")
+
+const ROW_COUNT = 1000
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  const apiKey = process.argv[2]
+  const app = await createApp(apiKey)
+  console.log(`App created: ${app._id}`)
+  const table = await getTable(apiKey, app._id)
+  console.log(`Table found: ${table.name}`)
+  const promises = []
+  for (let i = 0; i < ROW_COUNT; i++) {
+    promises.push(await createRow(apiKey, app._id, table))
+  }
+  await Promise.all(promises)
+}
+
+run()
+  .then(() => {
+    console.log(`Finished creating ${ROW_COUNT} rows.`)
+  })
+  .catch(err => {
+    console.error(err)
+  })

packages/server/scripts/load/utils.js

@@ -0,0 +1,66 @@
+const fetch = require("node-fetch")
+const uuid = require("uuid/v4")
+
+const URL_APP = "http://localhost:10000/api/public/v1/applications"
+const URL_TABLE = "http://localhost:10000/api/public/v1/tables/search"
+
+async function request(apiKey, url, method, body, appId = undefined) {
+  const headers = {
+    "x-budibase-api-key": apiKey,
+    "Content-Type": "application/json",
+  }
+  if (appId) {
+    headers["x-budibase-app-id"] = appId
+  }
+  const res = await fetch(url, {
+    method,
+    headers,
+    body: JSON.stringify(body),
+  })
+  if (res.status !== 200) {
+    throw new Error(await res.text())
+  }
+  return res
+}
+
+exports.createApp = async apiKey => {
+  const name = uuid().replace(/-/g, "")
+  const body = {
+    name,
+    url: `/${name}`,
+    useTemplate: "true",
+    templateKey: "app/school-admin-panel",
+    templateName: "School Admin Panel",
+  }
+  const res = await request(apiKey, URL_APP, "POST", body)
+  const json = await res.json()
+  return json.data
+}
+
+exports.getTable = async (apiKey, appId) => {
+  const res = await request(apiKey, URL_TABLE, "POST", {}, appId)
+  const json = await res.json()
+  return json.data[0]
+}
+
+exports.createRow = async (apiKey, appId, table) => {
+  const body = {}
+  for (let [key, schema] of Object.entries(table.schema)) {
+    let fake
+    switch (schema.type) {
+      default:
+      case "string":
+        fake = schema.constraints.inclusion
+          ? schema.constraints.inclusion[0]
+          : "a"
+        break
+      case "number":
+        fake = 1
+        break
+    }
+    body[key] = fake
+  }
+  const url = `http://localhost:10000/api/public/v1/tables/${table._id}/rows`
+  const res = await request(apiKey, url, "POST", body, appId)
+  return (await res.json()).data
+}

packages/server/src/api/routes/public/applications.ts

@@ -1,7 +1,6 @@
 import controller from "../../controllers/public/applications"
 import Endpoint from "./utils/Endpoint"
 const { nameValidator, applicationValidator } = require("../utils/validators")
-import { db } from "@budibase/backend-core"
 
 const read = [],
   write = []

packages/server/src/api/routes/tests/automation.spec.js

@@ -149,7 +149,7 @@ describe("/automations", () => {
         let elements = await getAllTableRows(config)
         // don't test it unless there are values to test
         if (elements.length > 1) {
-          expect(elements.length).toEqual(5)
+          expect(elements.length).toBeGreaterThanOrEqual(MAX_RETRIES)
           expect(elements[0].name).toEqual("Test")
           expect(elements[0].description).toEqual("TEST")
           return

packages/server/src/api/routes/tests/cloud.seq.spec.ts

@@ -1,3 +1,5 @@
+jest.setTimeout(30000)
+
 import { AppStatus } from "../../../db/utils"
 
 import * as setup from "./utilities"

packages/server/src/sdk/app/backups/exports.ts

@@ -45,12 +45,18 @@ function tarFilesToTmp(tmpDir: string, files: string[]) {
  * @return {*} either a readable stream or a string
  */
 export async function exportDB(dbName: string, opts: ExportOpts = {}) {
+  const exportOpts = {
+    filter: opts?.filter,
+    batch_size: 1000,
+    batch_limit: 5,
+    style: "main_only",
+  }
   return dbCore.doWithDB(dbName, async (db: any) => {
     // Write the dump to file if required
     if (opts?.exportPath) {
       const path = opts?.exportPath
       const writeStream = fs.createWriteStream(path)
-      await db.dump(writeStream, { filter: opts?.filter })
+      await db.dump(writeStream, exportOpts)
       return path
     } else {
       // Stringify the dump in memory if required
@@ -59,7 +65,7 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) {
       memStream.on("data", (chunk: any) => {
         appString += chunk.toString()
       })
-      await db.dump(memStream, { filter: opts?.filter })
+      await db.dump(memStream, exportOpts)
       return appString
     }
   })