diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index ff0251f98e..00606e073c 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -48,7 +48,7 @@ "posthog-node": "1.3.0", "pouchdb": "7.3.0", "pouchdb-find": "7.2.2", - "pouchdb-replication-stream": "1.2.9", + "@budibase/pouchdb-replication-stream": "1.2.10", "redlock": "4.2.0", "sanitize-s3-objectkey": "0.0.1", "semver": "7.3.7", diff --git a/packages/backend-core/src/db/couch/pouchDB.ts b/packages/backend-core/src/db/couch/pouchDB.ts index a6f4323d88..f83127d466 100644 --- a/packages/backend-core/src/db/couch/pouchDB.ts +++ b/packages/backend-core/src/db/couch/pouchDB.ts @@ -39,7 +39,7 @@ export const getPouch = (opts: PouchOptions = {}) => { } if (opts.replication) { - const replicationStream = require("pouchdb-replication-stream") + const replicationStream = require("@budibase/pouchdb-replication-stream") PouchDB.plugin(replicationStream.plugin) // @ts-ignore PouchDB.adapter("writableStream", replicationStream.adapters.writableStream) diff --git a/packages/backend-core/src/db/couch/pouchDump.ts b/packages/backend-core/src/db/couch/pouchDump.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index fbfc52c6e8..d88b1058f9 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -487,6 +487,19 @@ qs "^6.11.0" tough-cookie "^4.1.2" +"@budibase/pouchdb-replication-stream@1.2.10": + version "1.2.10" + resolved "https://registry.yarnpkg.com/@budibase/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.10.tgz#4100df2effd7c823edadddcdbdc380f6827eebf5" + integrity sha512-1zeorOwbelZ7HF5vFB+pKE8Mnh31om8k1M6T3AZXVULYTHLsyJrMTozSv5CJ1P8ZfOIJab09HDzCXDh2icFekg== + dependencies: + argsarray "0.0.1" + inherits "^2.0.3" + lodash.pick "^4.0.0" + ndjson "^1.4.3" + pouch-stream "^0.4.0" + pouchdb-promise "^6.0.4" + through2 "^2.0.0" + "@cspotcode/source-map-support@^0.8.0": version "0.8.1" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" @@ -5066,19 +5079,6 @@ pouchdb-promise@^6.0.4: dependencies: lie "3.1.1" -pouchdb-replication-stream@1.2.9: - version "1.2.9" - resolved "https://registry.yarnpkg.com/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.9.tgz#aa4fa5d8f52df4825392f18e07c7e11acffc650a" - integrity sha512-hM8XRBfamTTUwRhKwLS/jSNouBhn9R/4ugdHNRD1EvJzwV8iImh6sDYbCU9PGuznjyOjXz6vpFRzKeI2KYfwnQ== - dependencies: - argsarray "0.0.1" - inherits "^2.0.3" - lodash.pick "^4.0.0" - ndjson "^1.4.3" - pouch-stream "^0.4.0" - pouchdb-promise "^6.0.4" - through2 "^2.0.0" - pouchdb-selector-core@7.2.2: version "7.2.2" resolved "https://registry.yarnpkg.com/pouchdb-selector-core/-/pouchdb-selector-core-7.2.2.tgz#264d7436a8c8ac3801f39960e79875ef7f3879a0" diff --git a/packages/cli/src/backups/utils.js b/packages/cli/src/backups/utils.js index 645b90379b..21d3504373 100644 --- a/packages/cli/src/backups/utils.js +++ b/packages/cli/src/backups/utils.js @@ -74,17 +74,17 @@ exports.getConfig = async (envFile = true) => { return config } -exports.replication = (from, to) => { - return new Promise((resolve, reject) => { - from.replicate - .to(to) - .on("complete", () => { - resolve() - }) - .on("error", err => { - reject(err) - }) - }) +exports.replication = async (from, to) => { + const pouch = getPouch() + try { + await pouch.replicate(from, to, { + batch_size: 1000, + batch_limit: 5, + style: "main_only", + }) + } catch (err) { + throw new Error(`Replication failed - ${JSON.stringify(err)}`) + } } exports.getPouches = config => { diff --git a/packages/server/scripts/load/create-many-apps.js b/packages/server/scripts/load/create-many-apps.js new file mode 100755 index 0000000000..9532313880 --- /dev/null +++ b/packages/server/scripts/load/create-many-apps.js @@ -0,0 +1,24 @@ +#!/bin/node +const { createApp } = require("./utils") + +const APP_COUNT = 100 + +if (!process.argv[2]) { + console.error("Please specify an API key as script argument.") + process.exit(-1) +} + +async function run() { + for (let i = 0; i < APP_COUNT; i++) { + const app = await createApp(process.argv[2]) + console.log(`App created: ${app._id}`) + } +} + +run() + .then(() => { + console.log(`Finished creating ${APP_COUNT} apps.`) + }) + .catch(err => { + console.error(err) + }) diff --git a/packages/server/scripts/load/create-many-rows.js b/packages/server/scripts/load/create-many-rows.js new file mode 100755 index 0000000000..85c0c292a0 --- /dev/null +++ b/packages/server/scripts/load/create-many-rows.js @@ -0,0 +1,30 @@ +#!/bin/node +const { createApp, getTable, createRow } = require("./utils") + +const ROW_COUNT = 1000 + +if (!process.argv[2]) { + console.error("Please specify an API key as script argument.") + process.exit(-1) +} + +async function run() { + const apiKey = process.argv[2] + const app = await createApp(apiKey) + console.log(`App created: ${app._id}`) + const table = await getTable(apiKey, app._id) + console.log(`Table found: ${table.name}`) + const promises = [] + for (let i = 0; i < ROW_COUNT; i++) { + promises.push(await createRow(apiKey, app._id, table)) + } + await Promise.all(promises) +} + +run() + .then(() => { + console.log(`Finished creating ${ROW_COUNT} rows.`) + }) + .catch(err => { + console.error(err) + }) diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js new file mode 100644 index 0000000000..97ff8f1e13 --- /dev/null +++ b/packages/server/scripts/load/utils.js @@ -0,0 +1,66 @@ +const fetch = require("node-fetch") +const uuid = require("uuid/v4") + +const URL_APP = "http://localhost:10000/api/public/v1/applications" +const URL_TABLE = "http://localhost:10000/api/public/v1/tables/search" + +async function request(apiKey, url, method, body, appId = undefined) { + const headers = { + "x-budibase-api-key": apiKey, + "Content-Type": "application/json", + } + if (appId) { + headers["x-budibase-app-id"] = appId + } + const res = await fetch(url, { + method, + headers, + body: JSON.stringify(body), + }) + if (res.status !== 200) { + throw new Error(await res.text()) + } + return res +} + +exports.createApp = async apiKey => { + const name = uuid().replace(/-/g, "") + const body = { + name, + url: `/${name}`, + useTemplate: "true", + templateKey: "app/school-admin-panel", + templateName: "School Admin Panel", + } + const res = await request(apiKey, URL_APP, "POST", body) + const json = await res.json() + return json.data +} + +exports.getTable = async (apiKey, appId) => { + const res = await request(apiKey, URL_TABLE, "POST", {}, appId) + const json = await res.json() + return json.data[0] +} + +exports.createRow = async (apiKey, appId, table) => { + const body = {} + for (let [key, schema] of Object.entries(table.schema)) { + let fake + switch (schema.type) { + default: + case "string": + fake = schema.constraints.inclusion + ? schema.constraints.inclusion[0] + : "a" + break + case "number": + fake = 1 + break + } + body[key] = fake + } + const url = `http://localhost:10000/api/public/v1/tables/${table._id}/rows` + const res = await request(apiKey, url, "POST", body, appId) + return (await res.json()).data +} diff --git a/packages/server/src/api/routes/public/applications.ts b/packages/server/src/api/routes/public/applications.ts index 935321cb7b..9d6c380e60 100644 --- a/packages/server/src/api/routes/public/applications.ts +++ b/packages/server/src/api/routes/public/applications.ts @@ -1,7 +1,6 @@ import controller from "../../controllers/public/applications" import Endpoint from "./utils/Endpoint" const { nameValidator, applicationValidator } = require("../utils/validators") -import { db } from "@budibase/backend-core" const read = [], write = [] diff --git a/packages/server/src/api/routes/tests/automation.spec.js b/packages/server/src/api/routes/tests/automation.spec.js index 796ee97c6a..2eae5e2a61 100644 --- a/packages/server/src/api/routes/tests/automation.spec.js +++ b/packages/server/src/api/routes/tests/automation.spec.js @@ -149,7 +149,7 @@ describe("/automations", () => { let elements = await getAllTableRows(config) // don't test it unless there are values to test if (elements.length > 1) { - expect(elements.length).toEqual(5) + expect(elements.length).toBeGreaterThanOrEqual(MAX_RETRIES) expect(elements[0].name).toEqual("Test") expect(elements[0].description).toEqual("TEST") return diff --git a/packages/server/src/api/routes/tests/cloud.seq.spec.ts b/packages/server/src/api/routes/tests/cloud.seq.spec.ts index 3ed9241a51..1c2006fc36 100644 --- a/packages/server/src/api/routes/tests/cloud.seq.spec.ts +++ b/packages/server/src/api/routes/tests/cloud.seq.spec.ts @@ -1,3 +1,5 @@ +jest.setTimeout(30000) + import { AppStatus } from "../../../db/utils" import * as setup from "./utilities" diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 9acad1344c..25bd1f4795 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -45,12 +45,18 @@ function tarFilesToTmp(tmpDir: string, files: string[]) { * @return {*} either a readable stream or a string */ export async function exportDB(dbName: string, opts: ExportOpts = {}) { + const exportOpts = { + filter: opts?.filter, + batch_size: 1000, + batch_limit: 5, + style: "main_only", + } return dbCore.doWithDB(dbName, async (db: any) => { // Write the dump to file if required if (opts?.exportPath) { const path = opts?.exportPath const writeStream = fs.createWriteStream(path) - await db.dump(writeStream, { filter: opts?.filter }) + await db.dump(writeStream, exportOpts) return path } else { // Stringify the dump in memory if required @@ -59,7 +65,7 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) { memStream.on("data", (chunk: any) => { appString += chunk.toString() }) - await db.dump(memStream, { filter: opts?.filter }) + await db.dump(memStream, exportOpts) return appString } })