CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-06-01 18:20:18 +12:00
Code Issues Projects Releases Wiki Activity

Adding main functionality for exporting CouchDB to disk, just need to add MinIO export.

Browse source
This commit is contained in:
mike12345567 2022-06-30 19:20:50 +01:00
parent 59e3737250
commit c633c643e2
9 changed files with 1352 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
packages/backend-core/src/db/pouch.js
View file

@ -102,6 +102,13 @@ exports.getPouch = (opts = {}) => {
}
}
if (opts.onDisk) {
POUCH_DB_DEFAULTS = {
prefix: undefined,
adapter: "leveldb",
}
}
if (opts.replication) {
const replicationStream = require("pouchdb-replication-stream")
PouchDB.plugin(replicationStream.plugin)

1
packages/backend-core/src/db/utils.ts
View file

@ -185,6 +185,7 @@ export async function getAllDbs(opts = { efficient: false }) {
}
}
let couchUrl = `${url}/_all_dbs`
console.log(couchUrl)
let tenantId = getTenantId()
if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) {
// just get all DBs when:

6
packages/cli/package.json
View file

@ -20,8 +20,10 @@
"outputPath": "build"
},
"dependencies": {
"@budibase/backend-core": "^1.0.212-alpha.7",
"axios": "^0.21.1",
"chalk": "^4.1.0",
"cli-progress": "^3.11.2",
"commander": "^7.1.0",
"docker-compose": "^0.23.6",
"dotenv": "^16.0.1",
@ -29,8 +31,8 @@
"lookpath": "^1.1.0",
"pkg": "^5.3.0",
"posthog-node": "1.0.7",
"pouchdb": "^7.3.0",
"randomstring": "^1.1.5"
"randomstring": "^1.1.5",
"tar": "^6.1.11"
},
"devDependencies": {
"eslint": "^7.20.0"

157
packages/cli/src/backups/index.js
View file

@ -1,41 +1,48 @@
const Command = require("../structures/Command")
const { CommandWords } = require("../constants")
//const pouchdb = require("pouchdb")
const dotenv = require("dotenv")
const fs = require("fs")
const { join } = require("path")
const { string } = require("../questions")
const { env } = require("@budibase/backend-core")
const { getPouch, getAllDbs } = require("@budibase/backend-core/db")
const tar = require("tar")
const { progressBar } = require("../utils")
const DEFAULT_COUCH = "http://budibase:budibase@localhost:10000/db/"
const DEFAULT_MINIO = "http://localhost:10000/"
const TEMP_DIR = ".temp"
const REQUIRED = [
{ value: "MAIN_PORT", key: "Budibase Port", default: "10000" },
{
value: "COUCH_DB_URL",
key: "CouchDB URL",
default: "http://budibase:budibase@localhost:10000/db/",
},
{ value: "MINIO_URL", key: "MinIO URL", default: "http://localhost:10000/" },
{ value: "MINIO_ACCESS_KEY", key: "MinIO Access Key" },
{ value: "MINIO_SECRET_KEY", key: "MinIO Secret Key" },
{ value: "MAIN_PORT", default: "10000" },
{ value: "COUCH_DB_URL", default: DEFAULT_COUCH },
{ value: "MINIO_URL", default: DEFAULT_MINIO },
{ value: "MINIO_ACCESS_KEY" },
{ value: "MINIO_SECRET_KEY" },
]
function checkCouchURL(config) {
if (config["COUCH_DB_URL"]) {
return config
}
function checkURLs(config) {
const mainPort = config["MAIN_PORT"],
username = config["COUCH_DB_USERNAME"],
username = config["COUCH_DB_USER"],
password = config["COUCH_DB_PASSWORD"]
if (mainPort && username && password) {
if (!config["COUCH_DB_URL"] && mainPort && username && password) {
config[
"COUCH_DB_URL"
] = `http://${username}:${password}@localhost:${mainPort}/db/`
}
if (!config["MINIO_URL"]) {
config["MINIO_URL"] = DEFAULT_MINIO
}
return config
}
async function askQuestions() {
console.log(
"*** NOTE: use a .env file to load these parameters repeatedly ***"
)
let config = {}
for (let property of REQUIRED) {
config[property.value] = await string(property.key, property.default)
config[property.value] = await string(property.value, property.default)
}
return config
}
@ -45,7 +52,7 @@ function loadEnvironment(path) {
throw "Unable to file specified .env file"
}
const env = fs.readFileSync(path, "utf8")
const config = checkCouchURL(dotenv.parse(env))
const config = checkURLs(dotenv.parse(env))
for (let required of REQUIRED) {
if (!config[required.value]) {
throw `Cannot find "${required.value}" property in .env file`
@ -56,21 +63,110 @@ function loadEnvironment(path) {
// true is the default value passed by commander
async function getConfig(envFile = true) {
let config
if (envFile !== true) {
return loadEnvironment(envFile)
config = loadEnvironment(envFile)
} else {
return askQuestions()
config = askQuestions()
}
for (let required of REQUIRED) {
env._set(required.value, config[required.value])
}
return config
}
async function exportBackup(envFile) {
const config = await getConfig(envFile)
console.log(config)
function replication(from, to) {
return new Promise((resolve, reject) => {
from.replicate
.to(to)
.on("complete", () => {
resolve()
})
.on("error", err => {
reject(err)
})
})
}
async function importBackup(envFile) {
const config = await getConfig(envFile)
console.log(config)
function getPouches() {
const Remote = getPouch({ replication: true })
const Local = getPouch({ onDisk: true, directory: TEMP_DIR })
return { Remote, Local }
}
async function exportBackup(opts) {
const envFile = opts.env || undefined
await getConfig(envFile)
let filename = opts["export"] || opts
if (typeof filename !== "string") {
filename = `backup-${new Date().toISOString()}.tar.gz`
}
await getConfig(envFile)
const dbList = await getAllDbs()
const { Remote, Local } = getPouches()
if (fs.existsSync(TEMP_DIR)) {
fs.rmSync(TEMP_DIR, { recursive: true })
}
const couchDir = join(TEMP_DIR, "couchdb")
fs.mkdirSync(TEMP_DIR)
fs.mkdirSync(couchDir)
const bar = progressBar(dbList.length)
let count = 0
for (let db of dbList) {
bar.update(++count)
const remote = new Remote(db)
const local = new Local(join(TEMP_DIR, "couchdb", db))
await replication(remote, local)
}
bar.stop()
tar.create(
{
sync: true,
gzip: true,
file: filename,
cwd: join(TEMP_DIR),
},
["couchdb"]
)
fs.rmSync(TEMP_DIR, { recursive: true })
console.log(`Generated export file - ${filename}`)
}
async function importBackup(opts) {
const envFile = opts.env || undefined
const filename = opts["import"] || opts
await getConfig(envFile)
if (!filename || !fs.existsSync(filename)) {
console.error("Cannot import without specifying a valid file to import")
process.exit(-1)
}
fs.mkdirSync(TEMP_DIR)
tar.extract({
sync: true,
cwd: join(TEMP_DIR),
file: filename,
})
const { Remote, Local } = getPouches()
const dbList = fs.readdirSync(join(TEMP_DIR, "couchdb"))
const bar = progressBar(dbList.length)
let count = 0
for (let db of dbList) {
bar.update(++count)
const remote = new Remote(db)
const local = new Local(join(TEMP_DIR, "couchdb", db))
await replication(local, remote)
}
bar.stop()
console.log("Import complete")
fs.rmSync(TEMP_DIR, { recursive: true })
}
async function pickOne(opts) {
if (opts["import"]) {
return importBackup(opts)
} else if (opts["export"]) {
return exportBackup(opts)
}
}
const command = new Command(`${CommandWords.BACKUPS}`)
@ -78,14 +174,19 @@ const command = new Command(`${CommandWords.BACKUPS}`)
"Allows building backups of Budibase, as well as importing a backup to a new instance."
)
.addSubOption(
"--export [envFile]",
"--export [filename]",
"Export a backup from an existing Budibase installation.",
exportBackup
)
.addSubOption(
"--import [envFile]",
"--import [filename]",
"Import a backup to a new Budibase installation.",
importBackup
)
.addSubOption(
"--env [envFile]",
"Provide an environment variable file to configure the CLI.",
pickOne
)
exports.command = command

6
packages/cli/src/structures/Command.js
View file

@ -39,8 +39,10 @@ class Command {
let executed = false
for (let opt of thisCmd.opts) {
const lookup = opt.command.split(" ")[0].replace("--", "")
if (options[lookup]) {
await opt.func(options[lookup])
if (!executed && options[lookup]) {
const input =
Object.keys(options).length > 1 ? options : options[lookup]
await opt.func(input)
executed = true
}
}

7
packages/cli/src/utils.js
View file

@ -2,6 +2,7 @@ const chalk = require("chalk")
const fs = require("fs")
const axios = require("axios")
const path = require("path")
const progress = require("cli-progress")
exports.downloadFile = async (url, filePath) => {
filePath = path.resolve(filePath)
@ -56,3 +57,9 @@ exports.parseEnv = env => {
}
return result
}
exports.progressBar = total => {
const bar = new progress.SingleBar({}, progress.Presets.shades_classic)
bar.start(total, 0)
return bar
}

1194
packages/cli/yarn.lock
View file

File diff suppressed because it is too large Load diff

28
packages/server/yarn.lock
View file

@ -1094,12 +1094,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.212-alpha.0":
version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
"@budibase/backend-core@1.0.212-alpha.7":
version "1.0.212-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.7.tgz#33502474c79f1b417a0456ced68ea437eb6badd3"
integrity sha512-aQWQbRasLwk0+lVGUp7Yzc4Yui/v9J02Al8SLhv8NDF81JNLmEgGKD/uo8hY+eseuNZe4rLCAoHkoP+0PeHNVQ==
dependencies:
"@budibase/types" "^1.0.212-alpha.0"
"@budibase/types" "^1.0.212-alpha.7"
"@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0"
bcrypt "5.0.1"
@ -1176,12 +1176,12 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/pro@1.0.212-alpha.0":
version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
"@budibase/pro@1.0.212-alpha.7":
version "1.0.212-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.7.tgz#c2bdddcd3823f361126a47943c3207dbf213377e"
integrity sha512-EMF3a9KTuNJ506EKeJpOLQug0KkuryN952SIJ/+Q22cKaBAvo4NDal7w4rRvld9iQVLBhKhqHwuJ+l948WWffA==
dependencies:
"@budibase/backend-core" "1.0.212-alpha.0"
"@budibase/backend-core" "1.0.212-alpha.7"
node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139":
@ -1202,10 +1202,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/types@^1.0.212-alpha.0":
version "1.0.212"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
"@budibase/types@^1.0.212-alpha.7":
version "1.0.214"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.214.tgz#400724fb23a1ce4b80bd04ed472a376d59126069"
integrity sha512-KD9XkCdL+R4Nal/X2IX0Bv+AxsylGzAAmo7M6tuoM7YAegskmsX5Kfk5yasnbMnhOdICyvbJ1D6Hz7WKwdApzw==
"@bull-board/api@3.7.0":
version "3.7.0"

28
packages/worker/yarn.lock
View file

@ -291,12 +291,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.212-alpha.0":
version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
"@budibase/backend-core@1.0.212-alpha.7":
version "1.0.212-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.7.tgz#33502474c79f1b417a0456ced68ea437eb6badd3"
integrity sha512-aQWQbRasLwk0+lVGUp7Yzc4Yui/v9J02Al8SLhv8NDF81JNLmEgGKD/uo8hY+eseuNZe4rLCAoHkoP+0PeHNVQ==
dependencies:
"@budibase/types" "^1.0.212-alpha.0"
"@budibase/types" "^1.0.212-alpha.7"
"@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0"
bcrypt "5.0.1"
@ -323,18 +323,18 @@
uuid "8.3.2"
zlib "1.0.5"
"@budibase/pro@1.0.212-alpha.0":
version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
"@budibase/pro@1.0.212-alpha.7":
version "1.0.212-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.7.tgz#c2bdddcd3823f361126a47943c3207dbf213377e"
integrity sha512-EMF3a9KTuNJ506EKeJpOLQug0KkuryN952SIJ/+Q22cKaBAvo4NDal7w4rRvld9iQVLBhKhqHwuJ+l948WWffA==
dependencies:
"@budibase/backend-core" "1.0.212-alpha.0"
"@budibase/backend-core" "1.0.212-alpha.7"
node-fetch "^2.6.1"
"@budibase/types@^1.0.212-alpha.0":
version "1.0.212"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
"@budibase/types@^1.0.212-alpha.7":
version "1.0.214"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.214.tgz#400724fb23a1ce4b80bd04ed472a376d59126069"
integrity sha512-KD9XkCdL+R4Nal/X2IX0Bv+AxsylGzAAmo7M6tuoM7YAegskmsX5Kfk5yasnbMnhOdICyvbJ1D6Hz7WKwdApzw==
"@cspotcode/source-map-consumer@0.8.0":
version "0.8.0"