1
0
Fork 0
mirror of synced 2024-06-13 16:05:06 +12:00

Updating export/import to include the budibase client and manifest.

This commit is contained in:
mike12345567 2022-10-18 16:42:25 +01:00
parent 07d4d9ee4d
commit bd1300b82a
4 changed files with 63 additions and 33 deletions

View file

@ -670,6 +670,11 @@ has@^1.0.3:
dependencies:
function-bind "^1.1.1"
html5-qrcode@^2.2.1:
version "2.2.3"
resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.2.3.tgz#5acb826860365e7c7ab91e1e14528ea16a502e8a"
integrity sha512-9CtEz5FVT56T76entiQxyrASzBWl8Rm30NHiQH8T163Eml5LS14BoZlYel9igxbikOt7O8KhvrT3awN1Y2HMqw==
htmlparser2@^6.0.0:
version "6.1.0"
resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7"

View file

@ -1,16 +1,23 @@
import { backups } from "@budibase/pro"
import { objectStore, tenancy } from "@budibase/backend-core"
import { AppBackupQueueData } from "@budibase/types"
import { exportApp } from "./exports"
import { Job } from "bull"
import fs from "fs"
import env from "../../../environment"
async function importProcessor(job: Job) {}
async function importProcessor(job: Job) {
const data: AppBackupQueueData = job.data
const appId = data.appId,
backupId = data.import!.backupId
const { path, metadata } = await backups.downloadAppBackup(backupId)
}
async function exportProcessor(job: Job) {
const appId = job.data.appId,
trigger = job.data.trigger,
name = job.data.name
const data: AppBackupQueueData = job.data
const appId = data.appId,
trigger = data.export!.trigger,
name = data.export!.name
const tenantId = tenancy.getTenantIDFromAppID(appId)
await tenancy.doInTenant(tenantId, async () => {
const createdAt = new Date().toISOString()

View file

@ -1,6 +1,9 @@
import { db as dbCore } from "@budibase/backend-core"
import { budibaseTempDir } from "../../../utilities/budibaseDir"
import { retrieveDirectory } from "../../../utilities/fileSystem/utilities"
import {
retrieveDirectory,
retrieve,
} from "../../../utilities/fileSystem/utilities"
import { streamFile } from "../../../utilities/fileSystem"
import { ObjectStoreBuckets } from "../../../constants"
import {
@ -8,11 +11,7 @@ import {
TABLE_ROW_PREFIX,
USER_METDATA_PREFIX,
} from "../../../db/utils"
import {
DB_EXPORT_FILE,
GLOBAL_DB_EXPORT_FILE,
ATTACHMENT_DIR,
} from "./constants"
import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
import fs from "fs"
import { join } from "path"
const uuid = require("uuid/v4")
@ -87,21 +86,19 @@ function defineFilter(excludeRows?: boolean) {
*/
export async function exportApp(appId: string, config?: ExportOpts) {
const prodAppId = dbCore.getProdAppID(appId)
const attachmentsPath = `${prodAppId}/${ATTACHMENT_DIR}`
// export attachments to tmp
const tmpPath = await retrieveDirectory(
ObjectStoreBuckets.APPS,
attachmentsPath
)
const downloadedPath = join(tmpPath, attachmentsPath),
tmpAttachmentPath = join(tmpPath, ATTACHMENT_DIR)
const appPath = `${prodAppId}/`
// export bucket contents
const tmpPath = await retrieveDirectory(ObjectStoreBuckets.APPS, appPath)
const downloadedPath = join(tmpPath, appPath)
if (fs.existsSync(downloadedPath)) {
// move out of app directory, simplify structure
fs.renameSync(downloadedPath, tmpAttachmentPath)
const allFiles = fs.readdirSync(downloadedPath)
for (let file of allFiles) {
const path = join(downloadedPath, file)
// move out of app directory, simplify structure
fs.renameSync(path, join(downloadedPath, "..", file))
}
// remove the old app directory created by object export
fs.rmdirSync(join(tmpPath, prodAppId))
} else {
fs.mkdirSync(tmpAttachmentPath)
fs.rmdirSync(downloadedPath)
}
// enforce an export of app DB to the tmp path
const dbPath = join(tmpPath, DB_EXPORT_FILE)
@ -113,7 +110,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
// if tar requested, return where the tarball is
if (config?.tar) {
// now the tmpPath contains both the DB export and attachments, tar this
const tarPath = tarFilesToTmp(tmpPath, [ATTACHMENT_DIR, DB_EXPORT_FILE])
const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
// cleanup the tmp export files as tarball returned
fs.rmSync(tmpPath, { recursive: true, force: true })
return tarPath

View file

@ -6,9 +6,12 @@ import {
ATTACHMENT_DIR,
GLOBAL_DB_EXPORT_FILE,
} from "./constants"
import { uploadDirectory } from "../../../utilities/fileSystem/utilities"
import {
uploadDirectory,
upload,
} from "../../../utilities/fileSystem/utilities"
import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
import { join } from "path"
import { join, basename } from "path"
import fs from "fs"
import sdk from "../../"
import { CouchFindOptions, RowAttachment } from "@budibase/types"
@ -127,14 +130,32 @@ export async function importApp(
template.file && fs.lstatSync(template.file.path).isDirectory()
if (template.file && (isTar || isDirectory)) {
const tmpPath = isTar ? untarFile(template.file) : template.file.path
const attachmentPath = join(tmpPath, ATTACHMENT_DIR)
const contents = fs.readdirSync(tmpPath)
// have to handle object import
if (fs.existsSync(attachmentPath)) {
await uploadDirectory(
ObjectStoreBuckets.APPS,
attachmentPath,
join(prodAppId, ATTACHMENT_DIR)
)
if (contents.length) {
let promises = []
let excludedFiles = [GLOBAL_DB_EXPORT_FILE, DB_EXPORT_FILE]
for (let filename of contents) {
const path = join(tmpPath, filename)
if (excludedFiles.includes(filename)) {
continue
}
filename = join(prodAppId, filename)
if (fs.lstatSync(path).isDirectory()) {
promises.push(
uploadDirectory(ObjectStoreBuckets.APPS, path, filename)
)
} else {
promises.push(
upload({
bucket: ObjectStoreBuckets.APPS,
path,
filename,
})
)
}
}
await Promise.all(promises)
}
dbStream = fs.createReadStream(join(tmpPath, DB_EXPORT_FILE))
} else {