CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-03 21:40:55 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into remove-vm2-from-stringtemplates

Browse source
This commit is contained in:
Adria Navarro 2024-02-09 13:22:00 +01:00 committed by GitHub
parent 0db98f7fb1 aaf138c9a2
commit 3d936069cd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 35 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/backend-core/package.json
View file

@ -32,7 +32,7 @@
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bull": "4.10.1", "bull": "4.10.1",
"correlation-id": "4.0.0", "correlation-id": "4.0.0",
"dd-trace": "5.0.0", "dd-trace": "5.2.0",
"dotenv": "16.0.1", "dotenv": "16.0.1",
"ioredis": "5.3.2", "ioredis": "5.3.2",
"joi": "17.6.0", "joi": "17.6.0",

2
packages/server/package.json
View file

@ -69,7 +69,7 @@
"cookies": "0.8.0", "cookies": "0.8.0",
"csvtojson": "2.0.10", "csvtojson": "2.0.10",
"curlconverter": "3.21.0", "curlconverter": "3.21.0",
"dd-trace": "5.0.0", "dd-trace": "5.2.0",
"dotenv": "8.2.0", "dotenv": "8.2.0",
"form-data": "4.0.0", "form-data": "4.0.0",
"global-agent": "3.0.0", "global-agent": "3.0.0",

17
packages/server/src/sdk/app/backups/exports.ts
View file

@ -14,6 +14,7 @@ import {
ATTACHMENT_DIRECTORY, ATTACHMENT_DIRECTORY,
} from "./constants" } from "./constants"
import fs from "fs" import fs from "fs"
import fsp from "fs/promises"
import { join } from "path" import { join } from "path"
import env from "../../../environment" import env from "../../../environment"
import { v4 as uuid } from "uuid" import { v4 as uuid } from "uuid"
@ -117,7 +118,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
ObjectStoreBuckets.APPS, ObjectStoreBuckets.APPS,
join(appPath, path) join(appPath, path)
) )
fs.writeFileSync(join(tmpPath, path), contents) await fsp.writeFile(join(tmpPath, path), contents)
} }
} }
// get all the files // get all the files
@ -131,14 +132,14 @@ export async function exportApp(appId: string, config?: ExportOpts) {
const downloadedPath = join(tmpPath, appPath) const downloadedPath = join(tmpPath, appPath)
if (fs.existsSync(downloadedPath)) { if (fs.existsSync(downloadedPath)) {
const allFiles = fs.readdirSync(downloadedPath) const allFiles = await fsp.readdir(downloadedPath)
for (let file of allFiles) { for (let file of allFiles) {
const path = join(downloadedPath, file) const path = join(downloadedPath, file)
// move out of app directory, simplify structure // move out of app directory, simplify structure
fs.renameSync(path, join(downloadedPath, "..", file)) await fsp.rename(path, join(downloadedPath, "..", file))
} }
// remove the old app directory created by object export // remove the old app directory created by object export
fs.rmdirSync(downloadedPath) await fsp.rmdir(downloadedPath)
} }
// enforce an export of app DB to the tmp path // enforce an export of app DB to the tmp path
const dbPath = join(tmpPath, DB_EXPORT_FILE) const dbPath = join(tmpPath, DB_EXPORT_FILE)
@ -148,7 +149,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
}) })
if (config?.encryptPassword) { if (config?.encryptPassword) {
for (let file of fs.readdirSync(tmpPath)) { for (let file of await fsp.readdir(tmpPath)) {
const path = join(tmpPath, file) const path = join(tmpPath, file)
// skip the attachments - too big to encrypt // skip the attachments - too big to encrypt
@ -157,7 +158,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
{ dir: tmpPath, filename: file }, { dir: tmpPath, filename: file },
config.encryptPassword config.encryptPassword
) )
fs.rmSync(path) await fsp.rm(path)
} }
} }
} }
@ -165,9 +166,9 @@ export async function exportApp(appId: string, config?: ExportOpts) {
// if tar requested, return where the tarball is // if tar requested, return where the tarball is
if (config?.tar) { if (config?.tar) {
// now the tmpPath contains both the DB export and attachments, tar this // now the tmpPath contains both the DB export and attachments, tar this
const tarPath = await tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath)) const tarPath = await tarFilesToTmp(tmpPath, await fsp.readdir(tmpPath))
// cleanup the tmp export files as tarball returned // cleanup the tmp export files as tarball returned
fs.rmSync(tmpPath, { recursive: true, force: true }) await fsp.rm(tmpPath, { recursive: true, force: true })
return tarPath return tarPath
} }

15
packages/server/src/sdk/app/backups/imports.ts
View file

@ -17,6 +17,7 @@ import { downloadTemplate } from "../../../utilities/fileSystem"
import { ObjectStoreBuckets } from "../../../constants" import { ObjectStoreBuckets } from "../../../constants"
import { join } from "path" import { join } from "path"
import fs from "fs" import fs from "fs"
import fsp from "fs/promises"
import sdk from "../../" import sdk from "../../"
import { v4 as uuid } from "uuid" import { v4 as uuid } from "uuid"
import tar from "tar" import tar from "tar"
@ -119,7 +120,7 @@ async function getTemplateStream(template: TemplateType) {
export async function untarFile(file: { path: string }) { export async function untarFile(file: { path: string }) {
const tmpPath = join(budibaseTempDir(), uuid()) const tmpPath = join(budibaseTempDir(), uuid())
fs.mkdirSync(tmpPath) await fsp.mkdir(tmpPath)
// extract the tarball // extract the tarball
await tar.extract({ await tar.extract({
cwd: tmpPath, cwd: tmpPath,
@ -130,12 +131,12 @@ export async function untarFile(file: { path: string }) {
async function decryptFiles(path: string, password: string) { async function decryptFiles(path: string, password: string) {
try { try {
for (let file of fs.readdirSync(path)) { for (let file of await fsp.readdir(path)) {
const inputPath = join(path, file) const inputPath = join(path, file)
if (!inputPath.endsWith(ATTACHMENT_DIRECTORY)) { if (!inputPath.endsWith(ATTACHMENT_DIRECTORY)) {
const outputPath = inputPath.replace(/\.enc$/, "") const outputPath = inputPath.replace(/\.enc$/, "")
await encryption.decryptFile(inputPath, outputPath, password) await encryption.decryptFile(inputPath, outputPath, password)
fs.rmSync(inputPath) await fsp.rm(inputPath)
} }
} }
} catch (err: any) { } catch (err: any) {
@ -164,14 +165,14 @@ export async function importApp(
let dbStream: any let dbStream: any
const isTar = template.file && template?.file?.type?.endsWith("gzip") const isTar = template.file && template?.file?.type?.endsWith("gzip")
const isDirectory = const isDirectory =
template.file && fs.lstatSync(template.file.path).isDirectory() template.file && (await fsp.lstat(template.file.path)).isDirectory()
let tmpPath: string | undefined = undefined let tmpPath: string | undefined = undefined
if (template.file && (isTar || isDirectory)) { if (template.file && (isTar || isDirectory)) {
tmpPath = isTar ? await untarFile(template.file) : template.file.path tmpPath = isTar ? await untarFile(template.file) : template.file.path
if (isTar && template.file.password) { if (isTar && template.file.password) {
await decryptFiles(tmpPath, template.file.password) await decryptFiles(tmpPath, template.file.password)
} }
const contents = fs.readdirSync(tmpPath) const contents = await fsp.readdir(tmpPath)
// have to handle object import // have to handle object import
if (contents.length && opts.importObjStoreContents) { if (contents.length && opts.importObjStoreContents) {
let promises = [] let promises = []
@ -182,7 +183,7 @@ export async function importApp(
continue continue
} }
filename = join(prodAppId, filename) filename = join(prodAppId, filename)
if (fs.lstatSync(path).isDirectory()) { if ((await fsp.lstat(path)).isDirectory()) {
promises.push( promises.push(
objectStore.uploadDirectory(ObjectStoreBuckets.APPS, path, filename) objectStore.uploadDirectory(ObjectStoreBuckets.APPS, path, filename)
) )
@ -211,7 +212,7 @@ export async function importApp(
await updateAutomations(prodAppId, db) await updateAutomations(prodAppId, db)
// clear up afterward // clear up afterward
if (tmpPath) { if (tmpPath) {
fs.rmSync(tmpPath, { recursive: true, force: true }) await fsp.rm(tmpPath, { recursive: true, force: true })
} }
return ok return ok
} }

2
packages/worker/package.json
View file

@ -48,7 +48,7 @@
"bcrypt": "5.1.0", "bcrypt": "5.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bull": "4.10.1", "bull": "4.10.1",
"dd-trace": "5.0.0", "dd-trace": "5.2.0",
"dotenv": "8.6.0", "dotenv": "8.6.0",
"global-agent": "3.0.0", "global-agent": "3.0.0",
"ical-generator": "4.1.0", "ical-generator": "4.1.0",

38
yarn.lock
View file

@ -2220,10 +2220,10 @@
enabled "2.0.x" enabled "2.0.x"
kuler "^2.0.0" kuler "^2.0.0"
"@datadog/native-appsec@6.0.0": "@datadog/native-appsec@7.0.0":
version "6.0.0" version "7.0.0"
resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-6.0.0.tgz#da753f8566ec5180ad9e83014cb44984b4bc878e" resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-7.0.0.tgz#a380174dd49aef2d9bb613a0ec8ead6dc7822095"
integrity sha512-e7vH5usFoqov7FraPcA99fe80t2/qm4Cmno1T3iBhYlhyO6HD01ArDsCZ/sUvNIUR1ujxtbr8Z9WRGJ0qQ/FDA== integrity sha512-bywstWFW2hWxzPuS0+mFMVHHL0geulx5yQFtsjfszaH2LTAgk2D+Rt40MKbAoZ8q3tRw2dy6aYQ7svO3ca8jpA==
dependencies: dependencies:
node-gyp-build "^3.9.0" node-gyp-build "^3.9.0"
@ -8792,12 +8792,12 @@ dc-polyfill@^0.1.2:
resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.3.tgz#fe9eefc86813439dd46d6f9ad9582ec079c39720" resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.3.tgz#fe9eefc86813439dd46d6f9ad9582ec079c39720"
integrity sha512-Wyk5n/5KUj3GfVKV2jtDbtChC/Ff9fjKsBcg4ZtYW1yQe3DXNHcGURvmoxhqQdfOQ9TwyMjnfyv1lyYcOkFkFA== integrity sha512-Wyk5n/5KUj3GfVKV2jtDbtChC/Ff9fjKsBcg4ZtYW1yQe3DXNHcGURvmoxhqQdfOQ9TwyMjnfyv1lyYcOkFkFA==
dd-trace@5.0.0: dd-trace@5.2.0:
version "5.0.0" version "5.2.0"
resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.0.0.tgz#1e9848d6b6212ca67f8a3d62ce1f9ecd93fb5ebb" resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.2.0.tgz#6ca2d76ece95f08d98468d7782c22f24192afa53"
integrity sha512-MmbM05l0qFeM73kDyyQAHWvyeZl2m6FYlv3hgtBU8GSpFmNu/33llyYp4TDpoEJ7hqd5LWT7mKKQFq8lRbTH3w== integrity sha512-Z5ql3ZKzVW3DPstHPkTPcIPvKljHNtzTYY/WuZRlgT4XK7rMaN0j5nA8LlUh7m+tOPWs05IiKngbYVZjsqhRgA==
dependencies: dependencies:
"@datadog/native-appsec" "6.0.0" "@datadog/native-appsec" "7.0.0"
"@datadog/native-iast-rewriter" "2.2.2" "@datadog/native-iast-rewriter" "2.2.2"
"@datadog/native-iast-taint-tracking" "1.6.4" "@datadog/native-iast-taint-tracking" "1.6.4"
"@datadog/native-metrics" "^2.0.0" "@datadog/native-metrics" "^2.0.0"
@ -8808,17 +8808,14 @@ dd-trace@5.0.0:
crypto-randomuuid "^1.0.0" crypto-randomuuid "^1.0.0"
dc-polyfill "^0.1.2" dc-polyfill "^0.1.2"
ignore "^5.2.4" ignore "^5.2.4"
import-in-the-middle "^1.7.1" import-in-the-middle "^1.7.3"
int64-buffer "^0.1.9" int64-buffer "^0.1.9"
ipaddr.js "^2.1.0" ipaddr.js "^2.1.0"
istanbul-lib-coverage "3.2.0" istanbul-lib-coverage "3.2.0"
jest-docblock "^29.7.0" jest-docblock "^29.7.0"
koalas "^1.0.2" koalas "^1.0.2"
limiter "1.1.5" limiter "1.1.5"
lodash.kebabcase "^4.1.1"
lodash.pick "^4.4.0"
lodash.sortby "^4.7.0" lodash.sortby "^4.7.0"
lodash.uniq "^4.5.0"
lru-cache "^7.14.0" lru-cache "^7.14.0"
methods "^1.1.2" methods "^1.1.2"
module-details-from-path "^1.0.3" module-details-from-path "^1.0.3"
@ -12082,10 +12079,10 @@ import-from@^3.0.0:
dependencies: dependencies:
resolve-from "^5.0.0" resolve-from "^5.0.0"
import-in-the-middle@^1.7.1: import-in-the-middle@^1.7.3:
version "1.7.2" version "1.7.3"
resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.7.2.tgz#31c44088271b50ecb9cacbdfb1e5732c802e0658" resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.7.3.tgz#ffa784cdd57a47d2b68d2e7dd33070ff06baee43"
integrity sha512-coz7AjRnPyKW36J6JX5Bjz1mcX7MX1H2XsEGseVcnXMdzsAbbAu0HBZhiAem+3SAmuZdi+p8OwoB2qUpTRgjOQ== integrity sha512-R2I11NRi0lI3jD2+qjqyVlVEahsejw7LDnYEbGb47QEFjczE3bZYsmWheCTQA+LFs2DzOQxR7Pms7naHW1V4bQ==
dependencies: dependencies:
acorn "^8.8.2" acorn "^8.8.2"
acorn-import-assertions "^1.9.0" acorn-import-assertions "^1.9.0"
@ -14481,11 +14478,6 @@ lodash.isstring@^4.0.1:
resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451" resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
integrity sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw== integrity sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==
lodash.kebabcase@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/lodash.kebabcase/-/lodash.kebabcase-4.1.1.tgz#8489b1cb0d29ff88195cceca448ff6d6cc295c36"
integrity sha512-N8XRTIMMqqDgSy4VLKPnJ/+hpGZN+PHQiJnSenYqPaVV/NCqEogTnAdZLQiGKhxX+JCs8waWq2t1XHWKOmlY8g==
lodash.keys@^4.2.0: lodash.keys@^4.2.0:
version "4.2.0" version "4.2.0"
resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-4.2.0.tgz#a08602ac12e4fb83f91fc1fb7a360a4d9ba35205" resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-4.2.0.tgz#a08602ac12e4fb83f91fc1fb7a360a4d9ba35205"
@ -14516,7 +14508,7 @@ lodash.once@^4.0.0:
resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg== integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==
lodash.pick@^4.0.0, lodash.pick@^4.4.0: lodash.pick@^4.0.0:
version "4.4.0" version "4.4.0"
resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3" resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
integrity sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q== integrity sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==