upload assets to s3
This commit is contained in:
Martin McKeaveney
parent
ca7a5a1ff1
commit
59c2f247a7
|
|
@ -46,6 +46,7 @@
|
|||
"@budibase/core": "^0.0.32",
|
||||
"@koa/router": "^8.0.0",
|
||||
"@sendgrid/mail": "^7.1.1",
|
||||
"aws-sdk": "^2.706.0",
|
||||
"bcryptjs": "^2.4.3",
|
||||
"dotenv": "^8.2.0",
|
||||
"electron": "^9.0.4",
|
||||
|
|
|
|||
|
|
@ -1,21 +1,83 @@
|
|||
const fs = require("fs")
|
||||
const AWS = require("aws-sdk")
|
||||
const CouchDB = require("../../db")
|
||||
const { resolve, join } = require("path")
|
||||
const {
|
||||
budibaseAppsDir,
|
||||
budibaseTempDir,
|
||||
} = require("../../utilities/budibaseDir")
|
||||
|
||||
exports.deployApp = async function(ctx) {
|
||||
// upload all the assets to s3
|
||||
const s3 = new AWS.S3({
|
||||
params: {
|
||||
Bucket: process.env.BUDIBASE_APP_ASSETS_BUCKET
|
||||
}
|
||||
})
|
||||
|
||||
// replicate the DB to the couchDB cluster in prod
|
||||
const localDb = new CouchDB(ctx.user.instanceId)
|
||||
const remoteDb = new CouchDB(`${process.env.COUCH_DB_URL}/${ctx.user.instanceId}`)
|
||||
localDb.replicate.to(remoteDb)
|
||||
.on("complete", () => {
|
||||
console.log("Replication is complete")
|
||||
})
|
||||
.on("error", () => {
|
||||
console.error("Error replicating DB")
|
||||
});
|
||||
async function uploadAppAssets({ clientId, appId }) {
|
||||
const appAssetsPath = `${budibaseAppsDir()}/${appId}/public`
|
||||
|
||||
const appPages = fs.readdirSync(appAssetsPath)
|
||||
|
||||
const uploads = []
|
||||
|
||||
for (let page of appPages) {
|
||||
for (let filename of fs.readdirSync(`${appAssetsPath}/${page}`)) {
|
||||
const filePath = `${appAssetsPath}/${page}/${filename}`
|
||||
const stat = await fs.lstatSync(filePath)
|
||||
|
||||
// TODO: need to account for recursively traversing dirs
|
||||
if (stat.isFile()) {
|
||||
const fileBytes = fs.readFileSync(`${appAssetsPath}/${page}/${filename}`)
|
||||
|
||||
const upload = s3.upload({
|
||||
Key: `${clientId}/${appId}/${page}/${filename}`,
|
||||
Body: fileBytes
|
||||
}).promise()
|
||||
|
||||
uploads.push(upload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await Promise.all(uploads)
|
||||
} catch (err) {
|
||||
console.error("Error uploading budibase app assets to s3", err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
function replicateCouch(instanceId) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const localDb = new CouchDB(instanceId)
|
||||
const remoteDb = new CouchDB(`${process.env.COUCH_DB_REMOTE}/${instanceId}`)
|
||||
|
||||
const replication = localDb.replicate.to(remoteDb);
|
||||
|
||||
replication.on("complete", () => resolve())
|
||||
replication.on("error", err => reject(err))
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
exports.deployApp = async function(ctx) {
|
||||
const {
|
||||
user = {
|
||||
instanceId: "test_instance_id",
|
||||
clientId: "test123",
|
||||
appId: "24602c068a2d494cb09cb48f44bfdd8f"
|
||||
}
|
||||
} = ctx;
|
||||
|
||||
// TODO: This should probably be async - it could take a while
|
||||
try {
|
||||
// upload all the assets to s3
|
||||
await uploadAppAssets(user)
|
||||
|
||||
// replicate the DB to the couchDB cluster in prod
|
||||
await replicateCouch(user.instanceId);
|
||||
|
||||
ctx.body = "Deployment Successful!"
|
||||
|
||||
} catch (err) {
|
||||
ctx.throw(err.status || 500, `Deployment Failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ const { BUILDER } = require("../../utilities/accessLevels")
|
|||
|
||||
const router = Router()
|
||||
|
||||
router.post("/:appId/deploy", authorized(BUILDER), controller.deployApp)
|
||||
// router.post("/:appId/deploy", authorized(BUILDER), controller.deployApp)
|
||||
router.post("/:appId/deploy", controller.deployApp)
|
||||
|
||||
module.exports = router
|
||||
|
|
|
|||
|
|
@ -60,6 +60,13 @@ module.exports = async (ctx, next) => {
|
|||
await next()
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the full access level object either from constants
|
||||
* or the database based on the access level ID passed.
|
||||
*
|
||||
* @param {*} instanceId - instanceId of the user
|
||||
* @param {*} accessLevelId - the id of the users access level
|
||||
*/
|
||||
const getAccessLevel = async (instanceId, accessLevelId) => {
|
||||
if (
|
||||
accessLevelId === POWERUSER_LEVEL_ID ||
|
||||
|
|
|
|||
|
|
@ -947,6 +947,21 @@ available-typed-arrays@^1.0.0, available-typed-arrays@^1.0.2:
|
|||
dependencies:
|
||||
array-filter "^1.0.0"
|
||||
|
||||
aws-sdk@^2.706.0:
|
||||
version "2.706.0"
|
||||
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953"
|
||||
integrity sha512-7GT+yrB5Wb/zOReRdv/Pzkb2Qt+hz6B/8FGMVaoysX3NryHvQUdz7EQWi5yhg9CxOjKxdw5lFwYSs69YlSp1KA==
|
||||
dependencies:
|
||||
buffer "4.9.2"
|
||||
events "1.1.1"
|
||||
ieee754 "1.1.13"
|
||||
jmespath "0.15.0"
|
||||
querystring "0.2.0"
|
||||
sax "1.2.1"
|
||||
url "0.10.3"
|
||||
uuid "3.3.2"
|
||||
xml2js "0.4.19"
|
||||
|
||||
aws-sign2@~0.7.0:
|
||||
version "0.7.0"
|
||||
resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
|
||||
|
|
@ -1169,6 +1184,15 @@ buffer-from@^1.0.0:
|
|||
resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
|
||||
integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
|
||||
|
||||
buffer@4.9.2:
|
||||
version "4.9.2"
|
||||
resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
|
||||
integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==
|
||||
dependencies:
|
||||
base64-js "^1.0.2"
|
||||
ieee754 "^1.1.4"
|
||||
isarray "^1.0.0"
|
||||
|
||||
buffer@^5.1.0, buffer@^5.5.0:
|
||||
version "5.6.0"
|
||||
resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786"
|
||||
|
|
@ -2304,6 +2328,11 @@ event-target-shim@^5.0.0:
|
|||
resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
|
||||
integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==
|
||||
|
||||
events@1.1.1:
|
||||
version "1.1.1"
|
||||
resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
|
||||
integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
|
||||
|
||||
exec-sh@^0.3.2:
|
||||
version "0.3.4"
|
||||
resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5"
|
||||
|
|
@ -2959,7 +2988,7 @@ iconv-lite@^0.5.1:
|
|||
dependencies:
|
||||
safer-buffer ">= 2.1.2 < 3"
|
||||
|
||||
ieee754@^1.1.4:
|
||||
ieee754@1.1.13, ieee754@^1.1.4:
|
||||
version "1.1.13"
|
||||
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
|
||||
integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
|
||||
|
|
@ -3363,7 +3392,7 @@ isarray@0.0.1:
|
|||
resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
|
||||
integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=
|
||||
|
||||
isarray@1.0.0, isarray@~1.0.0:
|
||||
isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
|
||||
integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
|
||||
|
|
@ -3809,7 +3838,7 @@ jest@^24.8.0:
|
|||
import-local "^2.0.0"
|
||||
jest-cli "^24.9.0"
|
||||
|
||||
jmespath@^0.15.0:
|
||||
jmespath@0.15.0, jmespath@^0.15.0:
|
||||
version "0.15.0"
|
||||
resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
|
||||
integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
|
||||
|
|
@ -5321,6 +5350,11 @@ pump@^3.0.0:
|
|||
end-of-stream "^1.1.0"
|
||||
once "^1.3.1"
|
||||
|
||||
punycode@1.3.2:
|
||||
version "1.3.2"
|
||||
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
|
||||
integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=
|
||||
|
||||
punycode@^1.3.2:
|
||||
version "1.4.1"
|
||||
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
|
||||
|
|
@ -5353,6 +5387,11 @@ qs@~6.5.2:
|
|||
resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
|
||||
integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
|
||||
|
||||
querystring@0.2.0:
|
||||
version "0.2.0"
|
||||
resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
|
||||
integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
|
||||
|
||||
quick-format-unescaped@^4.0.1:
|
||||
version "4.0.1"
|
||||
resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz#437a5ea1a0b61deb7605f8ab6a8fd3858dbeb701"
|
||||
|
|
@ -5767,7 +5806,12 @@ sanitize-filename@^1.6.2, sanitize-filename@^1.6.3:
|
|||
dependencies:
|
||||
truncate-utf8-bytes "^1.0.0"
|
||||
|
||||
sax@^1.2.4:
|
||||
sax@1.2.1:
|
||||
version "1.2.1"
|
||||
resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
|
||||
integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
|
||||
|
||||
sax@>=0.6.0, sax@^1.2.4:
|
||||
version "1.2.4"
|
||||
resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
|
||||
integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
|
||||
|
|
@ -6645,6 +6689,14 @@ url-parse-lax@^3.0.0:
|
|||
dependencies:
|
||||
prepend-http "^2.0.0"
|
||||
|
||||
url@0.10.3:
|
||||
version "0.10.3"
|
||||
resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"
|
||||
integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=
|
||||
dependencies:
|
||||
punycode "1.3.2"
|
||||
querystring "0.2.0"
|
||||
|
||||
use@^3.1.0:
|
||||
version "3.1.1"
|
||||
resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"
|
||||
|
|
@ -6670,6 +6722,11 @@ util.promisify@^1.0.0:
|
|||
has-symbols "^1.0.1"
|
||||
object.getownpropertydescriptors "^2.1.0"
|
||||
|
||||
uuid@3.3.2:
|
||||
version "3.3.2"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
|
||||
integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
|
||||
|
||||
uuid@3.3.3:
|
||||
version "3.3.3"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866"
|
||||
|
|
@ -6896,6 +6953,19 @@ xml-name-validator@^3.0.0:
|
|||
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
|
||||
integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
|
||||
|
||||
xml2js@0.4.19:
|
||||
version "0.4.19"
|
||||
resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
|
||||
integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
|
||||
dependencies:
|
||||
sax ">=0.6.0"
|
||||
xmlbuilder "~9.0.1"
|
||||
|
||||
xmlbuilder@~9.0.1:
|
||||
version "9.0.7"
|
||||
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
|
||||
integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=
|
||||
|
||||
"xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.2, xtend@~4.0.0:
|
||||
version "4.0.2"
|
||||
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
|
||||
|
|
|
|||
Loading…
Reference in a new issue