From 59c2f247a75206dac0749954578a87338ec91d9e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 29 Jun 2020 18:57:17 +0100 Subject: [PATCH] upload assets to s3 --- packages/server/package.json | 1 + packages/server/src/api/controllers/deploy.js | 90 ++++++++++++++++--- packages/server/src/api/routes/deploy.js | 3 +- .../server/src/middleware/authenticated.js | 7 ++ packages/server/yarn.lock | 78 +++++++++++++++- 5 files changed, 160 insertions(+), 19 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 07ae312bee..c10905436b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -46,6 +46,7 @@ "@budibase/core": "^0.0.32", "@koa/router": "^8.0.0", "@sendgrid/mail": "^7.1.1", + "aws-sdk": "^2.706.0", "bcryptjs": "^2.4.3", "dotenv": "^8.2.0", "electron": "^9.0.4", diff --git a/packages/server/src/api/controllers/deploy.js b/packages/server/src/api/controllers/deploy.js index d675940e81..a7ee2333f8 100644 --- a/packages/server/src/api/controllers/deploy.js +++ b/packages/server/src/api/controllers/deploy.js @@ -1,21 +1,83 @@ +const fs = require("fs") +const AWS = require("aws-sdk") const CouchDB = require("../../db") -const { resolve, join } = require("path") const { budibaseAppsDir, - budibaseTempDir, } = require("../../utilities/budibaseDir") -exports.deployApp = async function(ctx) { - // upload all the assets to s3 +const s3 = new AWS.S3({ + params: { + Bucket: process.env.BUDIBASE_APP_ASSETS_BUCKET + } +}) - // replicate the DB to the couchDB cluster in prod - const localDb = new CouchDB(ctx.user.instanceId) - const remoteDb = new CouchDB(`${process.env.COUCH_DB_URL}/${ctx.user.instanceId}`) - localDb.replicate.to(remoteDb) - .on("complete", () => { - console.log("Replication is complete") - }) - .on("error", () => { - console.error("Error replicating DB") - }); +async function uploadAppAssets({ clientId, appId }) { + const appAssetsPath = `${budibaseAppsDir()}/${appId}/public` + + const appPages = fs.readdirSync(appAssetsPath) + + const uploads = [] + + for (let page of appPages) { + for (let filename of fs.readdirSync(`${appAssetsPath}/${page}`)) { + const filePath = `${appAssetsPath}/${page}/${filename}` + const stat = await fs.lstatSync(filePath) + + // TODO: need to account for recursively traversing dirs + if (stat.isFile()) { + const fileBytes = fs.readFileSync(`${appAssetsPath}/${page}/${filename}`) + + const upload = s3.upload({ + Key: `${clientId}/${appId}/${page}/${filename}`, + Body: fileBytes + }).promise() + + uploads.push(upload) + } + } + } + + try { + await Promise.all(uploads) + } catch (err) { + console.error("Error uploading budibase app assets to s3", err) + throw err + } +} + +function replicateCouch(instanceId) { + return new Promise((resolve, reject) => { + const localDb = new CouchDB(instanceId) + const remoteDb = new CouchDB(`${process.env.COUCH_DB_REMOTE}/${instanceId}`) + + const replication = localDb.replicate.to(remoteDb); + + replication.on("complete", () => resolve()) + replication.on("error", err => reject(err)) + }); + +} + +exports.deployApp = async function(ctx) { + const { + user = { + instanceId: "test_instance_id", + clientId: "test123", + appId: "24602c068a2d494cb09cb48f44bfdd8f" + } + } = ctx; + + // TODO: This should probably be async - it could take a while + try { + // upload all the assets to s3 + await uploadAppAssets(user) + + // replicate the DB to the couchDB cluster in prod + await replicateCouch(user.instanceId); + + ctx.body = "Deployment Successful!" + + } catch (err) { + ctx.throw(err.status || 500, `Deployment Failed: ${err.message}`); + } } \ No newline at end of file diff --git a/packages/server/src/api/routes/deploy.js b/packages/server/src/api/routes/deploy.js index 2e444b93bd..4964d555e5 100644 --- a/packages/server/src/api/routes/deploy.js +++ b/packages/server/src/api/routes/deploy.js @@ -5,6 +5,7 @@ const { BUILDER } = require("../../utilities/accessLevels") const router = Router() -router.post("/:appId/deploy", authorized(BUILDER), controller.deployApp) +// router.post("/:appId/deploy", authorized(BUILDER), controller.deployApp) +router.post("/:appId/deploy", controller.deployApp) module.exports = router diff --git a/packages/server/src/middleware/authenticated.js b/packages/server/src/middleware/authenticated.js index 36e2776abe..4e2f6e2689 100644 --- a/packages/server/src/middleware/authenticated.js +++ b/packages/server/src/middleware/authenticated.js @@ -60,6 +60,13 @@ module.exports = async (ctx, next) => { await next() } +/** + * Return the full access level object either from constants + * or the database based on the access level ID passed. + * + * @param {*} instanceId - instanceId of the user + * @param {*} accessLevelId - the id of the users access level + */ const getAccessLevel = async (instanceId, accessLevelId) => { if ( accessLevelId === POWERUSER_LEVEL_ID || diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 47a7fbb723..92b1d87ba6 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -947,6 +947,21 @@ available-typed-arrays@^1.0.0, available-typed-arrays@^1.0.2: dependencies: array-filter "^1.0.0" +aws-sdk@^2.706.0: + version "2.706.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953" + integrity sha512-7GT+yrB5Wb/zOReRdv/Pzkb2Qt+hz6B/8FGMVaoysX3NryHvQUdz7EQWi5yhg9CxOjKxdw5lFwYSs69YlSp1KA== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -1169,6 +1184,15 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== +buffer@4.9.2: + version "4.9.2" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" + integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + buffer@^5.1.0, buffer@^5.5.0: version "5.6.0" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786" @@ -2304,6 +2328,11 @@ event-target-shim@^5.0.0: resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= + exec-sh@^0.3.2: version "0.3.4" resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5" @@ -2959,7 +2988,7 @@ iconv-lite@^0.5.1: dependencies: safer-buffer ">= 2.1.2 < 3" -ieee754@^1.1.4: +ieee754@1.1.13, ieee754@^1.1.4: version "1.1.13" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== @@ -3363,7 +3392,7 @@ isarray@0.0.1: resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= -isarray@1.0.0, isarray@~1.0.0: +isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= @@ -3809,7 +3838,7 @@ jest@^24.8.0: import-local "^2.0.0" jest-cli "^24.9.0" -jmespath@^0.15.0: +jmespath@0.15.0, jmespath@^0.15.0: version "0.15.0" resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= @@ -5321,6 +5350,11 @@ pump@^3.0.0: end-of-stream "^1.1.0" once "^1.3.1" +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= + punycode@^1.3.2: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" @@ -5353,6 +5387,11 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + quick-format-unescaped@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz#437a5ea1a0b61deb7605f8ab6a8fd3858dbeb701" @@ -5767,7 +5806,12 @@ sanitize-filename@^1.6.2, sanitize-filename@^1.6.3: dependencies: truncate-utf8-bytes "^1.0.0" -sax@^1.2.4: +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= + +sax@>=0.6.0, sax@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -6645,6 +6689,14 @@ url-parse-lax@^3.0.0: dependencies: prepend-http "^2.0.0" +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= + dependencies: + punycode "1.3.2" + querystring "0.2.0" + use@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" @@ -6670,6 +6722,11 @@ util.promisify@^1.0.0: has-symbols "^1.0.1" object.getownpropertydescriptors "^2.1.0" +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + uuid@3.3.3: version "3.3.3" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866" @@ -6896,6 +6953,19 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + "xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.2, xtend@~4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"