From 103161c7a83eb3cb525d9856f5537c5c73c35e37 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Dec 2020 13:39:34 +0000 Subject: [PATCH] Expanding on deployment, attempting to hide all the quota stuff behind layers that we can more easily abstract, as well as hiding away any AWS specific features in the index of deployment. --- .../src/api/controllers/deploy/Deployment.js | 66 ++++++++++ .../deploy/{aws.js => awsDeploy.js} | 124 ++++++------------ .../src/api/controllers/deploy/index.js | 89 ++++--------- .../src/api/controllers/deploy/quota.js | 27 ++++ .../src/api/controllers/deploy/selfDeploy.js | 5 + .../src/api/controllers/deploy/utils.js | 31 +++++ .../src/api/controllers/static/index.js | 6 +- packages/server/src/utilities/index.js | 19 +++ 8 files changed, 214 insertions(+), 153 deletions(-) create mode 100644 packages/server/src/api/controllers/deploy/Deployment.js rename packages/server/src/api/controllers/deploy/{aws.js => awsDeploy.js} (61%) create mode 100644 packages/server/src/api/controllers/deploy/quota.js create mode 100644 packages/server/src/api/controllers/deploy/selfDeploy.js create mode 100644 packages/server/src/api/controllers/deploy/utils.js diff --git a/packages/server/src/api/controllers/deploy/Deployment.js b/packages/server/src/api/controllers/deploy/Deployment.js new file mode 100644 index 0000000000..609ce0e8fb --- /dev/null +++ b/packages/server/src/api/controllers/deploy/Deployment.js @@ -0,0 +1,66 @@ +const { getAppQuota } = require("./quota") +const env = require("../../../environment") + +/** + * This is used to pass around information about the deployment that is occurring + */ +class Deployment { + constructor(id, appId) { + this._id = id + this.appId = appId + } + + // purely so that we can do quota stuff outside the main deployment context + async init() { + if (!env.SELF_HOSTED) { + this.setQuota(await getAppQuota(this.appId)) + } + } + + setQuota(quota) { + this.quota = quota + } + + getQuota() { + return this.quota + } + + getAppId() { + return this.appId + } + + setVerification(verification) { + this.verification = verification + } + + getVerification() { + return this.verification + } + + setStatus(status, err = null) { + this.status = status + if (err) { + this.err = err + } + } + + getJSON() { + const obj = { + _id: this._id, + appId: this.appId, + status: this.status, + } + if (this.err) { + obj.err = this.err + } + if (this.verification && this.verification.cfDistribution) { + obj.cfDistribution = this.verification.cfDistribution + } + if (this.verification && this.verification.quota) { + obj.quota = this.verification.quota + } + return obj + } +} + +module.exports = Deployment diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/awsDeploy.js similarity index 61% rename from packages/server/src/api/controllers/deploy/aws.js rename to packages/server/src/api/controllers/deploy/awsDeploy.js index d478c4efde..5a36871ae8 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/awsDeploy.js @@ -1,57 +1,26 @@ const fs = require("fs") const { join } = require("../../../utilities/centralPath") -const AWS = require("aws-sdk") +const AwsDeploy = require("aws-sdk") const fetch = require("node-fetch") -const sanitize = require("sanitize-s3-objectkey") const { budibaseAppsDir } = require("../../../utilities/budibaseDir") const PouchDB = require("../../../db") const env = require("../../../environment") - -/** - * Finalises the deployment, updating the quota for the user API key - * The verification process returns the levels to update to. - * Calls the "deployment-success" lambda. - * @param {object} quota The usage quota levels returned from the verifyDeploy - * @returns {Promise} The usage has been updated against the user API key. - */ -exports.updateDeploymentQuota = async function(quota) { - const DEPLOYMENT_SUCCESS_URL = - env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success" - - const response = await fetch(DEPLOYMENT_SUCCESS_URL, { - method: "POST", - body: JSON.stringify({ - apiKey: env.BUDIBASE_API_KEY, - quota, - }), - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - }) - - if (response.status !== 200) { - throw new Error(`Error updating deployment quota for API Key`) - } - - return await response.json() -} +const { prepareUpload } = require("./utils") +const { walkDir } = require("../../../utilities") /** * Verifies the users API key and * Verifies that the deployment fits within the quota of the user * Links to the "check-api-key" lambda. - * @param {String} appId - appId being deployed - * @param {String} appId - appId being deployed - * @param {quota} quota - current quota being changed with this application + * @param {object} deployment - information about the active deployment, including the appId and quota. */ -exports.verifyDeployment = async function({ appId, quota }) { +exports.preDeployment = async function(deployment) { const response = await fetch(env.DEPLOYMENT_CREDENTIALS_URL, { method: "POST", body: JSON.stringify({ apiKey: env.BUDIBASE_API_KEY, - appId, - quota, + appId: deployment.getAppId(), + quota: deployment.getQuota(), }), }) @@ -68,7 +37,7 @@ exports.verifyDeployment = async function({ appId, quota }) { // set credentials here, means any time we're verified we're ready to go if (json.credentials) { - AWS.config.update({ + AwsDeploy.config.update({ accessKeyId: json.credentials.AccessKeyId, secretAccessKey: json.credentials.SecretAccessKey, sessionToken: json.credentials.SessionToken, @@ -78,57 +47,40 @@ exports.verifyDeployment = async function({ appId, quota }) { return json } -const CONTENT_TYPE_MAP = { - html: "text/html", - css: "text/css", - js: "application/javascript", -} - /** - * Recursively walk a directory tree and execute a callback on all files. - * @param {String} dirPath - Directory to traverse - * @param {Function} callback - callback to execute on files + * Finalises the deployment, updating the quota for the user API key + * The verification process returns the levels to update to. + * Calls the "deployment-success" lambda. + * @param {object} deployment information about the active deployment, including the quota info. + * @returns {Promise} The usage has been updated against the user API key. */ -function walkDir(dirPath, callback) { - for (let filename of fs.readdirSync(dirPath)) { - const filePath = `${dirPath}/${filename}` - const stat = fs.lstatSync(filePath) +exports.postDeployment = async function(deployment) { + const DEPLOYMENT_SUCCESS_URL = + env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success" - if (stat.isFile()) { - callback(filePath) - } else { - walkDir(filePath, callback) - } + const response = await fetch(DEPLOYMENT_SUCCESS_URL, { + method: "POST", + body: JSON.stringify({ + apiKey: env.BUDIBASE_API_KEY, + quota: deployment.getQuota(), + }), + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + }) + + if (response.status !== 200) { + throw new Error(`Error updating deployment quota for API Key`) } + + return await response.json() } -async function prepareUploadForS3({ s3Key, metadata, s3, file }) { - const extension = [...file.name.split(".")].pop() - const fileBytes = fs.readFileSync(file.path) - - const upload = await s3 - .upload({ - // windows filepaths need to be converted to forward slashes for s3 - Key: sanitize(s3Key).replace(/\\/g, "/"), - Body: fileBytes, - ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()], - Metadata: metadata, - }) - .promise() - - return { - size: file.size, - name: file.name, - extension, - url: upload.Location, - key: upload.Key, - } -} - -exports.prepareUploadForS3 = prepareUploadForS3 - -exports.uploadAppAssets = async function({ appId, bucket, accountId }) { - const s3 = new AWS.S3({ +exports.deploy = async function(deployment) { + const appId = deployment.getAppId() + const { bucket, accountId } = deployment.getVerification() + const s3 = new AwsDeploy.S3({ params: { Bucket: bucket, }, @@ -143,7 +95,7 @@ exports.uploadAppAssets = async function({ appId, bucket, accountId }) { for (let page of appPages) { // Upload HTML, CSS and JS for each page of the web app walkDir(join(appAssetsPath, page), function(filePath) { - const appAssetUpload = prepareUploadForS3({ + const appAssetUpload = prepareUpload({ file: { path: filePath, name: [...filePath.split("/")].pop(), @@ -168,7 +120,7 @@ exports.uploadAppAssets = async function({ appId, bucket, accountId }) { for (let file of fileUploads.uploads) { if (file.uploaded) continue - const attachmentUpload = prepareUploadForS3({ + const attachmentUpload = prepareUpload({ file, s3Key: `assets/${appId}/attachments/${file.processedFileName}`, s3, diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index 03f91430db..00cd7254bc 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -1,13 +1,12 @@ const CouchDB = require("pouchdb") const PouchDB = require("../../../db") -const { - uploadAppAssets, - verifyDeployment, - updateDeploymentQuota, -} = require("./aws") -const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils") const newid = require("../../../db/newid") const env = require("../../../environment") +const deployment = env.SELF_HOSTED + ? require("./selfDeploy") + : require("./awsDeploy") +const { deploy, preDeployment, postDeployment } = deployment +const Deployment = require("./Deployment") // the max time we can wait for an invalidation to complete before considering it failed const MAX_PENDING_TIME_MS = 30 * 60000 @@ -44,7 +43,9 @@ function replicate(local, remote) { }) } -async function replicateCouch({ appId, session }) { +async function replicateCouch(deployment) { + const appId = deployment.getAppId() + const { session } = deployment.getVerification() const localDb = new PouchDB(appId) const remoteDb = new CouchDB(`${env.DEPLOYMENT_DB_URL}/${appId}`, { fetch: function(url, opts) { @@ -56,33 +57,10 @@ async function replicateCouch({ appId, session }) { return replicate(localDb, remoteDb) } -async function getCurrentInstanceQuota(appId) { - const db = new PouchDB(appId) - - const rows = await db.allDocs({ - startkey: DocumentTypes.ROW + SEPARATOR, - endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX, - }) - - const users = await db.allDocs({ - startkey: DocumentTypes.USER + SEPARATOR, - endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX, - }) - - const existingRows = rows.rows.length - const existingUsers = users.rows.length - - const designDoc = await db.get("_design/database") - - return { - rows: existingRows, - users: existingUsers, - views: Object.keys(designDoc.views).length, - } -} - async function storeLocalDeploymentHistory(deployment) { - const db = new PouchDB(deployment.appId) + const appId = deployment.getAppId() + const deploymentJSON = deployment.getJSON() + const db = new PouchDB(appId) let deploymentDoc try { @@ -91,7 +69,7 @@ async function storeLocalDeploymentHistory(deployment) { deploymentDoc = { _id: "_local/deployments", history: {} } } - const deploymentId = deployment._id || newid() + const deploymentId = deploymentJSON._id || newid() // first time deployment if (!deploymentDoc.history[deploymentId]) @@ -99,7 +77,7 @@ async function storeLocalDeploymentHistory(deployment) { deploymentDoc.history[deploymentId] = { ...deploymentDoc.history[deploymentId], - ...deployment, + ...deploymentJSON, updatedAt: Date.now(), } @@ -111,43 +89,26 @@ async function storeLocalDeploymentHistory(deployment) { } async function deployApp({ appId, deploymentId }) { + const deployment = new Deployment(deploymentId, appId) try { - const instanceQuota = await getCurrentInstanceQuota(appId) - const verification = await verifyDeployment({ - appId, - quota: instanceQuota, - }) + await deployment.init() + deployment.setVerification(await preDeployment(deployment)) - console.log(`Uploading assets for appID ${appId} assets to s3..`) + console.log(`Uploading assets for appID ${appId}..`) - await uploadAppAssets({ - appId, - ...verification, - }) + await deploy(deployment) // replicate the DB to the couchDB cluster in prod console.log("Replicating local PouchDB to remote..") - await replicateCouch({ - appId, - session: verification.couchDbSession, - }) + await replicateCouch(deployment) - await updateDeploymentQuota(verification.quota) + await postDeployment(deployment) - await storeLocalDeploymentHistory({ - _id: deploymentId, - appId, - cfDistribution: verification.cfDistribution, - quota: verification.quota, - status: DeploymentStatus.SUCCESS, - }) + deployment.setStatus(DeploymentStatus.SUCCESS) + await storeLocalDeploymentHistory(deployment) } catch (err) { - await storeLocalDeploymentHistory({ - _id: deploymentId, - appId, - status: DeploymentStatus.FAILURE, - err: err.message, - }) + deployment.setStatus(DeploymentStatus.FAILURE, err.message) + await storeLocalDeploymentHistory(deployment) throw new Error(`Deployment Failed: ${err.message}`) } } @@ -188,7 +149,7 @@ exports.deployApp = async function(ctx) { status: DeploymentStatus.PENDING, }) - deployApp({ + await deployApp({ ...ctx.user, deploymentId: deployment._id, }) diff --git a/packages/server/src/api/controllers/deploy/quota.js b/packages/server/src/api/controllers/deploy/quota.js new file mode 100644 index 0000000000..4e4c869670 --- /dev/null +++ b/packages/server/src/api/controllers/deploy/quota.js @@ -0,0 +1,27 @@ +const PouchDB = require("../../../db") +const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils") + +exports.getAppQuota = async function(appId) { + const db = new PouchDB(appId) + + const rows = await db.allDocs({ + startkey: DocumentTypes.ROW + SEPARATOR, + endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX, + }) + + const users = await db.allDocs({ + startkey: DocumentTypes.USER + SEPARATOR, + endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX, + }) + + const existingRows = rows.rows.length + const existingUsers = users.rows.length + + const designDoc = await db.get("_design/database") + + return { + rows: existingRows, + users: existingUsers, + views: Object.keys(designDoc.views).length, + } +} diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js new file mode 100644 index 0000000000..ad81b6943a --- /dev/null +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -0,0 +1,5 @@ +exports.preDeployment = async function(deployment) {} + +exports.postDeployment = async function(deployment) {} + +exports.deploy = async function(deployment) {} diff --git a/packages/server/src/api/controllers/deploy/utils.js b/packages/server/src/api/controllers/deploy/utils.js new file mode 100644 index 0000000000..75fe3133ba --- /dev/null +++ b/packages/server/src/api/controllers/deploy/utils.js @@ -0,0 +1,31 @@ +const fs = require("fs") +const sanitize = require("sanitize-s3-objectkey") + +const CONTENT_TYPE_MAP = { + html: "text/html", + css: "text/css", + js: "application/javascript", +} + +exports.prepareUpload = async function({ s3Key, metadata, s3, file }) { + const extension = [...file.name.split(".")].pop() + const fileBytes = fs.readFileSync(file.path) + + const upload = await s3 + .upload({ + // windows file paths need to be converted to forward slashes for s3 + Key: sanitize(s3Key).replace(/\\/g, "/"), + Body: fileBytes, + ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()], + Metadata: metadata, + }) + .promise() + + return { + size: file.size, + name: file.name, + extension, + url: upload.Location, + key: upload.Key, + } +} diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index b048bbd9fe..a510291b41 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -6,7 +6,7 @@ const fetch = require("node-fetch") const fs = require("fs-extra") const uuid = require("uuid") const AWS = require("aws-sdk") -const { prepareUploadForS3 } = require("../deploy/aws") +const { prepareUpload } = require("../deploy/utils") const handlebars = require("handlebars") const { budibaseAppsDir, @@ -54,7 +54,7 @@ exports.uploadFile = async function(ctx) { const fileExtension = [...file.name.split(".")].pop() const processedFileName = `${uuid.v4()}.${fileExtension}` - return prepareUploadForS3({ + return prepareUpload({ file, s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`, s3, @@ -235,5 +235,5 @@ exports.serveComponentLibrary = async function(ctx) { return } - await send(ctx, "/index.js", { root: componentLibraryPath }) + await send(ctx, "/awsDeploy.js", { root: componentLibraryPath }) } diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index cde10b6b62..9fc3863118 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -1,5 +1,6 @@ const env = require("../environment") const { DocumentTypes, SEPARATOR } = require("../db/utils") +const fs = require("fs") const APP_PREFIX = DocumentTypes.APP + SEPARATOR @@ -74,3 +75,21 @@ exports.setCookie = (ctx, name, value) => { exports.isClient = ctx => { return ctx.headers["x-budibase-type"] === "client" } + +/** + * Recursively walk a directory tree and execute a callback on all files. + * @param {String} dirPath - Directory to traverse + * @param {Function} callback - callback to execute on files + */ +exports.walkDir = (dirPath, callback) => { + for (let filename of fs.readdirSync(dirPath)) { + const filePath = `${dirPath}/${filename}` + const stat = fs.lstatSync(filePath) + + if (stat.isFile()) { + callback(filePath) + } else { + exports.walkDir(filePath, callback) + } + } +}