From 3dca82a5ff016598dca48cbe6da7e2b32dfc7081 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 7 Oct 2020 20:37:55 +0100 Subject: [PATCH 01/11] check that deployment is possible using lambda API --- .../server/src/api/controllers/deploy/aws.js | 12 +++++- .../src/api/controllers/deploy/index.js | 37 +++++++++++++++++-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index 2f89d97742..6b73751695 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -21,11 +21,21 @@ async function invalidateCDN(cfDistribution, appId) { .promise() } -exports.fetchTemporaryCredentials = async function() { +/** + * Verifies the users API key and + * Verifies that the deployment fits within the quota of the user, + * @param {String} instanceId - instanceId being deployed + * @param {String} appId - appId being deployed + * @param {quota} quota - current quota being changed with this application + */ +exports.verifyDeployment = async function({ instanceId, appId, quota }) { const response = await fetch(process.env.DEPLOYMENT_CREDENTIALS_URL, { method: "POST", body: JSON.stringify({ apiKey: process.env.BUDIBASE_API_KEY, + instanceId, + appId, + quota, }), }) diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index 34579b64c7..a155fbbf81 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -1,10 +1,17 @@ const CouchDB = require("pouchdb") const PouchDB = require("../../../db") -const { uploadAppAssets, fetchTemporaryCredentials } = require("./aws") +const { + uploadAppAssets, + verifyDeployment, + determineDeploymentAllowed, +} = require("./aws") +const { getRecordParams } = require("../../db/utils") function replicate(local, remote) { return new Promise((resolve, reject) => { - const replication = local.sync(remote) + const replication = local.sync(remote, { + retry: true, + }) replication.on("complete", () => resolve()) replication.on("error", err => reject(err)) @@ -31,13 +38,37 @@ async function replicateCouch({ instanceId, clientId, credentials }) { await Promise.all(replications) } +async function getCurrentInstanceQuota(instanceId) { + const db = new PouchDB(instanceId) + const records = await db.allDocs( + getRecordParams("", null, { + include_docs: true, + }) + ) + const existingRecords = records.rows.length + return { + records: existingRecords, + } +} + exports.deployApp = async function(ctx) { try { const clientAppLookupDB = new PouchDB("client_app_lookup") const { clientId } = await clientAppLookupDB.get(ctx.user.appId) + const instanceQuota = await getCurrentInstanceQuota(ctx.user.instanceId) + const credentials = await verifyDeployment({ + instanceId: ctx.user.instanceId, + appId: ctx.user.appId, + quota: instanceQuota, + }) + ctx.log.info(`Uploading assets for appID ${ctx.user.appId} assets to s3..`) - const credentials = await fetchTemporaryCredentials() + + if (credentials.errors) { + ctx.throw(500, credentials.errors) + return + } await uploadAppAssets({ clientId, From 39d2adb9e3f4e48be76ef04a5a0344812ea78e68 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 8 Oct 2020 10:56:32 +0100 Subject: [PATCH 02/11] hitting deployment success endpoint --- .../server/src/api/controllers/deploy/aws.js | 21 +++++++++++++++++++ .../src/api/controllers/deploy/index.js | 7 ++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index 6b73751695..b56d3c7788 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -21,6 +21,27 @@ async function invalidateCDN(cfDistribution, appId) { .promise() } +exports.updateDeploymentQuota = async function(quota) { + const response = await fetch( + `${process.env.DEPLOYMENT_CREDENTIALS_URL}/deploy/success`, + { + method: "POST", + body: JSON.stringify({ + apiKey: process.env.BUDIBASE_API_KEY, + quota, + }), + } + ) + + if (response.status !== 200) { + throw new Error(`Error updating deployment quota for app`) + } + + const json = await response.json() + + return json +} + /** * Verifies the users API key and * Verifies that the deployment fits within the quota of the user, diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index a155fbbf81..c5d1169962 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -3,7 +3,7 @@ const PouchDB = require("../../../db") const { uploadAppAssets, verifyDeployment, - determineDeploymentAllowed, + updateDeploymentQuota, } = require("./aws") const { getRecordParams } = require("../../db/utils") @@ -85,6 +85,11 @@ exports.deployApp = async function(ctx) { credentials: credentials.couchDbCreds, }) + const deployedInstanceQuota = await getCurrentInstanceQuota( + ctx.user.instanceId + ) + updateDeploymentQuota(deployedInstanceQuota) + ctx.body = { status: "SUCCESS", completed: Date.now(), From ee73c7f4187a98a82c79d8a7d52175e8cec05df7 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 8 Oct 2020 15:06:27 +0100 Subject: [PATCH 03/11] update deployment quota after deploy --- packages/server/src/api/controllers/deploy/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index c5d1169962..7f6e4588a5 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -46,8 +46,12 @@ async function getCurrentInstanceQuota(instanceId) { }) ) const existingRecords = records.rows.length + + const designDoc = await db.get("_design/database") + return { records: existingRecords, + views: Object.keys(designDoc.views).length, } } @@ -88,7 +92,7 @@ exports.deployApp = async function(ctx) { const deployedInstanceQuota = await getCurrentInstanceQuota( ctx.user.instanceId ) - updateDeploymentQuota(deployedInstanceQuota) + await updateDeploymentQuota(deployedInstanceQuota) ctx.body = { status: "SUCCESS", From 3994816c69e242af9e8b990af2f52449b403c4de Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 8 Oct 2020 20:23:58 +0100 Subject: [PATCH 04/11] tidy up --- .../server/src/api/controllers/deploy/aws.js | 66 ++++++++++--------- .../src/api/controllers/deploy/index.js | 14 ++-- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index b56d3c7788..f421e2f4fc 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -22,19 +22,20 @@ async function invalidateCDN(cfDistribution, appId) { } exports.updateDeploymentQuota = async function(quota) { - const response = await fetch( - `${process.env.DEPLOYMENT_CREDENTIALS_URL}/deploy/success`, - { - method: "POST", - body: JSON.stringify({ - apiKey: process.env.BUDIBASE_API_KEY, - quota, - }), - } - ) + const DEPLOYMENT_SUCCESS_URL = + process.env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success" + + console.log(DEPLOYMENT_SUCCESS_URL) + const response = await fetch(DEPLOYMENT_SUCCESS_URL, { + method: "POST", + body: JSON.stringify({ + apiKey: process.env.BUDIBASE_API_KEY, + quota, + }), + }) if (response.status !== 200) { - throw new Error(`Error updating deployment quota for app`) + throw new Error(`Error updating deployment quota for API Key`) } const json = await response.json() @@ -163,30 +164,33 @@ exports.uploadAppAssets = async function({ // Upload file attachments const db = new PouchDB(instanceId) - const fileUploads = await db.get("_local/fileuploads") - if (fileUploads) { - for (let file of fileUploads.uploads) { - if (file.uploaded) continue - - const attachmentUpload = prepareUploadForS3({ - file, - s3Key: `assets/${appId}/attachments/${file.processedFileName}`, - s3, - metadata: { accountId }, - }) - - uploads.push(attachmentUpload) - - // mark file as uploaded - file.uploaded = true - } - - db.put(fileUploads) + let fileUploads + try { + fileUploads = await db.get("_local/fileuploads") + } catch (err) { + fileUploads = { _id: "_local/fileuploads", uploads: [] } } + for (let file of fileUploads.uploads) { + if (file.uploaded) continue + + const attachmentUpload = prepareUploadForS3({ + file, + s3Key: `assets/${appId}/attachments/${file.processedFileName}`, + s3, + metadata: { accountId }, + }) + + uploads.push(attachmentUpload) + + // mark file as uploaded + file.uploaded = true + } + + db.put(fileUploads) + try { await Promise.all(uploads) - // TODO: update dynamoDB with a synopsis of the app deployment for historical purposes await invalidateCDN(cfDistribution, appId) } catch (err) { console.error("Error uploading budibase app assets to s3", err) diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index 7f6e4588a5..2b433140fe 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -5,13 +5,10 @@ const { verifyDeployment, updateDeploymentQuota, } = require("./aws") -const { getRecordParams } = require("../../db/utils") function replicate(local, remote) { return new Promise((resolve, reject) => { - const replication = local.sync(remote, { - retry: true, - }) + const replication = local.sync(remote) replication.on("complete", () => resolve()) replication.on("error", err => reject(err)) @@ -40,11 +37,10 @@ async function replicateCouch({ instanceId, clientId, credentials }) { async function getCurrentInstanceQuota(instanceId) { const db = new PouchDB(instanceId) - const records = await db.allDocs( - getRecordParams("", null, { - include_docs: true, - }) - ) + const records = await db.allDocs({ + startkey: "record:", + endkey: `record:\ufff0`, + }) const existingRecords = records.rows.length const designDoc = await db.get("_design/database") From 3715c2bf3676a669a074afdf233c3e693d723bde Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 8 Oct 2020 21:11:10 +0100 Subject: [PATCH 05/11] removing retry param --- .../pages/[application]/deploy/index.svelte | 7 +- packages/server/yarn.lock | 249 +----------------- 2 files changed, 9 insertions(+), 247 deletions(-) diff --git a/packages/builder/src/pages/[application]/deploy/index.svelte b/packages/builder/src/pages/[application]/deploy/index.svelte index de649a8dd2..39144df2bf 100644 --- a/packages/builder/src/pages/[application]/deploy/index.svelte +++ b/packages/builder/src/pages/[application]/deploy/index.svelte @@ -1,5 +1,5 @@