diff --git a/packages/server/src/api/controllers/auth.js b/packages/server/src/api/controllers/auth.js index b7a7c116b9..c16f76caca 100644 --- a/packages/server/src/api/controllers/auth.js +++ b/packages/server/src/api/controllers/auth.js @@ -3,7 +3,7 @@ const CouchDB = require("../../db") const ClientDb = require("../../db/clientDb") const bcrypt = require("../../utilities/bcrypt") const environment = require("../../environment") -const { apiKeyTable } = require("../../db/dynamoClient") +const { getAPIKey } = require("../../utilities/usageQuota") const { generateUserID } = require("../../db/utils") exports.authenticate = async ctx => { @@ -55,7 +55,7 @@ exports.authenticate = async ctx => { } // if in cloud add the user api key if (environment.CLOUD) { - payload.apiKey = await apiKeyTable.get({ primary: ctx.user.appId }) + payload.apiKey = getAPIKey(ctx.user.appId) } const token = jwt.sign(payload, ctx.config.jwtSecret, { diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js index 5414f3878c..92391da7e9 100644 --- a/packages/server/src/api/controllers/automation.js +++ b/packages/server/src/api/controllers/automation.js @@ -33,6 +33,7 @@ function cleanAutomationInputs(automation) { exports.create = async function(ctx) { const db = new CouchDB(ctx.user.instanceId) let automation = ctx.request.body + automation.appId = ctx.user.appId automation._id = generateAutomationID() @@ -54,6 +55,7 @@ exports.create = async function(ctx) { exports.update = async function(ctx) { const db = new CouchDB(ctx.user.instanceId) let automation = ctx.request.body + automation.appId = ctx.user.appId automation = cleanAutomationInputs(automation) const response = await db.put(automation) diff --git a/packages/server/src/api/routes/model.js b/packages/server/src/api/routes/model.js index 3c667df520..fe782d4cf5 100644 --- a/packages/server/src/api/routes/model.js +++ b/packages/server/src/api/routes/model.js @@ -1,7 +1,6 @@ const Router = require("@koa/router") const modelController = require("../controllers/model") const authorized = require("../../middleware/authorized") -const usage = require("../../middleware/usageQuota") const { BUILDER, READ_MODEL } = require("../../utilities/accessLevels") const router = Router() @@ -13,7 +12,7 @@ router authorized(READ_MODEL, ctx => ctx.params.id), modelController.find ) - .post("/api/models", authorized(BUILDER), usage, modelController.save) + .post("/api/models", authorized(BUILDER), modelController.save) .post( "/api/models/csv/validate", authorized(BUILDER), @@ -22,7 +21,6 @@ router .delete( "/api/models/:modelId/:revId", authorized(BUILDER), - usage, modelController.destroy ) diff --git a/packages/server/src/api/routes/static.js b/packages/server/src/api/routes/static.js index aa136a3d15..5c33900eca 100644 --- a/packages/server/src/api/routes/static.js +++ b/packages/server/src/api/routes/static.js @@ -4,6 +4,7 @@ const { budibaseTempDir } = require("../../utilities/budibaseDir") const env = require("../../environment") const authorized = require("../../middleware/authorized") const { BUILDER } = require("../../utilities/accessLevels") +const usage = require("../../middleware/usageQuota") const router = Router() @@ -28,7 +29,7 @@ router authorized(BUILDER), controller.performLocalFileProcessing ) - .post("/api/attachments/upload", controller.uploadFile) + .post("/api/attachments/upload", usage, controller.uploadFile) .get("/componentlibrary", controller.serveComponentLibrary) .get("/assets/:file*", controller.serveAppAsset) .get("/attachments/:file*", controller.serveAttachment) diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index 532943ea62..5289439e41 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -2,6 +2,7 @@ const Router = require("@koa/router") const controller = require("../controllers/user") const authorized = require("../../middleware/authorized") const { USER_MANAGEMENT, LIST_USERS } = require("../../utilities/accessLevels") +const usage = require("../../middleware/usageQuota") const router = Router() @@ -9,10 +10,11 @@ router .get("/api/users", authorized(LIST_USERS), controller.fetch) .get("/api/users/:username", authorized(USER_MANAGEMENT), controller.find) .put("/api/users/", authorized(USER_MANAGEMENT), controller.update) - .post("/api/users", authorized(USER_MANAGEMENT), controller.create) + .post("/api/users", authorized(USER_MANAGEMENT), usage, controller.create) .delete( "/api/users/:username", authorized(USER_MANAGEMENT), + usage, controller.destroy ) diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js index 2c88f6d19a..571e4494f1 100644 --- a/packages/server/src/api/routes/view.js +++ b/packages/server/src/api/routes/view.js @@ -3,6 +3,7 @@ const viewController = require("../controllers/view") const recordController = require("../controllers/record") const authorized = require("../../middleware/authorized") const { BUILDER, READ_VIEW } = require("../../utilities/accessLevels") +const usage = require("../../middleware/usageQuota") const router = Router() @@ -13,8 +14,13 @@ router recordController.fetchView ) .get("/api/views", authorized(BUILDER), viewController.fetch) - .delete("/api/views/:viewName", authorized(BUILDER), viewController.destroy) - .post("/api/views", authorized(BUILDER), viewController.save) + .delete( + "/api/views/:viewName", + authorized(BUILDER), + usage, + viewController.destroy + ) + .post("/api/views", authorized(BUILDER), usage, viewController.save) .post("/api/views/export", authorized(BUILDER), viewController.exportView) .get( "/api/views/export/download/:fileName", diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index e419985ce2..882f24391b 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -3,6 +3,7 @@ const actions = require("./actions") const environment = require("../environment") const workerFarm = require("worker-farm") const singleThread = require("./thread") +const { getAPIKey, update, Properties } = require("../utilities/usageQuota") let workers = workerFarm(require.resolve("./thread")) @@ -18,16 +19,32 @@ function runWorker(job) { }) } +async function updateQuota(automation) { + const appId = automation.appId + const apiKey = await getAPIKey(appId) + // this will fail, causing automation to escape if limits reached + await update(apiKey, Properties.AUTOMATION, 1) +} + /** * This module is built purely to kick off the worker farm and manage the inputs/outputs */ module.exports.init = function() { actions.init().then(() => { triggers.automationQueue.process(async job => { - if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { - await runWorker(job) - } else { - await singleThread(job) + try { + if (environment.CLOUD) { + await updateQuota(job.data.automation) + } + if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { + await runWorker(job) + } else { + await singleThread(job) + } + } catch (err) { + console.error( + `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}` + ) } }) }) diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 1d119f25c8..74df19a015 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,9 +1,5 @@ const CouchDB = require("../db") -const environment = require("../environment") -const { apiKeyTable } = require("../db/dynamoClient") - -// a normalised month in milliseconds -const QUOTA_RESET = 2592000000 +const usageQuota = require("../utilities/usageQuota") // currently only counting new writes and deletes const METHOD_MAP = { @@ -12,36 +8,32 @@ const METHOD_MAP = { } const DOMAIN_MAP = { - models: "model", - records: "record", + records: usageQuota.Properties.RECORD, + upload: usageQuota.Properties.UPLOAD, + views: usageQuota.Properties.VIEW, + users: usageQuota.Properties.USER, + // this will not be updated by endpoint calls + // instead it will be updated by triggers + automationRuns: usageQuota.Properties.AUTOMATION, } -function buildUpdateParams(key, property, usage) { - return { - primary: key, - condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now", - expression: "ADD #quota.#prop :usage", - names: { - "#quota": "usageQuota", - "#prop": property, - "#limits": "limits", - "#quotaReset": "quotaReset", - }, - values: { - ":usage": usage, - ":now": Date.now(), - }, +function getProperty(url) { + for (let domain of Object.keys(DOMAIN_MAP)) { + if (url.indexOf(domain) !== -1) { + return DOMAIN_MAP[domain] + } } } module.exports = async (ctx, next) => { const db = new CouchDB(ctx.user.instanceId) - const usage = METHOD_MAP[ctx.req.method] - const domainParts = ctx.req.url.split("/") - const property = DOMAIN_MAP[domainParts[domainParts.length - 1]] + let usage = METHOD_MAP[ctx.req.method] + const property = getProperty(ctx.req.url) + console.log(ctx.req.url) if (usage == null || property == null) { return next() } + console.log(`${usage} to ${property}`) // post request could be a save of a pre-existing entry if (ctx.request.body && ctx.request.body._id) { try { @@ -52,26 +44,17 @@ module.exports = async (ctx, next) => { return } } - // don't try validate in builder - if (!environment.CLOUD) { - return next() + // update usage for uploads to be the total size + if (property === usageQuota.Properties.UPLOAD) { + const files = + ctx.request.files.file.length > 1 + ? Array.from(ctx.request.files.file) + : [ctx.request.files.file] + usage = files.map(file => file.size).reduce((total, size) => total + size) } try { - await apiKeyTable.update(buildUpdateParams(ctx.apiKey, property, usage)) + await usageQuota.update(ctx.apiKey, property, usage) } catch (err) { - if (err.code !== "ConditionalCheckFailedException") { - // get the API key so we can check it - let apiKey = await apiKeyTable.get({ primary: ctx.apiKey }) - // we have infact breached the reset period - if (apiKey && apiKey.quotaReset >= Date.now()) { - // update the quota reset period and reset the values for all properties - apiKey.quotaReset = Date.now() + QUOTA_RESET - for (let prop of Object.keys(apiKey.usageQuota)) { - apiKey.usageQuota[prop] = 0 - } - await apiKeyTable.put({ item: apiKey }) - } - ctx.throw(403, `Resource limits have been reached`) - } + ctx.throw(403, err) } -} \ No newline at end of file +} diff --git a/packages/server/src/utilities/fileProcessor.js b/packages/server/src/utilities/fileProcessor.js index 3e580e9e37..734209733d 100644 --- a/packages/server/src/utilities/fileProcessor.js +++ b/packages/server/src/utilities/fileProcessor.js @@ -1,5 +1,5 @@ const fs = require("fs") -const sharp = require("sharp") +// const sharp = require("sharp") const fsPromises = fs.promises const FORMATS = { @@ -7,14 +7,14 @@ const FORMATS = { } async function processImage(file) { - const imgMeta = await sharp(file.path) - .resize(300) - .toFile(file.outputPath) - - return { - ...file, - ...imgMeta, - } + // const imgMeta = await sharp(file.path) + // .resize(300) + // .toFile(file.outputPath) + // + // return { + // ...file, + // ...imgMeta, + // } } async function process(file) { diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js new file mode 100644 index 0000000000..7ba0005213 --- /dev/null +++ b/packages/server/src/utilities/usageQuota.js @@ -0,0 +1,72 @@ +const environment = require("../environment") +const { apiKeyTable } = require("../db/dynamoClient") + +function buildUpdateParams(key, property, usage) { + return { + primary: key, + condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now", + expression: "ADD #quota.#prop :usage", + names: { + "#quota": "usageQuota", + "#prop": property, + "#limits": "limits", + "#quotaReset": "quotaReset", + }, + values: { + ":usage": usage, + ":now": Date.now(), + }, + } +} + +// a normalised month in milliseconds +const QUOTA_RESET = 2592000000 + +exports.Properties = { + RECORD: "records", + UPLOAD: "storage", + VIEW: "views", + USER: "users", + AUTOMATION: "automationRuns", +} + +exports.getAPIKey = async appId => { + return apiKeyTable.get({ primary: appId }) +} + +/** + * Given a specified API key this will add to the usage object for the specified property. + * @param {string} apiKey The API key which is to be updated. + * @param {string} property The property which is to be added to (within the nested usageQuota object). + * @param {number} usage The amount (this can be negative) to adjust the number by. + * @returns {Promise} When this completes the API key will now be up to date - the quota period may have + * also been reset after this call. + */ +exports.update = async (apiKey, property, usage) => { + // don't try validate in builder + if (!environment.CLOUD) { + return + } + try { + await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) + } catch (err) { + if (err.code !== "ConditionalCheckFailedException") { + // get the API key so we can check it + let apiKey = await apiKeyTable.get({ primary: apiKey }) + // we have infact breached the reset period + if (apiKey && apiKey.quotaReset >= Date.now()) { + // update the quota reset period and reset the values for all properties + apiKey.quotaReset = Date.now() + QUOTA_RESET + for (let prop of Object.keys(apiKey.usageQuota)) { + if (prop === property) { + apiKey.usageQuota[prop] = usage > 0 ? usage : 0 + } else { + apiKey.usageQuota[prop] = 0 + } + } + await apiKeyTable.put({ item: apiKey }) + } + throw "Resource limits have been reached" + } + } +}