From 4a2028c354a41845c7a7f145932ad808f136266e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 22:40:14 +0100 Subject: [PATCH 01/10] usageQuota middleware writing to couch --- packages/server/src/api/routes/application.js | 5 +- .../server/src/automations/steps/createRow.js | 4 +- packages/server/src/middleware/usageQuota.js | 25 ++-- packages/server/src/utilities/usageQuota.js | 110 +++++------------- .../server/src/utilities/usageQuota.old.js | 105 +++++++++++++++++ .../src/api/controllers/global/users.js | 24 ++++ 6 files changed, 177 insertions(+), 96 deletions(-) create mode 100644 packages/server/src/utilities/usageQuota.old.js diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js index c1d39acbd5..ef4aacf708 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.js @@ -2,11 +2,12 @@ const Router = require("@koa/router") const controller = require("../controllers/application") const authorized = require("../../middleware/authorized") const { BUILDER } = require("@budibase/auth/permissions") +const usage = require("../../middleware/usageQuota") const router = Router() router - .post("/api/applications", authorized(BUILDER), controller.create) + .post("/api/applications", authorized(BUILDER), usage, controller.create) .get("/api/applications/:appId/definition", controller.fetchAppDefinition) .get("/api/applications", controller.fetch) .get("/api/applications/:appId/appPackage", controller.fetchAppPackage) @@ -21,6 +22,6 @@ router authorized(BUILDER), controller.revertClient ) - .delete("/api/applications/:appId", authorized(BUILDER), controller.delete) + .delete("/api/applications/:appId", authorized(BUILDER), usage, controller.delete) module.exports = router diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index 9033004578..41e775b3de 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -60,7 +60,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, apiKey, emitter }) { +exports.run = async function ({ inputs, appId, tenantId, emitter }) { if (inputs.row == null || inputs.row.tableId == null) { return { success: false, @@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { inputs.row ) if (env.USE_QUOTAS) { - await usage.update(apiKey, usage.Properties.ROW, 1) + await usage.update(tenantId, usage.Properties.ROW, 1) } await rowController.save(ctx) return { diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 4647878721..4ad1092f6c 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -13,6 +13,7 @@ const DOMAIN_MAP = { upload: usageQuota.Properties.UPLOAD, views: usageQuota.Properties.VIEW, users: usageQuota.Properties.USER, + applications: usageQuota.Properties.APPS, // this will not be updated by endpoint calls // instead it will be updated by triggerInfo automationRuns: usageQuota.Properties.AUTOMATION, @@ -28,9 +29,9 @@ function getProperty(url) { module.exports = async (ctx, next) => { // if in development or a self hosted cloud usage quotas should not be executed - if (env.isDev() || env.SELF_HOSTED) { - return next() - } + // if (env.isDev() || env.SELF_HOSTED) { + // return next() + // } const db = new CouchDB(ctx.appId) let usage = METHOD_MAP[ctx.req.method] @@ -49,17 +50,17 @@ module.exports = async (ctx, next) => { } // update usage for uploads to be the total size - if (property === usageQuota.Properties.UPLOAD) { - const files = - ctx.request.files.file.length > 1 - ? Array.from(ctx.request.files.file) - : [ctx.request.files.file] - usage = files.map(file => file.size).reduce((total, size) => total + size) - } + // if (property === usageQuota.Properties.UPLOAD) { + // const files = + // ctx.request.files.file.length > 1 + // ? Array.from(ctx.request.files.file) + // : [ctx.request.files.file] + // usage = files.map(file => file.size).reduce((total, size) => total + size) + // } try { - await usageQuota.update(ctx.auth.apiKey, property, usage) + await usageQuota.update(ctx.user.tenantId, property, usage) return next() } catch (err) { - ctx.throw(403, err) + ctx.throw(400, err) } } diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index bfe71a4093..502fc4cad2 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,41 +1,6 @@ + const env = require("../environment") -const { apiKeyTable } = require("../db/dynamoClient") - -const DEFAULT_USAGE = { - rows: 0, - storage: 0, - views: 0, - automationRuns: 0, - users: 0, -} - -const DEFAULT_PLAN = { - rows: 1000, - // 1 GB - storage: 8589934592, - views: 10, - automationRuns: 100, - users: 10000, -} - -function buildUpdateParams(key, property, usage) { - return { - primary: key, - condition: - "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", - expression: "ADD #quota.#prop :usage", - names: { - "#quota": "usageQuota", - "#prop": property, - "#limits": "usageLimits", - "#quotaReset": "quotaReset", - }, - values: { - ":usage": usage, - ":now": Date.now(), - }, - } -} +const { getGlobalDB } = require("@budibase/auth/tenancy") function getNewQuotaReset() { return Date.now() + 2592000000 @@ -47,59 +12,44 @@ exports.Properties = { VIEW: "views", USER: "users", AUTOMATION: "automationRuns", -} - -exports.getAPIKey = async appId => { - if (!env.USE_QUOTAS) { - return { apiKey: null } - } - return apiKeyTable.get({ primary: appId }) + APPS: "apps" } /** - * Given a specified API key this will add to the usage object for the specified property. - * @param {string} apiKey The API key which is to be updated. + * Given a specified tenantId this will add to the usage object for the specified property. + * @param {string} tenantId The tenant to update the usage quotas for. * @param {string} property The property which is to be added to (within the nested usageQuota object). * @param {number} usage The amount (this can be negative) to adjust the number by. * @returns {Promise} When this completes the API key will now be up to date - the quota period may have * also been reset after this call. */ -exports.update = async (apiKey, property, usage) => { - if (!env.USE_QUOTAS) { - return - } +exports.update = async (tenantId, property, usage) => { + // if (!env.USE_QUOTAS) { + // return + // } try { - await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) + const db = getGlobalDB() + const quota = await db.get("usage_quota") + // TODO: check if the quota needs reset + if (Date.now() >= quota.quotaReset) { + quota.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(quota.usageQuota)) { + quota.usageQuota[prop] = 0 + } + } + + // increment the quota + quota.usageQuota[property] += usage + + if (quota.usageQuota[property] >= quota.usageLimits[property]) { + throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`) + } + + // update the usage quotas + await db.put(quota) } catch (err) { - // conditional check means the condition failed, need to check why - if (err.code === "ConditionalCheckFailedException") { - // get the API key so we can check it - const keyObj = await apiKeyTable.get({ primary: apiKey }) - // the usage quota or usage limits didn't exist - if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { - keyObj.usageQuota = - keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota - keyObj.usageLimits = - keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits - keyObj.quotaReset = getNewQuotaReset() - await apiKeyTable.put({ item: keyObj }) - return - } - // we have in fact breached the reset period - else if (keyObj && keyObj.quotaReset <= Date.now()) { - // update the quota reset period and reset the values for all properties - keyObj.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(keyObj.usageQuota)) { - if (prop === property) { - keyObj.usageQuota[prop] = usage > 0 ? usage : 0 - } else { - keyObj.usageQuota[prop] = 0 - } - } - await apiKeyTable.put({ item: keyObj }) - return - } - } + console.error(`Error updating usage quotas for ${property}`, err) throw err } + } diff --git a/packages/server/src/utilities/usageQuota.old.js b/packages/server/src/utilities/usageQuota.old.js new file mode 100644 index 0000000000..bfe71a4093 --- /dev/null +++ b/packages/server/src/utilities/usageQuota.old.js @@ -0,0 +1,105 @@ +const env = require("../environment") +const { apiKeyTable } = require("../db/dynamoClient") + +const DEFAULT_USAGE = { + rows: 0, + storage: 0, + views: 0, + automationRuns: 0, + users: 0, +} + +const DEFAULT_PLAN = { + rows: 1000, + // 1 GB + storage: 8589934592, + views: 10, + automationRuns: 100, + users: 10000, +} + +function buildUpdateParams(key, property, usage) { + return { + primary: key, + condition: + "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", + expression: "ADD #quota.#prop :usage", + names: { + "#quota": "usageQuota", + "#prop": property, + "#limits": "usageLimits", + "#quotaReset": "quotaReset", + }, + values: { + ":usage": usage, + ":now": Date.now(), + }, + } +} + +function getNewQuotaReset() { + return Date.now() + 2592000000 +} + +exports.Properties = { + ROW: "rows", + UPLOAD: "storage", + VIEW: "views", + USER: "users", + AUTOMATION: "automationRuns", +} + +exports.getAPIKey = async appId => { + if (!env.USE_QUOTAS) { + return { apiKey: null } + } + return apiKeyTable.get({ primary: appId }) +} + +/** + * Given a specified API key this will add to the usage object for the specified property. + * @param {string} apiKey The API key which is to be updated. + * @param {string} property The property which is to be added to (within the nested usageQuota object). + * @param {number} usage The amount (this can be negative) to adjust the number by. + * @returns {Promise} When this completes the API key will now be up to date - the quota period may have + * also been reset after this call. + */ +exports.update = async (apiKey, property, usage) => { + if (!env.USE_QUOTAS) { + return + } + try { + await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) + } catch (err) { + // conditional check means the condition failed, need to check why + if (err.code === "ConditionalCheckFailedException") { + // get the API key so we can check it + const keyObj = await apiKeyTable.get({ primary: apiKey }) + // the usage quota or usage limits didn't exist + if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { + keyObj.usageQuota = + keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota + keyObj.usageLimits = + keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits + keyObj.quotaReset = getNewQuotaReset() + await apiKeyTable.put({ item: keyObj }) + return + } + // we have in fact breached the reset period + else if (keyObj && keyObj.quotaReset <= Date.now()) { + // update the quota reset period and reset the values for all properties + keyObj.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(keyObj.usageQuota)) { + if (prop === property) { + keyObj.usageQuota[prop] = usage > 0 ? usage : 0 + } else { + keyObj.usageQuota[prop] = 0 + } + } + await apiKeyTable.put({ item: keyObj }) + return + } + } + throw err + } +} diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 1375240f34..b75c72290d 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -18,6 +18,7 @@ const { tryAddTenant, updateTenantId, } = require("@budibase/auth/tenancy") +const env = require("../../../environment") const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -138,6 +139,29 @@ exports.adminUser = async ctx => { include_docs: true, }) ) + + // write usage quotas for cloud + // if (!env.SELF_HOSTED) { + await db.post({ + _id: "usage_quota", + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + // storage: 1000, + // users: 10 + }, + }) + // } if (response.rows.some(row => row.doc.admin)) { ctx.throw( From c49b88b57aedde3588f8b96cf7831953ded323d2 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 23:25:25 +0100 Subject: [PATCH 02/10] automation runs quotas --- packages/server/src/api/routes/application.js | 7 +- packages/server/src/api/routes/user.js | 5 +- .../server/src/automations/steps/createRow.js | 4 +- .../server/src/automations/steps/deleteRow.js | 4 +- packages/server/src/automations/thread.js | 7 +- packages/server/src/integrations/mysql.ts | 6 +- packages/server/src/integrations/postgres.ts | 18 +- packages/server/src/integrations/utils.ts | 6 +- packages/server/src/middleware/usageQuota.js | 2 +- packages/server/src/utilities/usageQuota.js | 45 ++-- .../server/src/utilities/usageQuota.old.js | 196 +++++++++--------- .../src/api/controllers/global/users.js | 43 ++-- 12 files changed, 184 insertions(+), 159 deletions(-) diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js index ef4aacf708..4d67a0f4f4 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.js @@ -22,6 +22,11 @@ router authorized(BUILDER), controller.revertClient ) - .delete("/api/applications/:appId", authorized(BUILDER), usage, controller.delete) + .delete( + "/api/applications/:appId", + authorized(BUILDER), + usage, + controller.delete + ) module.exports = router diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index b3b486fe45..465fef82c8 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -5,7 +5,6 @@ const { PermissionLevels, PermissionTypes, } = require("@budibase/auth/permissions") -const usage = require("../../middleware/usageQuota") const router = Router() @@ -28,13 +27,13 @@ router .post( "/api/users/metadata/self", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - usage, + // usage, controller.updateSelfMetadata ) .delete( "/api/users/metadata/:id", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - usage, + // usage, controller.destroyMetadata ) diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index 41e775b3de..47d0b4eb99 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -60,7 +60,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, tenantId, emitter }) { +exports.run = async function ({ inputs, appId, emitter }) { if (inputs.row == null || inputs.row.tableId == null) { return { success: false, @@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, tenantId, emitter }) { inputs.row ) if (env.USE_QUOTAS) { - await usage.update(tenantId, usage.Properties.ROW, 1) + await usage.update(usage.Properties.ROW, 1) } await rowController.save(ctx) return { diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js index 0f9648cc51..225f00c5df 100644 --- a/packages/server/src/automations/steps/deleteRow.js +++ b/packages/server/src/automations/steps/deleteRow.js @@ -52,7 +52,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, apiKey, emitter }) { +exports.run = async function ({ inputs, appId, emitter }) { if (inputs.id == null || inputs.revision == null) { return { success: false, @@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { try { if (env.isProd()) { - await usage.update(apiKey, usage.Properties.ROW, -1) + await usage.update(usage.Properties.ROW, -1) } await rowController.destroy(ctx) return { diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js index a3e81a2274..37484e50bd 100644 --- a/packages/server/src/automations/thread.js +++ b/packages/server/src/automations/thread.js @@ -6,6 +6,8 @@ const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants const CouchDB = require("../db") const { DocumentTypes } = require("../db/utils") const { doInTenant } = require("@budibase/auth/tenancy") +const env = require("../environment") +const usage = require("../utilities/usageQuota") const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId @@ -80,7 +82,6 @@ class Orchestrator { return stepFn({ inputs: step.inputs, appId: this._appId, - apiKey: automation.apiKey, emitter: this._emitter, context: this._context, }) @@ -95,6 +96,10 @@ class Orchestrator { return err } } + // TODO: don't count test runs + if (!env.SELF_HOSTED) { + usage.update(usage.Properties.AUTOMATION, 1) + } return this.executionOutput } } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index c5db35ed2a..605e23024b 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils" module MySQLModule { const mysql = require("mysql") const Sql = require("./base/sql") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") const { FieldTypes } = require("../constants") interface MySQLConfig { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 23a8685648..707d779c65 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -12,7 +12,11 @@ module PostgresModule { const { Pool } = require("pg") const Sql = require("./base/sql") const { FieldTypes } = require("../constants") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") interface PostgresConfig { host: string @@ -179,10 +183,16 @@ module PostgresModule { } const type: string = convertType(column.data_type, TYPE_MAP) - const identity = !!(column.identity_generation || column.identity_start || column.identity_increment) - const hasDefault = typeof column.column_default === "string" && + const identity = !!( + column.identity_generation || + column.identity_start || + column.identity_increment + ) + const hasDefault = + typeof column.column_default === "string" && column.column_default.startsWith("nextval") - const isGenerated = column.is_generated && column.is_generated !== "NEVER" + const isGenerated = + column.is_generated && column.is_generated !== "NEVER" const isAuto: boolean = hasDefault || identity || isGenerated tables[tableName].schema[columnName] = { autocolumn: isAuto, diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 82c35bc2d9..6e3dc6f684 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -84,7 +84,11 @@ export function isIsoDateString(str: string) { } // add the existing relationships from the entities if they exist, to prevent them from being overridden -export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) { +export function copyExistingPropsOver( + tableName: string, + tables: { [key: string]: any }, + entities: { [key: string]: any } +) { if (entities && entities[tableName]) { if (entities[tableName].primaryDisplay) { tables[tableName].primaryDisplay = entities[tableName].primaryDisplay diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 4ad1092f6c..d18ffae205 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,6 +1,6 @@ const CouchDB = require("../db") const usageQuota = require("../utilities/usageQuota") -const env = require("../environment") +// const env = require("../environment") // currently only counting new writes and deletes const METHOD_MAP = { diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index 502fc4cad2..b98fb923cd 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,5 +1,4 @@ - -const env = require("../environment") +// const env = require("../environment") const { getGlobalDB } = require("@budibase/auth/tenancy") function getNewQuotaReset() { @@ -12,44 +11,44 @@ exports.Properties = { VIEW: "views", USER: "users", AUTOMATION: "automationRuns", - APPS: "apps" + APPS: "apps", } /** * Given a specified tenantId this will add to the usage object for the specified property. - * @param {string} tenantId The tenant to update the usage quotas for. * @param {string} property The property which is to be added to (within the nested usageQuota object). * @param {number} usage The amount (this can be negative) to adjust the number by. * @returns {Promise} When this completes the API key will now be up to date - the quota period may have * also been reset after this call. */ -exports.update = async (tenantId, property, usage) => { +exports.update = async (property, usage) => { // if (!env.USE_QUOTAS) { // return // } try { - const db = getGlobalDB() - const quota = await db.get("usage_quota") - // TODO: check if the quota needs reset - if (Date.now() >= quota.quotaReset) { - quota.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(quota.usageQuota)) { - quota.usageQuota[prop] = 0 - } - } + const db = getGlobalDB() + const quota = await db.get("usage_quota") + // TODO: check if the quota needs reset + if (Date.now() >= quota.quotaReset) { + quota.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(quota.usageQuota)) { + quota.usageQuota[prop] = 0 + } + } - // increment the quota - quota.usageQuota[property] += usage + // increment the quota + quota.usageQuota[property] += usage - if (quota.usageQuota[property] >= quota.usageLimits[property]) { - throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`) - } + if (quota.usageQuota[property] >= quota.usageLimits[property]) { + throw new Error( + `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` + ) + } - // update the usage quotas - await db.put(quota) + // update the usage quotas + await db.put(quota) } catch (err) { - console.error(`Error updating usage quotas for ${property}`, err) + console.error(`Error updating usage quotas for ${property}`, err) throw err } - } diff --git a/packages/server/src/utilities/usageQuota.old.js b/packages/server/src/utilities/usageQuota.old.js index bfe71a4093..39dad89c6d 100644 --- a/packages/server/src/utilities/usageQuota.old.js +++ b/packages/server/src/utilities/usageQuota.old.js @@ -1,105 +1,105 @@ -const env = require("../environment") -const { apiKeyTable } = require("../db/dynamoClient") +// const env = require("../environment") +// const { apiKeyTable } = require("../db/dynamoClient") -const DEFAULT_USAGE = { - rows: 0, - storage: 0, - views: 0, - automationRuns: 0, - users: 0, -} +// const DEFAULT_USAGE = { +// rows: 0, +// storage: 0, +// views: 0, +// automationRuns: 0, +// users: 0, +// } -const DEFAULT_PLAN = { - rows: 1000, - // 1 GB - storage: 8589934592, - views: 10, - automationRuns: 100, - users: 10000, -} +// const DEFAULT_PLAN = { +// rows: 1000, +// // 1 GB +// storage: 8589934592, +// views: 10, +// automationRuns: 100, +// users: 10000, +// } -function buildUpdateParams(key, property, usage) { - return { - primary: key, - condition: - "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", - expression: "ADD #quota.#prop :usage", - names: { - "#quota": "usageQuota", - "#prop": property, - "#limits": "usageLimits", - "#quotaReset": "quotaReset", - }, - values: { - ":usage": usage, - ":now": Date.now(), - }, - } -} +// function buildUpdateParams(key, property, usage) { +// return { +// primary: key, +// condition: +// "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", +// expression: "ADD #quota.#prop :usage", +// names: { +// "#quota": "usageQuota", +// "#prop": property, +// "#limits": "usageLimits", +// "#quotaReset": "quotaReset", +// }, +// values: { +// ":usage": usage, +// ":now": Date.now(), +// }, +// } +// } -function getNewQuotaReset() { - return Date.now() + 2592000000 -} +// function getNewQuotaReset() { +// return Date.now() + 2592000000 +// } -exports.Properties = { - ROW: "rows", - UPLOAD: "storage", - VIEW: "views", - USER: "users", - AUTOMATION: "automationRuns", -} +// exports.Properties = { +// ROW: "rows", +// UPLOAD: "storage", +// VIEW: "views", +// USER: "users", +// AUTOMATION: "automationRuns", +// } -exports.getAPIKey = async appId => { - if (!env.USE_QUOTAS) { - return { apiKey: null } - } - return apiKeyTable.get({ primary: appId }) -} +// exports.getAPIKey = async appId => { +// if (!env.USE_QUOTAS) { +// return { apiKey: null } +// } +// return apiKeyTable.get({ primary: appId }) +// } -/** - * Given a specified API key this will add to the usage object for the specified property. - * @param {string} apiKey The API key which is to be updated. - * @param {string} property The property which is to be added to (within the nested usageQuota object). - * @param {number} usage The amount (this can be negative) to adjust the number by. - * @returns {Promise} When this completes the API key will now be up to date - the quota period may have - * also been reset after this call. - */ -exports.update = async (apiKey, property, usage) => { - if (!env.USE_QUOTAS) { - return - } - try { - await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) - } catch (err) { - // conditional check means the condition failed, need to check why - if (err.code === "ConditionalCheckFailedException") { - // get the API key so we can check it - const keyObj = await apiKeyTable.get({ primary: apiKey }) - // the usage quota or usage limits didn't exist - if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { - keyObj.usageQuota = - keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota - keyObj.usageLimits = - keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits - keyObj.quotaReset = getNewQuotaReset() - await apiKeyTable.put({ item: keyObj }) - return - } - // we have in fact breached the reset period - else if (keyObj && keyObj.quotaReset <= Date.now()) { - // update the quota reset period and reset the values for all properties - keyObj.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(keyObj.usageQuota)) { - if (prop === property) { - keyObj.usageQuota[prop] = usage > 0 ? usage : 0 - } else { - keyObj.usageQuota[prop] = 0 - } - } - await apiKeyTable.put({ item: keyObj }) - return - } - } - throw err - } -} +// /** +// * Given a specified API key this will add to the usage object for the specified property. +// * @param {string} apiKey The API key which is to be updated. +// * @param {string} property The property which is to be added to (within the nested usageQuota object). +// * @param {number} usage The amount (this can be negative) to adjust the number by. +// * @returns {Promise} When this completes the API key will now be up to date - the quota period may have +// * also been reset after this call. +// */ +// exports.update = async (apiKey, property, usage) => { +// if (!env.USE_QUOTAS) { +// return +// } +// try { +// await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) +// } catch (err) { +// // conditional check means the condition failed, need to check why +// if (err.code === "ConditionalCheckFailedException") { +// // get the API key so we can check it +// const keyObj = await apiKeyTable.get({ primary: apiKey }) +// // the usage quota or usage limits didn't exist +// if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { +// keyObj.usageQuota = +// keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota +// keyObj.usageLimits = +// keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits +// keyObj.quotaReset = getNewQuotaReset() +// await apiKeyTable.put({ item: keyObj }) +// return +// } +// // we have in fact breached the reset period +// else if (keyObj && keyObj.quotaReset <= Date.now()) { +// // update the quota reset period and reset the values for all properties +// keyObj.quotaReset = getNewQuotaReset() +// for (let prop of Object.keys(keyObj.usageQuota)) { +// if (prop === property) { +// keyObj.usageQuota[prop] = usage > 0 ? usage : 0 +// } else { +// keyObj.usageQuota[prop] = 0 +// } +// } +// await apiKeyTable.put({ item: keyObj }) +// return +// } +// } +// throw err +// } +// } diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index b75c72290d..294a835f14 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -1,7 +1,6 @@ const { generateGlobalUserID, getGlobalUserParams, - StaticDatabases, } = require("@budibase/auth/db") const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils @@ -18,7 +17,7 @@ const { tryAddTenant, updateTenantId, } = require("@budibase/auth/tenancy") -const env = require("../../../environment") +// const env = require("../../../environment") const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -139,28 +138,28 @@ exports.adminUser = async ctx => { include_docs: true, }) ) - + // write usage quotas for cloud // if (!env.SELF_HOSTED) { - await db.post({ - _id: "usage_quota", - quotaReset: Date.now() + 2592000000, - usageQuota: { - automationRuns: 0, - rows: 0, - storage: 0, - apps: 0, - users: 0, - views: 0, - }, - usageLimits: { - automationRuns: 1000, - rows: 4000, - apps: 4, - // storage: 1000, - // users: 10 - }, - }) + await db.post({ + _id: "usage_quota", + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + storage: 1000, + users: 10 + }, + }) // } if (response.rows.some(row => row.doc.admin)) { From 83384faed84d14804ad2ec85bb30e380c1d8af4d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 23:26:09 +0100 Subject: [PATCH 03/10] remove old middleware --- .../server/src/utilities/usageQuota.old.js | 105 ------------------ 1 file changed, 105 deletions(-) delete mode 100644 packages/server/src/utilities/usageQuota.old.js diff --git a/packages/server/src/utilities/usageQuota.old.js b/packages/server/src/utilities/usageQuota.old.js deleted file mode 100644 index 39dad89c6d..0000000000 --- a/packages/server/src/utilities/usageQuota.old.js +++ /dev/null @@ -1,105 +0,0 @@ -// const env = require("../environment") -// const { apiKeyTable } = require("../db/dynamoClient") - -// const DEFAULT_USAGE = { -// rows: 0, -// storage: 0, -// views: 0, -// automationRuns: 0, -// users: 0, -// } - -// const DEFAULT_PLAN = { -// rows: 1000, -// // 1 GB -// storage: 8589934592, -// views: 10, -// automationRuns: 100, -// users: 10000, -// } - -// function buildUpdateParams(key, property, usage) { -// return { -// primary: key, -// condition: -// "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", -// expression: "ADD #quota.#prop :usage", -// names: { -// "#quota": "usageQuota", -// "#prop": property, -// "#limits": "usageLimits", -// "#quotaReset": "quotaReset", -// }, -// values: { -// ":usage": usage, -// ":now": Date.now(), -// }, -// } -// } - -// function getNewQuotaReset() { -// return Date.now() + 2592000000 -// } - -// exports.Properties = { -// ROW: "rows", -// UPLOAD: "storage", -// VIEW: "views", -// USER: "users", -// AUTOMATION: "automationRuns", -// } - -// exports.getAPIKey = async appId => { -// if (!env.USE_QUOTAS) { -// return { apiKey: null } -// } -// return apiKeyTable.get({ primary: appId }) -// } - -// /** -// * Given a specified API key this will add to the usage object for the specified property. -// * @param {string} apiKey The API key which is to be updated. -// * @param {string} property The property which is to be added to (within the nested usageQuota object). -// * @param {number} usage The amount (this can be negative) to adjust the number by. -// * @returns {Promise} When this completes the API key will now be up to date - the quota period may have -// * also been reset after this call. -// */ -// exports.update = async (apiKey, property, usage) => { -// if (!env.USE_QUOTAS) { -// return -// } -// try { -// await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) -// } catch (err) { -// // conditional check means the condition failed, need to check why -// if (err.code === "ConditionalCheckFailedException") { -// // get the API key so we can check it -// const keyObj = await apiKeyTable.get({ primary: apiKey }) -// // the usage quota or usage limits didn't exist -// if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { -// keyObj.usageQuota = -// keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota -// keyObj.usageLimits = -// keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits -// keyObj.quotaReset = getNewQuotaReset() -// await apiKeyTable.put({ item: keyObj }) -// return -// } -// // we have in fact breached the reset period -// else if (keyObj && keyObj.quotaReset <= Date.now()) { -// // update the quota reset period and reset the values for all properties -// keyObj.quotaReset = getNewQuotaReset() -// for (let prop of Object.keys(keyObj.usageQuota)) { -// if (prop === property) { -// keyObj.usageQuota[prop] = usage > 0 ? usage : 0 -// } else { -// keyObj.usageQuota[prop] = 0 -// } -// } -// await apiKeyTable.put({ item: keyObj }) -// return -// } -// } -// throw err -// } -// } From 6c7423e52e32e952b5f46fb1be8c263a86c23374 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 10:32:39 +0100 Subject: [PATCH 04/10] ensure automation quotas are not incremented for test runs --- packages/server/src/automations/thread.js | 7 ++++--- packages/worker/src/api/controllers/global/users.js | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js index 37484e50bd..ef12494165 100644 --- a/packages/server/src/automations/thread.js +++ b/packages/server/src/automations/thread.js @@ -4,7 +4,7 @@ const AutomationEmitter = require("../events/AutomationEmitter") const { processObject } = require("@budibase/string-templates") const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants const CouchDB = require("../db") -const { DocumentTypes } = require("../db/utils") +const { DocumentTypes, isDevAppID } = require("../db/utils") const { doInTenant } = require("@budibase/auth/tenancy") const env = require("../environment") const usage = require("../utilities/usageQuota") @@ -96,8 +96,9 @@ class Orchestrator { return err } } - // TODO: don't count test runs - if (!env.SELF_HOSTED) { + + // Increment quota for automation runs + if (!env.SELF_HOSTED && !isDevAppID(this._appId)) { usage.update(usage.Properties.AUTOMATION, 1) } return this.executionOutput diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 294a835f14..61c2981280 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -157,7 +157,7 @@ exports.adminUser = async ctx => { rows: 4000, apps: 4, storage: 1000, - users: 10 + users: 10, }, }) // } From 35d4eac78aa664c9f0c23b7be65392541d6d3e17 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 14:57:22 +0100 Subject: [PATCH 05/10] fallback SMTP configuration for cloud --- .../api/controllers/row/ExternalRequest.ts | 2 +- .../src/automations/tests/automation.spec.js | 2 -- .../src/automations/tests/createRow.spec.js | 2 +- .../src/automations/tests/deleteRow.spec.js | 2 +- packages/server/src/definitions/datasource.ts | 4 ++-- packages/server/src/integrations/base/sql.ts | 2 +- packages/server/src/integrations/mysql.ts | 6 +++--- .../src/middleware/tests/usageQuota.spec.js | 6 +++--- packages/server/src/middleware/usageQuota.js | 16 +++++++------- packages/server/src/utilities/usageQuota.js | 4 +++- .../src/api/controllers/global/users.js | 2 ++ packages/worker/src/environment.js | 5 +++++ packages/worker/src/utilities/email.js | 21 ++++++++++++++++--- 13 files changed, 48 insertions(+), 26 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 12db55efdc..75c3e9b492 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -546,7 +546,7 @@ module External { }, meta: { table, - } + }, } // can't really use response right now const response = await makeExternalQuery(appId, json) diff --git a/packages/server/src/automations/tests/automation.spec.js b/packages/server/src/automations/tests/automation.spec.js index 83b7b81a75..9444995ca1 100644 --- a/packages/server/src/automations/tests/automation.spec.js +++ b/packages/server/src/automations/tests/automation.spec.js @@ -13,8 +13,6 @@ const { makePartial } = require("../../tests/utilities") const { cleanInputValues } = require("../automationUtils") const setup = require("./utilities") -usageQuota.getAPIKey.mockReturnValue({ apiKey: "test" }) - describe("Run through some parts of the automations system", () => { let config = setup.getConfig() diff --git a/packages/server/src/automations/tests/createRow.spec.js b/packages/server/src/automations/tests/createRow.spec.js index 1004711d87..a04fc7aad4 100644 --- a/packages/server/src/automations/tests/createRow.spec.js +++ b/packages/server/src/automations/tests/createRow.spec.js @@ -46,7 +46,7 @@ describe("test the create row action", () => { await setup.runStep(setup.actions.CREATE_ROW.stepId, { row }) - expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", 1) }) }) diff --git a/packages/server/src/automations/tests/deleteRow.spec.js b/packages/server/src/automations/tests/deleteRow.spec.js index a3d73d3bf6..21246f22d0 100644 --- a/packages/server/src/automations/tests/deleteRow.spec.js +++ b/packages/server/src/automations/tests/deleteRow.spec.js @@ -37,7 +37,7 @@ describe("test the delete row action", () => { it("check usage quota attempts", async () => { await setup.runInProd(async () => { await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs) - expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", -1) }) }) diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index d7d4e77961..2daef8eda7 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -1,4 +1,4 @@ -import {Table} from "./common"; +import { Table } from "./common" export enum Operation { CREATE = "CREATE", @@ -139,7 +139,7 @@ export interface QueryJson { paginate?: PaginationJson body?: object meta?: { - table?: Table, + table?: Table } extra?: { idFilter?: SearchFilters diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 91af3e1a85..c5e9bdb0bb 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -148,7 +148,7 @@ function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery { if (!resource) { resource = { fields: [] } } - let selectStatement: string|string[] = "*" + let selectStatement: string | string[] = "*" // handle select if (resource.fields && resource.fields.length > 0) { // select the resources as the format "table.columnName" - this is what is provided diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index d43ae86bea..c17cca0745 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -108,7 +108,7 @@ module MySQLModule { client: any, query: SqlQuery, connect: boolean = true - ): Promise { + ): Promise { // Node MySQL is callback based, so we must wrap our call in a promise return new Promise((resolve, reject) => { if (connect) { @@ -252,9 +252,9 @@ module MySQLModule { json.extra = { idFilter: { equal: { - [primaryKey]: results.insertId + [primaryKey]: results.insertId, }, - } + }, } return json } diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js index 97d9c7794a..d828f2ca60 100644 --- a/packages/server/src/middleware/tests/usageQuota.spec.js +++ b/packages/server/src/middleware/tests/usageQuota.spec.js @@ -39,7 +39,7 @@ class TestConfiguration { if (bool) { env.isDev = () => false env.isProd = () => true - this.ctx.auth = { apiKey: "test" } + this.ctx.user = { tenantId: "test" } } else { env.isDev = () => true env.isProd = () => false @@ -114,7 +114,7 @@ describe("usageQuota middleware", () => { await config.executeMiddleware() - expect(usageQuota.update).toHaveBeenCalledWith("test", "rows", 1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", 1) expect(config.next).toHaveBeenCalled() }) @@ -131,7 +131,7 @@ describe("usageQuota middleware", () => { ]) await config.executeMiddleware() - expect(usageQuota.update).toHaveBeenCalledWith("test", "storage", 10100) + expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100) expect(config.next).toHaveBeenCalled() }) }) \ No newline at end of file diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index d18ffae205..d56a960615 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -50,15 +50,15 @@ module.exports = async (ctx, next) => { } // update usage for uploads to be the total size - // if (property === usageQuota.Properties.UPLOAD) { - // const files = - // ctx.request.files.file.length > 1 - // ? Array.from(ctx.request.files.file) - // : [ctx.request.files.file] - // usage = files.map(file => file.size).reduce((total, size) => total + size) - // } + if (property === usageQuota.Properties.UPLOAD) { + const files = + ctx.request.files.file.length > 1 + ? Array.from(ctx.request.files.file) + : [ctx.request.files.file] + usage = files.map(file => file.size).reduce((total, size) => total + size) + } try { - await usageQuota.update(ctx.user.tenantId, property, usage) + await usageQuota.update(property, usage) return next() } catch (err) { ctx.throw(400, err) diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index b98fb923cd..fb3f61f2f6 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -12,6 +12,7 @@ exports.Properties = { USER: "users", AUTOMATION: "automationRuns", APPS: "apps", + EMAILS: "emails", } /** @@ -28,7 +29,8 @@ exports.update = async (property, usage) => { try { const db = getGlobalDB() const quota = await db.get("usage_quota") - // TODO: check if the quota needs reset + + // Check if the quota needs reset if (Date.now() >= quota.quotaReset) { quota.quotaReset = getNewQuotaReset() for (let prop of Object.keys(quota.usageQuota)) { diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 61c2981280..c714f2b1ca 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -151,6 +151,7 @@ exports.adminUser = async ctx => { apps: 0, users: 0, views: 0, + emails: 0, }, usageLimits: { automationRuns: 1000, @@ -158,6 +159,7 @@ exports.adminUser = async ctx => { apps: 4, storage: 1000, users: 10, + emails: 50, }, }) // } diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index 646536f292..5f0556efc4 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -33,6 +33,11 @@ module.exports = { INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, MULTI_TENANCY: process.env.MULTI_TENANCY, ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL, + SMTP_USER: process.env.SMTP_USER, + SMTP_PASSWORD: process.env.SMTP_PASSWORD, + SMTP_HOST: process.env.SMTP_HOST, + SMTP_PORT: process.env.SMTP_PORT, + SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index d22933ef36..5843cb28ea 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -1,4 +1,5 @@ const nodemailer = require("nodemailer") +const env = require("../environment") const { getScopedConfig } = require("@budibase/auth/db") const { EmailTemplatePurpose, TemplateTypes, Configs } = require("../constants") const { getTemplateByPurpose } = require("../constants/templates") @@ -110,7 +111,21 @@ async function getSmtpConfiguration(db, workspaceId = null) { if (workspaceId) { params.workspace = workspaceId } - return getScopedConfig(db, params) + + if (!env.SMTP_FALLBACK_ENABLED) { + return getScopedConfig(db, params) + } else { + // Use an SMTP fallback configuration from env variables + return { + port: env.SMTP_PORT, + host: env.SMTP_HOST, + secure: false, + auth: { + user: env.SMTP_USER, + pass: env.SMTP_PASSWORD, + }, + } + } } /** @@ -118,8 +133,8 @@ async function getSmtpConfiguration(db, workspaceId = null) { * @return {Promise} returns true if there is a configuration that can be used. */ exports.isEmailConfigured = async (workspaceId = null) => { - // when "testing" simply return true - if (TEST_MODE) { + // when "testing" or smtp fallback is enabled simply return true + if (TEST_MODE || env.SMTP_FALLBACK_ENABLED) { return true } const db = getGlobalDB() From 5b4bf74e2c438c0549d6945b8f41a83f3410460d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 15:03:48 +0100 Subject: [PATCH 06/10] re-adding env var controls for cloud limit functionality --- packages/server/src/middleware/usageQuota.js | 8 ++-- packages/server/src/utilities/usageQuota.js | 9 ++-- .../src/api/controllers/global/users.js | 48 +++++++++---------- packages/worker/src/utilities/email.js | 6 ++- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index d56a960615..3a244ef5bc 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,6 +1,6 @@ const CouchDB = require("../db") const usageQuota = require("../utilities/usageQuota") -// const env = require("../environment") +const env = require("../environment") // currently only counting new writes and deletes const METHOD_MAP = { @@ -29,9 +29,9 @@ function getProperty(url) { module.exports = async (ctx, next) => { // if in development or a self hosted cloud usage quotas should not be executed - // if (env.isDev() || env.SELF_HOSTED) { - // return next() - // } + if (env.isDev() || env.SELF_HOSTED) { + return next() + } const db = new CouchDB(ctx.appId) let usage = METHOD_MAP[ctx.req.method] diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index fb3f61f2f6..4e645ec67d 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,4 +1,4 @@ -// const env = require("../environment") +const env = require("../environment") const { getGlobalDB } = require("@budibase/auth/tenancy") function getNewQuotaReset() { @@ -23,9 +23,10 @@ exports.Properties = { * also been reset after this call. */ exports.update = async (property, usage) => { - // if (!env.USE_QUOTAS) { - // return - // } + if (!env.USE_QUOTAS) { + return + } + try { const db = getGlobalDB() const quota = await db.get("usage_quota") diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index c714f2b1ca..e00fd3ecc0 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -17,7 +17,7 @@ const { tryAddTenant, updateTenantId, } = require("@budibase/auth/tenancy") -// const env = require("../../../environment") +const env = require("../../../environment") const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -140,29 +140,29 @@ exports.adminUser = async ctx => { ) // write usage quotas for cloud - // if (!env.SELF_HOSTED) { - await db.post({ - _id: "usage_quota", - quotaReset: Date.now() + 2592000000, - usageQuota: { - automationRuns: 0, - rows: 0, - storage: 0, - apps: 0, - users: 0, - views: 0, - emails: 0, - }, - usageLimits: { - automationRuns: 1000, - rows: 4000, - apps: 4, - storage: 1000, - users: 10, - emails: 50, - }, - }) - // } + if (!env.SELF_HOSTED) { + await db.post({ + _id: "usage_quota", + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + emails: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + storage: 1000, + users: 10, + emails: 50, + }, + }) + } if (response.rows.some(row => row.doc.admin)) { ctx.throw( diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index 5843cb28ea..cd29170348 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -112,8 +112,10 @@ async function getSmtpConfiguration(db, workspaceId = null) { params.workspace = workspaceId } - if (!env.SMTP_FALLBACK_ENABLED) { - return getScopedConfig(db, params) + const customConfig = getScopedConfig(db, params) + + if (customConfig && !env.SMTP_FALLBACK_ENABLED) { + return customConfig } else { // Use an SMTP fallback configuration from env variables return { From c25bd4861696714c17d8b66e415a779809ac49c2 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 15:10:06 +0100 Subject: [PATCH 07/10] SMTP config fallback logic --- packages/worker/src/utilities/email.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index cd29170348..5fd8c05939 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -114,10 +114,12 @@ async function getSmtpConfiguration(db, workspaceId = null) { const customConfig = getScopedConfig(db, params) - if (customConfig && !env.SMTP_FALLBACK_ENABLED) { + if (customConfig) { return customConfig - } else { - // Use an SMTP fallback configuration from env variables + } + + // Use an SMTP fallback configuration from env variables + if (env.SMTP_FALLBACK_ENABLED) { return { port: env.SMTP_PORT, host: env.SMTP_HOST, From 2da28ecc71942670cecb277d8ce5a0a0f46ac908 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 16:28:39 +0100 Subject: [PATCH 08/10] prevent SMTP fallback for automations --- .../server/src/automations/steps/sendSmtpEmail.js | 2 +- packages/server/src/utilities/workerRequests.js | 3 ++- .../worker/src/api/controllers/global/email.js | 13 +++++++++++-- packages/worker/src/utilities/email.js | 14 ++++++++------ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/packages/server/src/automations/steps/sendSmtpEmail.js b/packages/server/src/automations/steps/sendSmtpEmail.js index 9e4b5a6a3c..07a3059215 100644 --- a/packages/server/src/automations/steps/sendSmtpEmail.js +++ b/packages/server/src/automations/steps/sendSmtpEmail.js @@ -53,7 +53,7 @@ exports.run = async function ({ inputs }) { contents = "

No content

" } try { - let response = await sendSmtpEmail(to, from, subject, contents) + let response = await sendSmtpEmail(to, from, subject, contents, true) return { success: true, response, diff --git a/packages/server/src/utilities/workerRequests.js b/packages/server/src/utilities/workerRequests.js index 377658084f..2ace265ca0 100644 --- a/packages/server/src/utilities/workerRequests.js +++ b/packages/server/src/utilities/workerRequests.js @@ -34,7 +34,7 @@ function request(ctx, request) { exports.request = request // have to pass in the tenant ID as this could be coming from an automation -exports.sendSmtpEmail = async (to, from, subject, contents) => { +exports.sendSmtpEmail = async (to, from, subject, contents, automation) => { // tenant ID will be set in header const response = await fetch( checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`), @@ -46,6 +46,7 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => { contents, subject, purpose: "custom", + automation, }, }) ) diff --git a/packages/worker/src/api/controllers/global/email.js b/packages/worker/src/api/controllers/global/email.js index 57b78a6d7a..e194a30862 100644 --- a/packages/worker/src/api/controllers/global/email.js +++ b/packages/worker/src/api/controllers/global/email.js @@ -2,8 +2,16 @@ const { sendEmail } = require("../../../utilities/email") const { getGlobalDB } = require("@budibase/auth/tenancy") exports.sendEmail = async ctx => { - let { workspaceId, email, userId, purpose, contents, from, subject } = - ctx.request.body + let { + workspaceId, + email, + userId, + purpose, + contents, + from, + subject, + automation, + } = ctx.request.body let user if (userId) { const db = getGlobalDB() @@ -15,6 +23,7 @@ exports.sendEmail = async ctx => { contents, from, subject, + automation, }) ctx.body = { ...response, diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index 5fd8c05939..14c836952e 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -102,9 +102,10 @@ async function buildEmail(purpose, email, context, { user, contents } = {}) { * Utility function for finding most valid SMTP configuration. * @param {object} db The CouchDB database which is to be looked up within. * @param {string|null} workspaceId If using finer grain control of configs a workspace can be used. + * @param {boolean|null} automation Whether or not the configuration is being fetched for an email automation. * @return {Promise} returns the SMTP configuration if it exists */ -async function getSmtpConfiguration(db, workspaceId = null) { +async function getSmtpConfiguration(db, workspaceId = null, automation) { const params = { type: Configs.SMTP, } @@ -116,10 +117,10 @@ async function getSmtpConfiguration(db, workspaceId = null) { if (customConfig) { return customConfig - } - + } + // Use an SMTP fallback configuration from env variables - if (env.SMTP_FALLBACK_ENABLED) { + if (!automation && env.SMTP_FALLBACK_ENABLED) { return { port: env.SMTP_PORT, host: env.SMTP_HOST, @@ -157,16 +158,17 @@ exports.isEmailConfigured = async (workspaceId = null) => { * @param {string|undefined} contents If sending a custom email then can supply contents which will be added to it. * @param {string|undefined} subject A custom subject can be specified if the config one is not desired. * @param {object|undefined} info Pass in a structure of information to be stored alongside the invitation. + * @param {boolean|undefined} disableFallback Prevent email being sent from SMTP fallback to avoid spam. * @return {Promise} returns details about the attempt to send email, e.g. if it is successful; based on * nodemailer response. */ exports.sendEmail = async ( email, purpose, - { workspaceId, user, from, contents, subject, info } = {} + { workspaceId, user, from, contents, subject, info, automation } = {} ) => { const db = getGlobalDB() - let config = (await getSmtpConfiguration(db, workspaceId)) || {} + let config = (await getSmtpConfiguration(db, workspaceId, automation)) || {} if (Object.keys(config).length === 0 && !TEST_MODE) { throw "Unable to find SMTP configuration." } From f6014190c8860cc1529e1be50dad77eaed3337f7 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 16:39:08 +0100 Subject: [PATCH 09/10] tests --- packages/server/src/api/routes/view.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js index 7d390805c6..f096dd4185 100644 --- a/packages/server/src/api/routes/view.js +++ b/packages/server/src/api/routes/view.js @@ -25,9 +25,8 @@ router "/api/views/:viewName", paramResource("viewName"), authorized(BUILDER), - usage, viewController.destroy ) - .post("/api/views", authorized(BUILDER), usage, viewController.save) + .post("/api/views", authorized(BUILDER), viewController.save) module.exports = router From 1e3fca15c5a18b837b9222a72e35c09cb8dad16f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 17:35:22 +0100 Subject: [PATCH 10/10] abstract usagedoc creation into auth, create the doc if it doesn't exist --- packages/auth/src/db/constants.js | 1 + packages/auth/src/db/utils.js | 25 +++++++++++++++++++ packages/server/src/api/routes/user.js | 2 -- packages/server/src/api/routes/view.js | 1 - packages/server/src/utilities/usageQuota.js | 18 ++++++++++++- .../src/api/controllers/global/users.js | 23 ++--------------- 6 files changed, 45 insertions(+), 25 deletions(-) diff --git a/packages/auth/src/db/constants.js b/packages/auth/src/db/constants.js index 77643ce4c5..ad4f6c9f66 100644 --- a/packages/auth/src/db/constants.js +++ b/packages/auth/src/db/constants.js @@ -12,6 +12,7 @@ exports.StaticDatabases = { name: "global-info", docs: { tenants: "tenants", + usageQuota: "usage_quota", }, }, } diff --git a/packages/auth/src/db/utils.js b/packages/auth/src/db/utils.js index a1a831523e..09e2ff6314 100644 --- a/packages/auth/src/db/utils.js +++ b/packages/auth/src/db/utils.js @@ -368,8 +368,33 @@ async function getScopedConfig(db, params) { return configDoc && configDoc.config ? configDoc.config : configDoc } +function generateNewUsageQuotaDoc() { + return { + _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota, + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + emails: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + storage: 1000, + users: 10, + emails: 50, + }, + } +} + exports.Replication = Replication exports.getScopedConfig = getScopedConfig exports.generateConfigID = generateConfigID exports.getConfigParams = getConfigParams exports.getScopedFullConfig = getScopedFullConfig +exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index 465fef82c8..d171870215 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -27,13 +27,11 @@ router .post( "/api/users/metadata/self", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - // usage, controller.updateSelfMetadata ) .delete( "/api/users/metadata/:id", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - // usage, controller.destroyMetadata ) diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js index f096dd4185..b72fe1ac26 100644 --- a/packages/server/src/api/routes/view.js +++ b/packages/server/src/api/routes/view.js @@ -8,7 +8,6 @@ const { PermissionTypes, PermissionLevels, } = require("@budibase/auth/permissions") -const usage = require("../../middleware/usageQuota") const router = Router() diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index 4e645ec67d..80fddb8303 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,5 +1,9 @@ const env = require("../environment") const { getGlobalDB } = require("@budibase/auth/tenancy") +const { + StaticDatabases, + generateNewUsageQuotaDoc, +} = require("@budibase/auth/db") function getNewQuotaReset() { return Date.now() + 2592000000 @@ -15,6 +19,18 @@ exports.Properties = { EMAILS: "emails", } +async function getUsageQuotaDoc(db) { + let quota + try { + quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota) + } catch (err) { + // doc doesn't exist. Create it + quota = await db.post(generateNewUsageQuotaDoc()) + } + + return quota +} + /** * Given a specified tenantId this will add to the usage object for the specified property. * @param {string} property The property which is to be added to (within the nested usageQuota object). @@ -29,7 +45,7 @@ exports.update = async (property, usage) => { try { const db = getGlobalDB() - const quota = await db.get("usage_quota") + const quota = await getUsageQuotaDoc(db) // Check if the quota needs reset if (Date.now() >= quota.quotaReset) { diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index e00fd3ecc0..1d3f38698b 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -2,6 +2,7 @@ const { generateGlobalUserID, getGlobalUserParams, StaticDatabases, + generateNewUsageQuotaDoc, } = require("@budibase/auth/db") const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils const { UserStatus, EmailTemplatePurpose } = require("../../../constants") @@ -141,27 +142,7 @@ exports.adminUser = async ctx => { // write usage quotas for cloud if (!env.SELF_HOSTED) { - await db.post({ - _id: "usage_quota", - quotaReset: Date.now() + 2592000000, - usageQuota: { - automationRuns: 0, - rows: 0, - storage: 0, - apps: 0, - users: 0, - views: 0, - emails: 0, - }, - usageLimits: { - automationRuns: 1000, - rows: 4000, - apps: 4, - storage: 1000, - users: 10, - emails: 50, - }, - }) + await db.post(generateNewUsageQuotaDoc()) } if (response.rows.some(row => row.doc.admin)) {