From b161be85aec73b0e74464c65a14e0396072d51dc Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 23:25:25 +0100 Subject: [PATCH] automation runs quotas --- packages/server/src/api/routes/application.js | 7 +- packages/server/src/api/routes/user.js | 5 +- .../server/src/automations/steps/createRow.js | 4 +- .../server/src/automations/steps/deleteRow.js | 4 +- packages/server/src/automations/thread.js | 7 +- packages/server/src/integrations/mysql.ts | 6 +- packages/server/src/integrations/postgres.ts | 18 +- packages/server/src/integrations/utils.ts | 6 +- packages/server/src/middleware/usageQuota.js | 2 +- packages/server/src/utilities/usageQuota.js | 45 ++-- .../server/src/utilities/usageQuota.old.js | 196 +++++++++--------- .../src/api/controllers/global/users.js | 43 ++-- 12 files changed, 184 insertions(+), 159 deletions(-) diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js index ef4aacf708..4d67a0f4f4 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.js @@ -22,6 +22,11 @@ router authorized(BUILDER), controller.revertClient ) - .delete("/api/applications/:appId", authorized(BUILDER), usage, controller.delete) + .delete( + "/api/applications/:appId", + authorized(BUILDER), + usage, + controller.delete + ) module.exports = router diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index b3b486fe45..465fef82c8 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -5,7 +5,6 @@ const { PermissionLevels, PermissionTypes, } = require("@budibase/auth/permissions") -const usage = require("../../middleware/usageQuota") const router = Router() @@ -28,13 +27,13 @@ router .post( "/api/users/metadata/self", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - usage, + // usage, controller.updateSelfMetadata ) .delete( "/api/users/metadata/:id", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - usage, + // usage, controller.destroyMetadata ) diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index 41e775b3de..47d0b4eb99 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -60,7 +60,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, tenantId, emitter }) { +exports.run = async function ({ inputs, appId, emitter }) { if (inputs.row == null || inputs.row.tableId == null) { return { success: false, @@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, tenantId, emitter }) { inputs.row ) if (env.USE_QUOTAS) { - await usage.update(tenantId, usage.Properties.ROW, 1) + await usage.update(usage.Properties.ROW, 1) } await rowController.save(ctx) return { diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js index 0f9648cc51..225f00c5df 100644 --- a/packages/server/src/automations/steps/deleteRow.js +++ b/packages/server/src/automations/steps/deleteRow.js @@ -52,7 +52,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, apiKey, emitter }) { +exports.run = async function ({ inputs, appId, emitter }) { if (inputs.id == null || inputs.revision == null) { return { success: false, @@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { try { if (env.isProd()) { - await usage.update(apiKey, usage.Properties.ROW, -1) + await usage.update(usage.Properties.ROW, -1) } await rowController.destroy(ctx) return { diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js index a3e81a2274..37484e50bd 100644 --- a/packages/server/src/automations/thread.js +++ b/packages/server/src/automations/thread.js @@ -6,6 +6,8 @@ const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants const CouchDB = require("../db") const { DocumentTypes } = require("../db/utils") const { doInTenant } = require("@budibase/auth/tenancy") +const env = require("../environment") +const usage = require("../utilities/usageQuota") const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId @@ -80,7 +82,6 @@ class Orchestrator { return stepFn({ inputs: step.inputs, appId: this._appId, - apiKey: automation.apiKey, emitter: this._emitter, context: this._context, }) @@ -95,6 +96,10 @@ class Orchestrator { return err } } + // TODO: don't count test runs + if (!env.SELF_HOSTED) { + usage.update(usage.Properties.AUTOMATION, 1) + } return this.executionOutput } } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index c5db35ed2a..605e23024b 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils" module MySQLModule { const mysql = require("mysql") const Sql = require("./base/sql") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") const { FieldTypes } = require("../constants") interface MySQLConfig { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 23a8685648..707d779c65 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -12,7 +12,11 @@ module PostgresModule { const { Pool } = require("pg") const Sql = require("./base/sql") const { FieldTypes } = require("../constants") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") interface PostgresConfig { host: string @@ -179,10 +183,16 @@ module PostgresModule { } const type: string = convertType(column.data_type, TYPE_MAP) - const identity = !!(column.identity_generation || column.identity_start || column.identity_increment) - const hasDefault = typeof column.column_default === "string" && + const identity = !!( + column.identity_generation || + column.identity_start || + column.identity_increment + ) + const hasDefault = + typeof column.column_default === "string" && column.column_default.startsWith("nextval") - const isGenerated = column.is_generated && column.is_generated !== "NEVER" + const isGenerated = + column.is_generated && column.is_generated !== "NEVER" const isAuto: boolean = hasDefault || identity || isGenerated tables[tableName].schema[columnName] = { autocolumn: isAuto, diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 82c35bc2d9..6e3dc6f684 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -84,7 +84,11 @@ export function isIsoDateString(str: string) { } // add the existing relationships from the entities if they exist, to prevent them from being overridden -export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) { +export function copyExistingPropsOver( + tableName: string, + tables: { [key: string]: any }, + entities: { [key: string]: any } +) { if (entities && entities[tableName]) { if (entities[tableName].primaryDisplay) { tables[tableName].primaryDisplay = entities[tableName].primaryDisplay diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 4ad1092f6c..d18ffae205 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,6 +1,6 @@ const CouchDB = require("../db") const usageQuota = require("../utilities/usageQuota") -const env = require("../environment") +// const env = require("../environment") // currently only counting new writes and deletes const METHOD_MAP = { diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index 502fc4cad2..b98fb923cd 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,5 +1,4 @@ - -const env = require("../environment") +// const env = require("../environment") const { getGlobalDB } = require("@budibase/auth/tenancy") function getNewQuotaReset() { @@ -12,44 +11,44 @@ exports.Properties = { VIEW: "views", USER: "users", AUTOMATION: "automationRuns", - APPS: "apps" + APPS: "apps", } /** * Given a specified tenantId this will add to the usage object for the specified property. - * @param {string} tenantId The tenant to update the usage quotas for. * @param {string} property The property which is to be added to (within the nested usageQuota object). * @param {number} usage The amount (this can be negative) to adjust the number by. * @returns {Promise} When this completes the API key will now be up to date - the quota period may have * also been reset after this call. */ -exports.update = async (tenantId, property, usage) => { +exports.update = async (property, usage) => { // if (!env.USE_QUOTAS) { // return // } try { - const db = getGlobalDB() - const quota = await db.get("usage_quota") - // TODO: check if the quota needs reset - if (Date.now() >= quota.quotaReset) { - quota.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(quota.usageQuota)) { - quota.usageQuota[prop] = 0 - } - } + const db = getGlobalDB() + const quota = await db.get("usage_quota") + // TODO: check if the quota needs reset + if (Date.now() >= quota.quotaReset) { + quota.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(quota.usageQuota)) { + quota.usageQuota[prop] = 0 + } + } - // increment the quota - quota.usageQuota[property] += usage + // increment the quota + quota.usageQuota[property] += usage - if (quota.usageQuota[property] >= quota.usageLimits[property]) { - throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`) - } + if (quota.usageQuota[property] >= quota.usageLimits[property]) { + throw new Error( + `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` + ) + } - // update the usage quotas - await db.put(quota) + // update the usage quotas + await db.put(quota) } catch (err) { - console.error(`Error updating usage quotas for ${property}`, err) + console.error(`Error updating usage quotas for ${property}`, err) throw err } - } diff --git a/packages/server/src/utilities/usageQuota.old.js b/packages/server/src/utilities/usageQuota.old.js index bfe71a4093..39dad89c6d 100644 --- a/packages/server/src/utilities/usageQuota.old.js +++ b/packages/server/src/utilities/usageQuota.old.js @@ -1,105 +1,105 @@ -const env = require("../environment") -const { apiKeyTable } = require("../db/dynamoClient") +// const env = require("../environment") +// const { apiKeyTable } = require("../db/dynamoClient") -const DEFAULT_USAGE = { - rows: 0, - storage: 0, - views: 0, - automationRuns: 0, - users: 0, -} +// const DEFAULT_USAGE = { +// rows: 0, +// storage: 0, +// views: 0, +// automationRuns: 0, +// users: 0, +// } -const DEFAULT_PLAN = { - rows: 1000, - // 1 GB - storage: 8589934592, - views: 10, - automationRuns: 100, - users: 10000, -} +// const DEFAULT_PLAN = { +// rows: 1000, +// // 1 GB +// storage: 8589934592, +// views: 10, +// automationRuns: 100, +// users: 10000, +// } -function buildUpdateParams(key, property, usage) { - return { - primary: key, - condition: - "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", - expression: "ADD #quota.#prop :usage", - names: { - "#quota": "usageQuota", - "#prop": property, - "#limits": "usageLimits", - "#quotaReset": "quotaReset", - }, - values: { - ":usage": usage, - ":now": Date.now(), - }, - } -} +// function buildUpdateParams(key, property, usage) { +// return { +// primary: key, +// condition: +// "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", +// expression: "ADD #quota.#prop :usage", +// names: { +// "#quota": "usageQuota", +// "#prop": property, +// "#limits": "usageLimits", +// "#quotaReset": "quotaReset", +// }, +// values: { +// ":usage": usage, +// ":now": Date.now(), +// }, +// } +// } -function getNewQuotaReset() { - return Date.now() + 2592000000 -} +// function getNewQuotaReset() { +// return Date.now() + 2592000000 +// } -exports.Properties = { - ROW: "rows", - UPLOAD: "storage", - VIEW: "views", - USER: "users", - AUTOMATION: "automationRuns", -} +// exports.Properties = { +// ROW: "rows", +// UPLOAD: "storage", +// VIEW: "views", +// USER: "users", +// AUTOMATION: "automationRuns", +// } -exports.getAPIKey = async appId => { - if (!env.USE_QUOTAS) { - return { apiKey: null } - } - return apiKeyTable.get({ primary: appId }) -} +// exports.getAPIKey = async appId => { +// if (!env.USE_QUOTAS) { +// return { apiKey: null } +// } +// return apiKeyTable.get({ primary: appId }) +// } -/** - * Given a specified API key this will add to the usage object for the specified property. - * @param {string} apiKey The API key which is to be updated. - * @param {string} property The property which is to be added to (within the nested usageQuota object). - * @param {number} usage The amount (this can be negative) to adjust the number by. - * @returns {Promise} When this completes the API key will now be up to date - the quota period may have - * also been reset after this call. - */ -exports.update = async (apiKey, property, usage) => { - if (!env.USE_QUOTAS) { - return - } - try { - await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) - } catch (err) { - // conditional check means the condition failed, need to check why - if (err.code === "ConditionalCheckFailedException") { - // get the API key so we can check it - const keyObj = await apiKeyTable.get({ primary: apiKey }) - // the usage quota or usage limits didn't exist - if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { - keyObj.usageQuota = - keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota - keyObj.usageLimits = - keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits - keyObj.quotaReset = getNewQuotaReset() - await apiKeyTable.put({ item: keyObj }) - return - } - // we have in fact breached the reset period - else if (keyObj && keyObj.quotaReset <= Date.now()) { - // update the quota reset period and reset the values for all properties - keyObj.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(keyObj.usageQuota)) { - if (prop === property) { - keyObj.usageQuota[prop] = usage > 0 ? usage : 0 - } else { - keyObj.usageQuota[prop] = 0 - } - } - await apiKeyTable.put({ item: keyObj }) - return - } - } - throw err - } -} +// /** +// * Given a specified API key this will add to the usage object for the specified property. +// * @param {string} apiKey The API key which is to be updated. +// * @param {string} property The property which is to be added to (within the nested usageQuota object). +// * @param {number} usage The amount (this can be negative) to adjust the number by. +// * @returns {Promise} When this completes the API key will now be up to date - the quota period may have +// * also been reset after this call. +// */ +// exports.update = async (apiKey, property, usage) => { +// if (!env.USE_QUOTAS) { +// return +// } +// try { +// await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) +// } catch (err) { +// // conditional check means the condition failed, need to check why +// if (err.code === "ConditionalCheckFailedException") { +// // get the API key so we can check it +// const keyObj = await apiKeyTable.get({ primary: apiKey }) +// // the usage quota or usage limits didn't exist +// if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { +// keyObj.usageQuota = +// keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota +// keyObj.usageLimits = +// keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits +// keyObj.quotaReset = getNewQuotaReset() +// await apiKeyTable.put({ item: keyObj }) +// return +// } +// // we have in fact breached the reset period +// else if (keyObj && keyObj.quotaReset <= Date.now()) { +// // update the quota reset period and reset the values for all properties +// keyObj.quotaReset = getNewQuotaReset() +// for (let prop of Object.keys(keyObj.usageQuota)) { +// if (prop === property) { +// keyObj.usageQuota[prop] = usage > 0 ? usage : 0 +// } else { +// keyObj.usageQuota[prop] = 0 +// } +// } +// await apiKeyTable.put({ item: keyObj }) +// return +// } +// } +// throw err +// } +// } diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index b75c72290d..294a835f14 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -1,7 +1,6 @@ const { generateGlobalUserID, getGlobalUserParams, - StaticDatabases, } = require("@budibase/auth/db") const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils @@ -18,7 +17,7 @@ const { tryAddTenant, updateTenantId, } = require("@budibase/auth/tenancy") -const env = require("../../../environment") +// const env = require("../../../environment") const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -139,28 +138,28 @@ exports.adminUser = async ctx => { include_docs: true, }) ) - + // write usage quotas for cloud // if (!env.SELF_HOSTED) { - await db.post({ - _id: "usage_quota", - quotaReset: Date.now() + 2592000000, - usageQuota: { - automationRuns: 0, - rows: 0, - storage: 0, - apps: 0, - users: 0, - views: 0, - }, - usageLimits: { - automationRuns: 1000, - rows: 4000, - apps: 4, - // storage: 1000, - // users: 10 - }, - }) + await db.post({ + _id: "usage_quota", + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + storage: 1000, + users: 10 + }, + }) // } if (response.rows.some(row => row.doc.admin)) {