CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-06-27 18:40:42 +12:00
Code Issues Projects Releases Wiki Activity

automation runs quotas

Browse source
This commit is contained in:
Martin McKeaveney 2021-09-23 23:25:25 +01:00
parent 85c6fac8f5
commit b161be85ae
12 changed files with 184 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
packages/server/src/api/routes/application.js
View file

@ -22,6 +22,11 @@ router
authorized(BUILDER), authorized(BUILDER),
controller.revertClient controller.revertClient
) )
.delete("/api/applications/:appId", authorized(BUILDER), usage, controller.delete) .delete(
"/api/applications/:appId",
authorized(BUILDER),
usage,
controller.delete
)
module.exports = router module.exports = router

5
packages/server/src/api/routes/user.js
View file

@ -5,7 +5,6 @@ const {
PermissionLevels, PermissionLevels,
PermissionTypes, PermissionTypes,
} = require("@budibase/auth/permissions") } = require("@budibase/auth/permissions")
const usage = require("../../middleware/usageQuota")
const router = Router() const router = Router()
@ -28,13 +27,13 @@ router
.post( .post(
"/api/users/metadata/self", "/api/users/metadata/self",
authorized(PermissionTypes.USER, PermissionLevels.WRITE), authorized(PermissionTypes.USER, PermissionLevels.WRITE),
usage, // usage,
controller.updateSelfMetadata controller.updateSelfMetadata
) )
.delete( .delete(
"/api/users/metadata/:id", "/api/users/metadata/:id",
authorized(PermissionTypes.USER, PermissionLevels.WRITE), authorized(PermissionTypes.USER, PermissionLevels.WRITE),
usage, // usage,
controller.destroyMetadata controller.destroyMetadata
) )

4
packages/server/src/automations/steps/createRow.js
View file

@ -60,7 +60,7 @@ exports.definition = {
}, },
} }
exports.run = async function ({ inputs, appId, tenantId, emitter }) { exports.run = async function ({ inputs, appId, emitter }) {
if (inputs.row == null || inputs.row.tableId == null) { if (inputs.row == null || inputs.row.tableId == null) {
return { return {
success: false, success: false,
@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, tenantId, emitter }) {
inputs.row inputs.row
) )
if (env.USE_QUOTAS) { if (env.USE_QUOTAS) {
await usage.update(tenantId, usage.Properties.ROW, 1) await usage.update(usage.Properties.ROW, 1)
} }
await rowController.save(ctx) await rowController.save(ctx)
return { return {

4
packages/server/src/automations/steps/deleteRow.js
View file

@ -52,7 +52,7 @@ exports.definition = {
}, },
} }
exports.run = async function ({ inputs, appId, apiKey, emitter }) { exports.run = async function ({ inputs, appId, emitter }) {
if (inputs.id == null || inputs.revision == null) { if (inputs.id == null || inputs.revision == null) {
return { return {
success: false, success: false,
@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
try { try {
if (env.isProd()) { if (env.isProd()) {
await usage.update(apiKey, usage.Properties.ROW, -1) await usage.update(usage.Properties.ROW, -1)
} }
await rowController.destroy(ctx) await rowController.destroy(ctx)
return { return {

7
packages/server/src/automations/thread.js
View file

@ -6,6 +6,8 @@ const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
const CouchDB = require("../db") const CouchDB = require("../db")
const { DocumentTypes } = require("../db/utils") const { DocumentTypes } = require("../db/utils")
const { doInTenant } = require("@budibase/auth/tenancy") const { doInTenant } = require("@budibase/auth/tenancy")
const env = require("../environment")
const usage = require("../utilities/usageQuota")
const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
@ -80,7 +82,6 @@ class Orchestrator {
return stepFn({ return stepFn({
inputs: step.inputs, inputs: step.inputs,
appId: this._appId, appId: this._appId,
apiKey: automation.apiKey,
emitter: this._emitter, emitter: this._emitter,
context: this._context, context: this._context,
}) })
@ -95,6 +96,10 @@ class Orchestrator {
return err return err
} }
} }
// TODO: don't count test runs
if (!env.SELF_HOSTED) {
usage.update(usage.Properties.AUTOMATION, 1)
}
return this.executionOutput return this.executionOutput
} }
} }

6
packages/server/src/integrations/mysql.ts
View file

@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils"
module MySQLModule { module MySQLModule {
const mysql = require("mysql") const mysql = require("mysql")
const Sql = require("./base/sql") const Sql = require("./base/sql")
const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") const {
buildExternalTableId,
convertType,
copyExistingPropsOver,
} = require("./utils")
const { FieldTypes } = require("../constants") const { FieldTypes } = require("../constants")
interface MySQLConfig { interface MySQLConfig {

18
packages/server/src/integrations/postgres.ts
View file

@ -12,7 +12,11 @@ module PostgresModule {
const { Pool } = require("pg") const { Pool } = require("pg")
const Sql = require("./base/sql") const Sql = require("./base/sql")
const { FieldTypes } = require("../constants") const { FieldTypes } = require("../constants")
const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") const {
buildExternalTableId,
convertType,
copyExistingPropsOver,
} = require("./utils")
interface PostgresConfig { interface PostgresConfig {
host: string host: string
@ -179,10 +183,16 @@ module PostgresModule {
} }
const type: string = convertType(column.data_type, TYPE_MAP) const type: string = convertType(column.data_type, TYPE_MAP)
const identity = !!(column.identity_generation || column.identity_start || column.identity_increment) const identity = !!(
const hasDefault = typeof column.column_default === "string" && column.identity_generation ||
column.identity_start ||
column.identity_increment
)
const hasDefault =
typeof column.column_default === "string" &&
column.column_default.startsWith("nextval") column.column_default.startsWith("nextval")
const isGenerated = column.is_generated && column.is_generated !== "NEVER" const isGenerated =
column.is_generated && column.is_generated !== "NEVER"
const isAuto: boolean = hasDefault || identity || isGenerated const isAuto: boolean = hasDefault || identity || isGenerated
tables[tableName].schema[columnName] = { tables[tableName].schema[columnName] = {
autocolumn: isAuto, autocolumn: isAuto,

6
packages/server/src/integrations/utils.ts
View file

@ -84,7 +84,11 @@ export function isIsoDateString(str: string) {
} }
// add the existing relationships from the entities if they exist, to prevent them from being overridden // add the existing relationships from the entities if they exist, to prevent them from being overridden
export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) { export function copyExistingPropsOver(
tableName: string,
tables: { [key: string]: any },
entities: { [key: string]: any }
) {
if (entities && entities[tableName]) { if (entities && entities[tableName]) {
if (entities[tableName].primaryDisplay) { if (entities[tableName].primaryDisplay) {
tables[tableName].primaryDisplay = entities[tableName].primaryDisplay tables[tableName].primaryDisplay = entities[tableName].primaryDisplay

2
packages/server/src/middleware/usageQuota.js
View file

@ -1,6 +1,6 @@
const CouchDB = require("../db") const CouchDB = require("../db")
const usageQuota = require("../utilities/usageQuota") const usageQuota = require("../utilities/usageQuota")
const env = require("../environment") // const env = require("../environment")
// currently only counting new writes and deletes // currently only counting new writes and deletes
const METHOD_MAP = { const METHOD_MAP = {

45
packages/server/src/utilities/usageQuota.js
View file

@ -1,5 +1,4 @@
// const env = require("../environment")
const env = require("../environment")
const { getGlobalDB } = require("@budibase/auth/tenancy") const { getGlobalDB } = require("@budibase/auth/tenancy")
function getNewQuotaReset() { function getNewQuotaReset() {
@ -12,44 +11,44 @@ exports.Properties = {
VIEW: "views", VIEW: "views",
USER: "users", USER: "users",
AUTOMATION: "automationRuns", AUTOMATION: "automationRuns",
APPS: "apps" APPS: "apps",
} }
/** /**
* Given a specified tenantId this will add to the usage object for the specified property. * Given a specified tenantId this will add to the usage object for the specified property.
* @param {string} tenantId The tenant to update the usage quotas for.
* @param {string} property The property which is to be added to (within the nested usageQuota object). * @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by. * @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call. * also been reset after this call.
*/ */
exports.update = async (tenantId, property, usage) => { exports.update = async (property, usage) => {
// if (!env.USE_QUOTAS) { // if (!env.USE_QUOTAS) {
// return // return
// } // }
try { try {
const db = getGlobalDB() const db = getGlobalDB()
const quota = await db.get("usage_quota") const quota = await db.get("usage_quota")
// TODO: check if the quota needs reset // TODO: check if the quota needs reset
if (Date.now() >= quota.quotaReset) { if (Date.now() >= quota.quotaReset) {
quota.quotaReset = getNewQuotaReset() quota.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(quota.usageQuota)) { for (let prop of Object.keys(quota.usageQuota)) {
quota.usageQuota[prop] = 0 quota.usageQuota[prop] = 0
} }
} }
// increment the quota // increment the quota
quota.usageQuota[property] += usage quota.usageQuota[property] += usage
if (quota.usageQuota[property] >= quota.usageLimits[property]) { if (quota.usageQuota[property] >= quota.usageLimits[property]) {
throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`) throw new Error(
} `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
)
}
// update the usage quotas // update the usage quotas
await db.put(quota) await db.put(quota)
} catch (err) { } catch (err) {
console.error(`Error updating usage quotas for ${property}`, err) console.error(`Error updating usage quotas for ${property}`, err)
throw err throw err
} }
} }

196
packages/server/src/utilities/usageQuota.old.js
View file

@ -1,105 +1,105 @@
const env = require("../environment") // const env = require("../environment")
const { apiKeyTable } = require("../db/dynamoClient") // const { apiKeyTable } = require("../db/dynamoClient")
const DEFAULT_USAGE = { // const DEFAULT_USAGE = {
rows: 0, // rows: 0,
storage: 0, // storage: 0,
views: 0, // views: 0,
automationRuns: 0, // automationRuns: 0,
users: 0, // users: 0,
} // }
const DEFAULT_PLAN = { // const DEFAULT_PLAN = {
rows: 1000, // rows: 1000,
// 1 GB // // 1 GB
storage: 8589934592, // storage: 8589934592,
views: 10, // views: 10,
automationRuns: 100, // automationRuns: 100,
users: 10000, // users: 10000,
} // }
function buildUpdateParams(key, property, usage) { // function buildUpdateParams(key, property, usage) {
return { // return {
primary: key, // primary: key,
condition: // condition:
"attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", // "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now",
expression: "ADD #quota.#prop :usage", // expression: "ADD #quota.#prop :usage",
names: { // names: {
"#quota": "usageQuota", // "#quota": "usageQuota",
"#prop": property, // "#prop": property,
"#limits": "usageLimits", // "#limits": "usageLimits",
"#quotaReset": "quotaReset", // "#quotaReset": "quotaReset",
}, // },
values: { // values: {
":usage": usage, // ":usage": usage,
":now": Date.now(), // ":now": Date.now(),
}, // },
} // }
} // }
function getNewQuotaReset() { // function getNewQuotaReset() {
return Date.now() + 2592000000 // return Date.now() + 2592000000
} // }
exports.Properties = { // exports.Properties = {
ROW: "rows", // ROW: "rows",
UPLOAD: "storage", // UPLOAD: "storage",
VIEW: "views", // VIEW: "views",
USER: "users", // USER: "users",
AUTOMATION: "automationRuns", // AUTOMATION: "automationRuns",
} // }
exports.getAPIKey = async appId => { // exports.getAPIKey = async appId => {
if (!env.USE_QUOTAS) { // if (!env.USE_QUOTAS) {
return { apiKey: null } // return { apiKey: null }
} // }
return apiKeyTable.get({ primary: appId }) // return apiKeyTable.get({ primary: appId })
} // }
/** // /**
* Given a specified API key this will add to the usage object for the specified property. // * Given a specified API key this will add to the usage object for the specified property.
* @param {string} apiKey The API key which is to be updated. // * @param {string} apiKey The API key which is to be updated.
* @param {string} property The property which is to be added to (within the nested usageQuota object). // * @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by. // * @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have // * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call. // * also been reset after this call.
*/ // */
exports.update = async (apiKey, property, usage) => { // exports.update = async (apiKey, property, usage) => {
if (!env.USE_QUOTAS) { // if (!env.USE_QUOTAS) {
return // return
} // }
try { // try {
await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) // await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
} catch (err) { // } catch (err) {
// conditional check means the condition failed, need to check why // // conditional check means the condition failed, need to check why
if (err.code === "ConditionalCheckFailedException") { // if (err.code === "ConditionalCheckFailedException") {
// get the API key so we can check it // // get the API key so we can check it
const keyObj = await apiKeyTable.get({ primary: apiKey }) // const keyObj = await apiKeyTable.get({ primary: apiKey })
// the usage quota or usage limits didn't exist // // the usage quota or usage limits didn't exist
if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { // if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
keyObj.usageQuota = // keyObj.usageQuota =
keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota // keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
keyObj.usageLimits = // keyObj.usageLimits =
keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits // keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
keyObj.quotaReset = getNewQuotaReset() // keyObj.quotaReset = getNewQuotaReset()
await apiKeyTable.put({ item: keyObj }) // await apiKeyTable.put({ item: keyObj })
return // return
} // }
// we have in fact breached the reset period // // we have in fact breached the reset period
else if (keyObj && keyObj.quotaReset <= Date.now()) { // else if (keyObj && keyObj.quotaReset <= Date.now()) {
// update the quota reset period and reset the values for all properties // // update the quota reset period and reset the values for all properties
keyObj.quotaReset = getNewQuotaReset() // keyObj.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(keyObj.usageQuota)) { // for (let prop of Object.keys(keyObj.usageQuota)) {
if (prop === property) { // if (prop === property) {
keyObj.usageQuota[prop] = usage > 0 ? usage : 0 // keyObj.usageQuota[prop] = usage > 0 ? usage : 0
} else { // } else {
keyObj.usageQuota[prop] = 0 // keyObj.usageQuota[prop] = 0
} // }
} // }
await apiKeyTable.put({ item: keyObj }) // await apiKeyTable.put({ item: keyObj })
return // return
} // }
} // }
throw err // throw err
} // }
} // }

43
packages/worker/src/api/controllers/global/users.js
View file

@ -1,7 +1,6 @@
const { const {
generateGlobalUserID, generateGlobalUserID,
getGlobalUserParams, getGlobalUserParams,
StaticDatabases, StaticDatabases,
} = require("@budibase/auth/db") } = require("@budibase/auth/db")
const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils
@ -18,7 +17,7 @@ const {
tryAddTenant, tryAddTenant,
updateTenantId, updateTenantId,
} = require("@budibase/auth/tenancy") } = require("@budibase/auth/tenancy")
const env = require("../../../environment") // const env = require("../../../environment")
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
@ -139,28 +138,28 @@ exports.adminUser = async ctx => {
include_docs: true, include_docs: true,
}) })
) )
// write usage quotas for cloud // write usage quotas for cloud
// if (!env.SELF_HOSTED) { // if (!env.SELF_HOSTED) {
await db.post({ await db.post({
_id: "usage_quota", _id: "usage_quota",
quotaReset: Date.now() + 2592000000, quotaReset: Date.now() + 2592000000,
usageQuota: { usageQuota: {
automationRuns: 0, automationRuns: 0,
rows: 0, rows: 0,
storage: 0, storage: 0,
apps: 0, apps: 0,
users: 0, users: 0,
views: 0, views: 0,
}, },
usageLimits: { usageLimits: {
automationRuns: 1000, automationRuns: 1000,
rows: 4000, rows: 4000,
apps: 4, apps: 4,
// storage: 1000, storage: 1000,
// users: 10 users: 10
}, },
}) })
// } // }
if (response.rows.some(row => row.doc.admin)) { if (response.rows.some(row => row.doc.admin)) {