diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 8086c0ab20..cd43631992 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -99,7 +99,9 @@ spec: - name: PLATFORM_URL value: {{ .Values.globals.platformUrl | quote }} - name: USE_QUOTAS - value: "1" + value: {{ .Values.globals.useQuotas | quote }} + - name: EXCLUDE_QUOTAS_TENANTS + value: {{ .Values.globals.excludeQuotasTenants | quote }} - name: ACCOUNT_PORTAL_URL value: {{ .Values.globals.accountPortalUrl | quote }} - name: ACCOUNT_PORTAL_API_KEY diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 4666d01c70..9ea055c6c0 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -93,6 +93,8 @@ globals: logLevel: info selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs + useQuotas: "0" + excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas accountPortalUrl: "" accountPortalApiKey: "" cookieDomain: "" @@ -239,7 +241,8 @@ couchdb: hosts: - chart-example.local path: / - annotations: [] + annotations: + [] # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" tls: diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js index 30e04e198b..6b8eb3a95c 100644 --- a/packages/backend-core/src/migrations/index.js +++ b/packages/backend-core/src/migrations/index.js @@ -1,5 +1,5 @@ const { DocumentTypes } = require("../db/constants") -const { getGlobalDB } = require("../tenancy") +const { getGlobalDB, getTenantId } = require("../tenancy") exports.MIGRATION_DBS = { GLOBAL_DB: "GLOBAL_DB", @@ -7,13 +7,13 @@ exports.MIGRATION_DBS = { exports.MIGRATIONS = { USER_EMAIL_VIEW_CASING: "user_email_view_casing", - SYNC_APP_AND_RESET_ROWS_QUOTAS: "sync_app_and_reset_rows_quotas", + QUOTAS_1: "quotas_1", } const DB_LOOKUP = { [exports.MIGRATION_DBS.GLOBAL_DB]: [ exports.MIGRATIONS.USER_EMAIL_VIEW_CASING, - exports.MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS, + exports.MIGRATIONS.QUOTAS_1, ], } @@ -29,6 +29,7 @@ exports.getMigrationsDoc = async db => { } exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => { + const tenantId = getTenantId() try { let db if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) { @@ -49,15 +50,18 @@ exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => { return } - console.log(`Performing migration: ${migrationName}`) + console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`) await migrateFn() - console.log(`Migration complete: ${migrationName}`) + console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`) // mark as complete doc[migrationName] = Date.now() await db.put(doc) } catch (err) { - console.error(`Error performing migration: ${migrationName}: `, err) + console.error( + `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `, + err + ) throw err } } diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index 997864e165..a3b7ca81a6 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -53,16 +53,23 @@ } // Create table - const table = await tables.save(newTable) - notifications.success(`Table ${name} created successfully.`) - analytics.captureEvent(Events.TABLE.CREATED, { name }) + let table + try { + table = await tables.save(newTable) + notifications.success(`Table ${name} created successfully.`) + analytics.captureEvent(Events.TABLE.CREATED, { name }) - // Navigate to new table - const currentUrl = $url() - const path = currentUrl.endsWith("data") - ? `./table/${table._id}` - : `../../table/${table._id}` - $goto(path) + // Navigate to new table + const currentUrl = $url() + const path = currentUrl.endsWith("data") + ? `./table/${table._id}` + : `../../table/${table._id}` + $goto(path) + } catch (e) { + notifications.error(e) + // reload in case the table was created + await tables.fetch() + } } diff --git a/packages/server/src/api/controllers/table/internal.js b/packages/server/src/api/controllers/table/internal.js index 10a5c9746a..9f09e78219 100644 --- a/packages/server/src/api/controllers/table/internal.js +++ b/packages/server/src/api/controllers/table/internal.js @@ -8,6 +8,7 @@ const { getTable, handleDataImport, } = require("./utils") +const usageQuota = require("../../../utilities/usageQuota") exports.save = async function (ctx) { const appId = ctx.appId @@ -119,6 +120,7 @@ exports.destroy = async function (ctx) { }) ) await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true }))) + await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length) // update linked rows await linkRows.updateLinks({ diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index e4086e8071..e0311389e6 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -15,6 +15,7 @@ const { } = require("../../../integrations/utils") const { getViews, saveView } = require("../view/utils") const viewTemplate = require("../view/viewBuilder") +const usageQuota = require("../../../utilities/usageQuota") exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => { let updatedRows = [] @@ -111,7 +112,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => { finalData.push(row) } + await usageQuota.update(usageQuota.Properties.ROW, finalData.length, { + dryRun: true, + }) await db.bulkDocs(finalData) + await usageQuota.update(usageQuota.Properties.ROW, finalData.length) let response = await db.put(table) table._rev = response._rev return table diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index 8e5b44cc06..1937121062 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -1,6 +1,5 @@ const rowController = require("../../api/controllers/row") const automationUtils = require("../automationUtils") -const env = require("../../environment") const usage = require("../../utilities/usageQuota") const { buildCtx } = require("./utils") @@ -83,10 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) { inputs.row.tableId, inputs.row ) - if (env.USE_QUOTAS) { - await usage.update(usage.Properties.ROW, 1) - } + await usage.update(usage.Properties.ROW, 1, { dryRun: true }) await rowController.save(ctx) + await usage.update(usage.Properties.ROW, 1) return { row: inputs.row, response: ctx.body, diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js index c7bee577a5..e41e5ad263 100644 --- a/packages/server/src/automations/steps/deleteRow.js +++ b/packages/server/src/automations/steps/deleteRow.js @@ -1,5 +1,4 @@ const rowController = require("../../api/controllers/row") -const env = require("../../environment") const usage = require("../../utilities/usageQuota") const { buildCtx } = require("./utils") const automationUtils = require("../automationUtils") @@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) { }) try { - if (env.isProd()) { - await usage.update(usage.Properties.ROW, -1) - } + await usage.update(usage.Properties.ROW, -1) await rowController.destroy(ctx) return { response: ctx.body, diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index a92e113851..614f41a29f 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -38,6 +38,7 @@ module.exports = { MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, USE_QUOTAS: process.env.USE_QUOTAS, + EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS, REDIS_URL: process.env.REDIS_URL, REDIS_PASSWORD: process.env.REDIS_PASSWORD, INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 25b439fd58..273f221575 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -38,7 +38,7 @@ module S3Module { signatureVersion: { type: "string", required: false, - default: "v4" + default: "v4", }, }, query: { diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js index 2e92eb9957..1282615a50 100644 --- a/packages/server/src/middleware/tests/usageQuota.spec.js +++ b/packages/server/src/middleware/tests/usageQuota.spec.js @@ -1,11 +1,5 @@ jest.mock("../../db") jest.mock("../../utilities/usageQuota") -jest.mock("../../environment", () => ({ - isTest: () => true, - isProd: () => false, - isDev: () => true, - _set: () => {}, -})) jest.mock("@budibase/backend-core/tenancy", () => ({ getTenantId: () => "testing123" })) @@ -32,6 +26,7 @@ class TestConfiguration { url: "/applications" } } + usageQuota.useQuotas = () => true } executeMiddleware() { @@ -113,12 +108,10 @@ describe("usageQuota middleware", () => { it("calculates and persists the correct usage quota for the relevant action", async () => { config.setUrl("/rows") - config.setProd(true) await config.executeMiddleware() - // expect(usageQuota.update).toHaveBeenCalledWith("rows", 1) - expect(usageQuota.update).not.toHaveBeenCalledWith("rows", 1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", 1) expect(config.next).toHaveBeenCalled() }) diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index bce13816d1..4bafa75132 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,17 +1,11 @@ const CouchDB = require("../db") const usageQuota = require("../utilities/usageQuota") -const env = require("../environment") -const { getTenantId } = require("@budibase/backend-core/tenancy") +const { getUniqueRows } = require("../utilities/usageQuota/rows") const { isExternalTable, isRowId: isExternalRowId, } = require("../integrations/utils") -const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas") - -const testing = false - -// tenants without limits -const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"] +const migration = require("../migrations/usageQuotas") // currently only counting new writes and deletes const METHOD_MAP = { @@ -20,7 +14,7 @@ const METHOD_MAP = { } const DOMAIN_MAP = { - // rows: usageQuota.Properties.ROW, // works - disabled + rows: usageQuota.Properties.ROW, // upload: usageQuota.Properties.UPLOAD, // doesn't work yet // views: usageQuota.Properties.VIEW, // doesn't work yet // users: usageQuota.Properties.USER, // doesn't work yet @@ -39,13 +33,7 @@ function getProperty(url) { } module.exports = async (ctx, next) => { - const tenantId = getTenantId() - - // if in development or a self hosted cloud usage quotas should not be executed - if ( - (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) && - !testing - ) { + if (!usageQuota.useQuotas()) { return next() } @@ -86,10 +74,93 @@ module.exports = async (ctx, next) => { usage = files.map(file => file.size).reduce((total, size) => total + size) } try { - await quotaMigration.runIfRequired() - await usageQuota.update(property, usage) - return next() + await migration.run() + await performRequest(ctx, next, property, usage) } catch (err) { ctx.throw(400, err) } } + +const performRequest = async (ctx, next, property, usage) => { + const usageContext = { + skipNext: false, + skipUsage: false, + [usageQuota.Properties.APPS]: {}, + } + + if (usage === -1) { + if (PRE_DELETE[property]) { + await PRE_DELETE[property](ctx, usageContext) + } + } else { + if (PRE_CREATE[property]) { + await PRE_CREATE[property](ctx, usageContext) + } + } + + // run the request + if (!usageContext.skipNext) { + await usageQuota.update(property, usage, { dryRun: true }) + await next() + } + + if (usage === -1) { + if (POST_DELETE[property]) { + await POST_DELETE[property](ctx, usageContext) + } + } else { + if (POST_CREATE[property]) { + await POST_CREATE[property](ctx, usageContext) + } + } + + // update the usage + if (!usageContext.skipUsage) { + await usageQuota.update(property, usage) + } +} + +const appPreDelete = async (ctx, usageContext) => { + if (ctx.query.unpublish) { + // don't run usage decrement for unpublish + usageContext.skipUsage = true + return + } + + // store the row count to delete + const rows = await getUniqueRows([ctx.appId]) + if (rows.length) { + usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length } + } +} + +const appPostDelete = async (ctx, usageContext) => { + // delete the app rows from usage + const rowCount = usageContext[usageQuota.Properties.APPS].rowCount + if (rowCount) { + await usageQuota.update(usageQuota.Properties.ROW, -rowCount) + } +} + +const appPostCreate = async ctx => { + // app import & template creation + if (ctx.request.body.useTemplate === "true") { + const rows = await getUniqueRows([ctx.response.body.appId]) + const rowCount = rows ? rows.length : 0 + await usageQuota.update(usageQuota.Properties.ROW, rowCount) + } +} + +const PRE_DELETE = { + [usageQuota.Properties.APPS]: appPreDelete, +} + +const POST_DELETE = { + [usageQuota.Properties.APPS]: appPostDelete, +} + +const PRE_CREATE = {} + +const POST_CREATE = { + [usageQuota.Properties.APPS]: appPostCreate, +} diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js deleted file mode 100644 index 23651ec9b7..0000000000 --- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js +++ /dev/null @@ -1,30 +0,0 @@ -const { - MIGRATIONS, - MIGRATION_DBS, - migrateIfRequired, -} = require("@budibase/backend-core/migrations") -const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const { getAllApps } = require("@budibase/backend-core/db") -const CouchDB = require("../db") -const { getUsageQuotaDoc } = require("../utilities/usageQuota") - -exports.runIfRequired = async () => { - await migrateIfRequired( - MIGRATION_DBS.GLOBAL_DB, - MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS, - async () => { - const db = getGlobalDB() - const usageDoc = await getUsageQuotaDoc(db) - - // reset the rows - usageDoc.usageQuota.rows = 0 - - // sync the apps - const apps = await getAllApps(CouchDB, { dev: true }) - const appCount = apps ? apps.length : 0 - usageDoc.usageQuota.apps = appCount - - await db.put(usageDoc) - } - ) -} diff --git a/packages/server/src/migrations/tests/usageQuotas/index.spec.js b/packages/server/src/migrations/tests/usageQuotas/index.spec.js new file mode 100644 index 0000000000..0c5b982909 --- /dev/null +++ b/packages/server/src/migrations/tests/usageQuotas/index.spec.js @@ -0,0 +1,27 @@ +const env = require("../../../environment") +const TestConfig = require("../../../tests/utilities/TestConfiguration") + +const syncApps = jest.fn() +const syncRows = jest.fn() + +jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) +jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) + +const migrations = require("../../usageQuotas") + +describe("run", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + env._set("USE_QUOTAS", 1) + }) + + afterAll(config.end) + + it("runs the required migrations", async () => { + await migrations.run() + expect(syncApps).toHaveBeenCalledTimes(1) + expect(syncRows).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js similarity index 59% rename from packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js rename to packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js index cd533e3f06..160319a31b 100644 --- a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js +++ b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js @@ -1,10 +1,10 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const TestConfig = require("../../tests/utilities/TestConfiguration") -const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota") -const { runIfRequired } = require("../sync_app_and_reset_rows_quotas") -const env = require("../../environment") +const TestConfig = require("../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") +const syncApps = require("../../usageQuotas/syncApps") +const env = require("../../../environment") -describe("Sync App And Reset Rows Quotas Migration", () => { +describe("syncApps", () => { let config = new TestConfig(false) beforeEach(async () => { @@ -12,28 +12,26 @@ describe("Sync App And Reset Rows Quotas Migration", () => { env._set("USE_QUOTAS", 1) }) - afterAll(config.end) + afterAll(config.end) - it("migrates successfully", async () => { + it("runs successfully", async () => { // create the usage quota doc and mock usages const db = getGlobalDB() await getUsageQuotaDoc(db) await update(Properties.APPS, 3) - await update(Properties.ROW, 300) let usageDoc = await getUsageQuotaDoc(db) expect(usageDoc.usageQuota.apps).toEqual(3) - expect(usageDoc.usageQuota.rows).toEqual(300) // create an extra app to test the migration await config.createApp("quota-test") // migrate - await runIfRequired() + await syncApps.run() // assert the migration worked usageDoc = await getUsageQuotaDoc(db) expect(usageDoc.usageQuota.apps).toEqual(2) - expect(usageDoc.usageQuota.rows).toEqual(0) }) }) + diff --git a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js new file mode 100644 index 0000000000..a09bea60bd --- /dev/null +++ b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js @@ -0,0 +1,43 @@ +const { getGlobalDB } = require("@budibase/backend-core/tenancy") +const TestConfig = require("../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") +const syncRows = require("../../usageQuotas/syncRows") +const env = require("../../../environment") + +describe("syncRows", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + env._set("USE_QUOTAS", 1) + }) + + afterAll(config.end) + + it("runs successfully", async () => { + // create the usage quota doc and mock usages + const db = getGlobalDB() + await getUsageQuotaDoc(db) + await update(Properties.ROW, 300) + + let usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.rows).toEqual(300) + + // app 1 + await config.createTable() + await config.createRow() + // app 2 + await config.createApp() + await config.createTable() + await config.createRow() + await config.createRow() + + // migrate + await syncRows.run() + + // assert the migration worked + usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.rows).toEqual(3) + }) +}) + diff --git a/packages/server/src/migrations/usageQuotas/index.js b/packages/server/src/migrations/usageQuotas/index.js new file mode 100644 index 0000000000..39744093c2 --- /dev/null +++ b/packages/server/src/migrations/usageQuotas/index.js @@ -0,0 +1,24 @@ +const { + MIGRATIONS, + MIGRATION_DBS, + migrateIfRequired, +} = require("@budibase/backend-core/migrations") +const { useQuotas } = require("../../utilities/usageQuota") +const syncApps = require("./syncApps") +const syncRows = require("./syncRows") + +exports.run = async () => { + if (!useQuotas()) { + return + } + + // Jan 2022 + await migrateIfRequired( + MIGRATION_DBS.GLOBAL_DB, + MIGRATIONS.QUOTAS_1, + async () => { + await syncApps.run() + await syncRows.run() + } + ) +} diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/usageQuotas/syncApps.js new file mode 100644 index 0000000000..ee106129e6 --- /dev/null +++ b/packages/server/src/migrations/usageQuotas/syncApps.js @@ -0,0 +1,18 @@ +const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") +const { getAllApps } = require("@budibase/backend-core/db") +const CouchDB = require("../../db") +const { getUsageQuotaDoc } = require("../../utilities/usageQuota") + +exports.run = async () => { + const db = getGlobalDB() + // get app count + const devApps = await getAllApps(CouchDB, { dev: true }) + const appCount = devApps ? devApps.length : 0 + + // sync app count + const tenantId = getTenantId() + console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`) + const usageDoc = await getUsageQuotaDoc(db) + usageDoc.usageQuota.apps = appCount + await db.put(usageDoc) +} diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/usageQuotas/syncRows.js new file mode 100644 index 0000000000..7990f405de --- /dev/null +++ b/packages/server/src/migrations/usageQuotas/syncRows.js @@ -0,0 +1,21 @@ +const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") +const { getAllApps } = require("@budibase/backend-core/db") +const CouchDB = require("../../db") +const { getUsageQuotaDoc } = require("../../utilities/usageQuota") +const { getUniqueRows } = require("../../utilities/usageQuota/rows") + +exports.run = async () => { + const db = getGlobalDB() + // get all rows in all apps + const allApps = await getAllApps(CouchDB, { all: true }) + const appIds = allApps ? allApps.map(app => app.appId) : [] + const rows = await getUniqueRows(appIds) + const rowCount = rows ? rows.length : 0 + + // sync row count + const tenantId = getTenantId() + console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`) + const usageDoc = await getUsageQuotaDoc(db) + usageDoc.usageQuota.rows = rowCount + await db.put(usageDoc) +} diff --git a/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js new file mode 100644 index 0000000000..dcd7578f59 --- /dev/null +++ b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js @@ -0,0 +1,72 @@ +const getTenantId = jest.fn() +jest.mock("@budibase/backend-core/tenancy", () => ({ + getTenantId +})) +const usageQuota = require("../../usageQuota") +const env = require("../../../environment") + +class TestConfiguration { + constructor() { + this.enableQuotas() + } + + enableQuotas = () => { + env.USE_QUOTAS = 1 + } + + disableQuotas = () => { + env.USE_QUOTAS = null + } + + setTenantId = (tenantId) => { + getTenantId.mockReturnValue(tenantId) + } + + setExcludedTenants = (tenants) => { + env.EXCLUDE_QUOTAS_TENANTS = tenants + } + + reset = () => { + this.disableQuotas() + this.setExcludedTenants(null) + } +} + +describe("usageQuota", () => { + let config + + beforeEach(() => { + config = new TestConfiguration() + }) + + afterEach(() => { + config.reset() + }) + + describe("useQuotas", () => { + it("works when no settings have been provided", () => { + config.reset() + expect(usageQuota.useQuotas()).toBe(false) + }) + it("honours USE_QUOTAS setting", () => { + config.disableQuotas() + expect(usageQuota.useQuotas()).toBe(false) + + config.enableQuotas() + expect(usageQuota.useQuotas()).toBe(true) + }) + it("honours EXCLUDE_QUOTAS_TENANTS setting", () => { + config.setTenantId("test") + + // tenantId is in the list + config.setExcludedTenants("test, test2, test2") + expect(usageQuota.useQuotas()).toBe(false) + config.setExcludedTenants("test,test2,test2") + expect(usageQuota.useQuotas()).toBe(false) + + // tenantId is not in the list + config.setTenantId("other") + expect(usageQuota.useQuotas()).toBe(true) + }) + }) +}) \ No newline at end of file diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota/index.js similarity index 58% rename from packages/server/src/utilities/usageQuota.js rename to packages/server/src/utilities/usageQuota/index.js index 4d88b610e0..b0ff310aa3 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota/index.js @@ -1,17 +1,36 @@ -const env = require("../environment") -const { getGlobalDB } = require("@budibase/backend-core/tenancy") +const env = require("../../environment") +const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") const { StaticDatabases, generateNewUsageQuotaDoc, } = require("@budibase/backend-core/db") +exports.useQuotas = () => { + // check if quotas are enabled + if (env.USE_QUOTAS) { + // check if there are any tenants without limits + if (env.EXCLUDE_QUOTAS_TENANTS) { + const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace( + /\s/g, + "" + ).split(",") + const tenantId = getTenantId() + if (excludedTenants.includes(tenantId)) { + return false + } + } + return true + } + return false +} + exports.Properties = { - ROW: "rows", // mostly works - disabled - app / table deletion not yet accounted for + ROW: "rows", UPLOAD: "storage", // doesn't work yet VIEW: "views", // doesn't work yet USER: "users", // doesn't work yet AUTOMATION: "automationRuns", // doesn't work yet - APPS: "apps", // works + APPS: "apps", EMAILS: "emails", // doesn't work yet } @@ -36,8 +55,8 @@ exports.getUsageQuotaDoc = async db => { * @returns {Promise} When this completes the API key will now be up to date - the quota period may have * also been reset after this call. */ -exports.update = async (property, usage) => { - if (!env.USE_QUOTAS) { +exports.update = async (property, usage, opts = { dryRun: false }) => { + if (!exports.useQuotas()) { return } @@ -48,14 +67,24 @@ exports.update = async (property, usage) => { // increment the quota quota.usageQuota[property] += usage - if (quota.usageQuota[property] > quota.usageLimits[property]) { + if ( + quota.usageQuota[property] > quota.usageLimits[property] && + usage > 0 // allow for decrementing usage when the quota is already exceeded + ) { throw new Error( `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` ) } + if (quota.usageQuota[property] < 0) { + // never go negative if the quota has previously been exceeded + quota.usageQuota[property] = 0 + } + // update the usage quotas - await db.put(quota) + if (!opts.dryRun) { + await db.put(quota) + } } catch (err) { console.error(`Error updating usage quotas for ${property}`, err) throw err diff --git a/packages/server/src/utilities/usageQuota/rows.js b/packages/server/src/utilities/usageQuota/rows.js new file mode 100644 index 0000000000..67ad07410d --- /dev/null +++ b/packages/server/src/utilities/usageQuota/rows.js @@ -0,0 +1,74 @@ +const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils") +const CouchDB = require("../../db") +const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db") + +const ROW_EXCLUSIONS = [USER_METDATA_PREFIX] + +const getAppPairs = appIds => { + // collect the app ids into dev / prod pairs + // keyed by the dev app id + const pairs = {} + for (let appId of appIds) { + const devId = getDevelopmentAppID(appId) + if (!pairs[devId]) { + pairs[devId] = {} + } + if (isDevAppID(appId)) { + pairs[devId].devId = appId + } else { + pairs[devId].prodId = appId + } + } + return pairs +} + +const getAppRows = async appId => { + const appDb = new CouchDB(appId) + const response = await appDb.allDocs( + getRowParams(null, null, { + include_docs: false, + }) + ) + return response.rows + .map(r => r.id) + .filter(id => { + for (let exclusion of ROW_EXCLUSIONS) { + if (id.startsWith(exclusion)) { + return false + } + } + return true + }) +} + +/** + * Return a set of all rows in the given app ids. + * The returned rows will be unique on a per dev/prod app basis. + * Rows duplicates may exist across apps due to data import so they are not filtered out. + */ +exports.getUniqueRows = async appIds => { + let uniqueRows = [] + const pairs = getAppPairs(appIds) + + for (let pair of Object.values(pairs)) { + let appRows = [] + for (let appId of [pair.devId, pair.prodId]) { + if (!appId) { + continue + } + try { + appRows.push(await getAppRows(appId)) + } catch (e) { + console.error(e) + // don't error out if we can't count the app rows, just continue + } + } + + // ensure uniqueness on a per app pair basis + // this can't be done on all rows because app import results in + // duplicate row ids across apps + uniqueRows = uniqueRows.concat(...new Set(appRows)) + } + + return uniqueRows +} diff --git a/packages/server/src/utilities/usageQuoteReset.js b/packages/server/src/utilities/usageQuota/usageQuoteReset.js similarity index 100% rename from packages/server/src/utilities/usageQuoteReset.js rename to packages/server/src/utilities/usageQuota/usageQuoteReset.js