From 61aafa87cb1243c91696c6dc08541507497c706e Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 28 Sep 2022 13:13:37 +0100 Subject: [PATCH] Create generic quota sync migration --- .../src/migrations/definitions.ts | 6 +--- .../src/migrations/functions/pluginCount.ts | 12 -------- .../src/migrations/functions/quotas2.ts | 20 ------------- .../src/migrations/functions/syncQuotas.ts | 15 ++++++++++ .../functions/usageQuotas/syncAppRows.ts | 29 ------------------- .../functions/usageQuotas/syncApps.ts | 1 - .../functions/usageQuotas/syncPlugins.ts | 10 +++++++ .../functions/usageQuotas/syncRows.ts | 21 ++++++++++---- packages/server/src/migrations/index.ts | 18 ++---------- .../server/src/migrations/tests/index.spec.ts | 1 - packages/types/src/sdk/migrations.ts | 5 ++-- 11 files changed, 46 insertions(+), 92 deletions(-) delete mode 100644 packages/server/src/migrations/functions/pluginCount.ts delete mode 100644 packages/server/src/migrations/functions/quotas2.ts create mode 100644 packages/server/src/migrations/functions/syncQuotas.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncAppRows.ts create mode 100644 packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts index 946fc3f364..6eba56ab43 100644 --- a/packages/backend-core/src/migrations/definitions.ts +++ b/packages/backend-core/src/migrations/definitions.ts @@ -11,7 +11,7 @@ export const DEFINITIONS: MigrationDefinition[] = [ }, { type: MigrationType.GLOBAL, - name: MigrationName.QUOTAS_1, + name: MigrationName.SYNC_QUOTAS, }, { type: MigrationType.APP, @@ -33,8 +33,4 @@ export const DEFINITIONS: MigrationDefinition[] = [ type: MigrationType.GLOBAL, name: MigrationName.GLOBAL_INFO_SYNC_USERS, }, - { - type: MigrationType.GLOBAL, - name: MigrationName.PLUGIN_COUNT, - }, ] diff --git a/packages/server/src/migrations/functions/pluginCount.ts b/packages/server/src/migrations/functions/pluginCount.ts deleted file mode 100644 index a9dff9d7c8..0000000000 --- a/packages/server/src/migrations/functions/pluginCount.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { tenancy, logging } from "@budibase/backend-core" -import { plugins } from "@budibase/pro" - -export const run = async () => { - try { - await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => { - await plugins.checkPluginQuotas() - }) - } catch (err) { - logging.logAlert("Failed to update plugin quotas", err) - } -} diff --git a/packages/server/src/migrations/functions/quotas2.ts b/packages/server/src/migrations/functions/quotas2.ts deleted file mode 100644 index 457a23840a..0000000000 --- a/packages/server/src/migrations/functions/quotas2.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { runQuotaMigration } from "./usageQuotas" -import * as syncApps from "./usageQuotas/syncApps" -import * as syncAppRows from "./usageQuotas/syncAppRows" - -/** - * Date: - * January 2022 - * - * Description: - * Synchronise the app and row quotas to the state of the db after it was - * discovered that the quota resets were still in place and the row quotas - * weren't being decremented correctly. - */ - -export const run = async () => { - await runQuotaMigration(async () => { - await syncApps.run() - await syncAppRows.run() - }) -} diff --git a/packages/server/src/migrations/functions/syncQuotas.ts b/packages/server/src/migrations/functions/syncQuotas.ts new file mode 100644 index 0000000000..85aa7bb337 --- /dev/null +++ b/packages/server/src/migrations/functions/syncQuotas.ts @@ -0,0 +1,15 @@ +import { runQuotaMigration } from "./usageQuotas" +import * as syncApps from "./usageQuotas/syncApps" +import * as syncRows from "./usageQuotas/syncRows" +import * as syncPlugins from "./usageQuotas/syncPlugins" + +/** + * Synchronise quotas to the state of the db. + */ +export const run = async () => { + await runQuotaMigration(async () => { + await syncApps.run() + await syncRows.run() + await syncPlugins.run() + }) +} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncAppRows.ts b/packages/server/src/migrations/functions/usageQuotas/syncAppRows.ts deleted file mode 100644 index b9d0cfa3da..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncAppRows.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { getTenantId } from "@budibase/backend-core/tenancy" -import { getAllApps } from "@budibase/backend-core/db" -import { getUniqueRows } from "../../../utilities/usageQuota/rows" -import { quotas } from "@budibase/pro" -import { StaticQuotaName, QuotaUsageType } from "@budibase/types" - -export const run = async () => { - // get all rows in all apps - // @ts-ignore - const allApps = await getAllApps({ all: true }) - // @ts-ignore - const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : [] - const { appRows } = await getUniqueRows(appIds) - const counts: { [key: string]: number } = {} - let rowCount = 0 - Object.entries(appRows).forEach(([appId, rows]) => { - counts[appId] = rows.length - rowCount += rows.length - }) - - // sync row count - const tenantId = getTenantId() - console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`) - await quotas.setUsagePerApp( - counts, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts index 4770844a99..76e2dbdacd 100644 --- a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts +++ b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts @@ -5,7 +5,6 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types" export const run = async () => { // get app count - // @ts-ignore const devApps = await getAllApps({ dev: true }) const appCount = devApps ? devApps.length : 0 diff --git a/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts b/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts new file mode 100644 index 0000000000..b00970aea2 --- /dev/null +++ b/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts @@ -0,0 +1,10 @@ +import { logging } from "@budibase/backend-core" +import { plugins } from "@budibase/pro" + +export const run = async () => { + try { + await plugins.checkPluginQuotas() + } catch (err) { + logging.logAlert("Failed to update plugin quotas", err) + } +} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts index c4ddab1d64..0b123d2357 100644 --- a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts +++ b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts @@ -2,19 +2,28 @@ import { getTenantId } from "@budibase/backend-core/tenancy" import { getAllApps } from "@budibase/backend-core/db" import { getUniqueRows } from "../../../utilities/usageQuota/rows" import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" +import { StaticQuotaName, QuotaUsageType } from "@budibase/types" export const run = async () => { // get all rows in all apps - // @ts-ignore const allApps = await getAllApps({ all: true }) - // @ts-ignore const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : [] - const { rows } = await getUniqueRows(appIds) - const rowCount = rows ? rows.length : 0 + const { appRows } = await getUniqueRows(appIds) + + // get the counts per app + const counts: { [key: string]: number } = {} + let rowCount = 0 + Object.entries(appRows).forEach(([appId, rows]) => { + counts[appId] = rows.length + rowCount += rows.length + }) // sync row count const tenantId = getTenantId() console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`) - await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC) + await quotas.setUsagePerApp( + counts, + StaticQuotaName.ROWS, + QuotaUsageType.STATIC + ) } diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index 169b9a580c..cb1e6d1c82 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -4,11 +4,9 @@ import env from "../environment" // migration functions import * as userEmailViewCasing from "./functions/userEmailViewCasing" -import * as quota2 from "./functions/quotas2" +import * as syncQuotas from "./functions/syncQuotas" import * as appUrls from "./functions/appUrls" import * as backfill from "./functions/backfill" -import * as pluginCount from "./functions/pluginCount" - /** * Populate the migration function and additional configuration from * the static migration definitions. @@ -26,10 +24,10 @@ export const buildMigrations = () => { }) break } - case MigrationName.QUOTAS_1: { + case MigrationName.SYNC_QUOTAS: { serverMigrations.push({ ...definition, - fn: quota2.run, + fn: syncQuotas.run, }) break } @@ -69,16 +67,6 @@ export const buildMigrations = () => { }) break } - case MigrationName.PLUGIN_COUNT: { - if (env.SELF_HOSTED) { - serverMigrations.push({ - ...definition, - fn: pluginCount.run, - silent: !!env.SELF_HOSTED, - preventRetry: false, - }) - } - } } } diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts index e2c2fb5c96..8effaec52b 100644 --- a/packages/server/src/migrations/tests/index.spec.ts +++ b/packages/server/src/migrations/tests/index.spec.ts @@ -4,7 +4,6 @@ import { tenancy, DocumentType, context, - db, } from "@budibase/backend-core" import TestConfig from "../../tests/utilities/TestConfiguration" import structures from "../../tests/utilities/structures" diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts index bf12c6c0ea..4f5315d003 100644 --- a/packages/types/src/sdk/migrations.ts +++ b/packages/types/src/sdk/migrations.ts @@ -39,14 +39,13 @@ export interface MigrationOptions { export enum MigrationName { USER_EMAIL_VIEW_CASING = "user_email_view_casing", - QUOTAS_1 = "quotas_1", APP_URLS = "app_urls", EVENT_APP_BACKFILL = "event_app_backfill", EVENT_GLOBAL_BACKFILL = "event_global_backfill", EVENT_INSTALLATION_BACKFILL = "event_installation_backfill", GLOBAL_INFO_SYNC_USERS = "global_info_sync_users", - PLATFORM_USERS_EMAIL_CASING = "platform_users_email_casing", - PLUGIN_COUNT = "plugin_count", + // increment this number to re-activate this migration + SYNC_QUOTAS = "sync_quotas_1", } export interface MigrationDefinition {