diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index cd43631992..d9def8c641 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -108,6 +108,8 @@ spec: value: {{ .Values.globals.accountPortalApiKey | quote }} - name: COOKIE_DOMAIN value: {{ .Values.globals.cookieDomain | quote }} + - name: HTTP_MIGRATIONS + value: {{ .Values.globals.httpMigrations | quote }} image: budibase/apps:{{ .Values.globals.appVersion }} imagePullPolicy: Always name: bbapps diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 9ea055c6c0..bb582f69c4 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -99,6 +99,7 @@ globals: accountPortalApiKey: "" cookieDomain: "" platformUrl: "" + httpMigrations: "0" createSecrets: true # creates an internal API key, JWT secrets and redis password for you diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js index a7b38821a7..47854ca9c7 100644 --- a/packages/backend-core/db.js +++ b/packages/backend-core/db.js @@ -1,4 +1,5 @@ module.exports = { ...require("./src/db/utils"), ...require("./src/db/constants"), + ...require("./src/db/views"), } diff --git a/packages/backend-core/src/auth.js b/packages/backend-core/src/auth.js index 41d2bb1cc5..f6d53522d5 100644 --- a/packages/backend-core/src/auth.js +++ b/packages/backend-core/src/auth.js @@ -13,6 +13,7 @@ const { appTenancy, authError, csrf, + internalApi, } = require("./middleware") // Strategies @@ -44,4 +45,5 @@ module.exports = { auditLog, authError, buildCsrfMiddleware: csrf, + internalApi, } diff --git a/packages/backend-core/src/middleware/index.js b/packages/backend-core/src/middleware/index.js index 0d01fb3952..5878479152 100644 --- a/packages/backend-core/src/middleware/index.js +++ b/packages/backend-core/src/middleware/index.js @@ -7,6 +7,7 @@ const authenticated = require("./authenticated") const auditLog = require("./auditLog") const tenancy = require("./tenancy") const appTenancy = require("./appTenancy") +const internalApi = require("./internalApi") const datasourceGoogle = require("./passport/datasource/google") const csrf = require("./csrf") @@ -20,6 +21,7 @@ module.exports = { tenancy, appTenancy, authError, + internalApi, datasource: { google: datasourceGoogle, }, diff --git a/packages/backend-core/src/middleware/internalApi.js b/packages/backend-core/src/middleware/internalApi.js new file mode 100644 index 0000000000..275d559a9e --- /dev/null +++ b/packages/backend-core/src/middleware/internalApi.js @@ -0,0 +1,14 @@ +const env = require("../environment") +const { Headers } = require("../constants") + +/** + * API Key only endpoint. + */ +module.exports = async (ctx, next) => { + const apiKey = ctx.request.headers[Headers.API_KEY] + if (apiKey !== env.INTERNAL_API_KEY) { + ctx.throw(403, "Unauthorized") + } + + return next() +} diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js index 6b8eb3a95c..e2ed75d407 100644 --- a/packages/backend-core/src/migrations/index.js +++ b/packages/backend-core/src/migrations/index.js @@ -1,20 +1,17 @@ +const { DEFAULT_TENANT_ID } = require("../constants") const { DocumentTypes } = require("../db/constants") -const { getGlobalDB, getTenantId } = require("../tenancy") +const { getAllApps } = require("../db/utils") +const environment = require("../environment") +const { + doInTenant, + getTenantIds, + getGlobalDBName, + getTenantId, +} = require("../tenancy") -exports.MIGRATION_DBS = { - GLOBAL_DB: "GLOBAL_DB", -} - -exports.MIGRATIONS = { - USER_EMAIL_VIEW_CASING: "user_email_view_casing", - QUOTAS_1: "quotas_1", -} - -const DB_LOOKUP = { - [exports.MIGRATION_DBS.GLOBAL_DB]: [ - exports.MIGRATIONS.USER_EMAIL_VIEW_CASING, - exports.MIGRATIONS.QUOTAS_1, - ], +exports.MIGRATION_TYPES = { + GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument + APP: "app", // run per app, recorded in each app db, app db is provided as an argument } exports.getMigrationsDoc = async db => { @@ -28,40 +25,90 @@ exports.getMigrationsDoc = async db => { } } -exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => { +const runMigration = async (CouchDB, migration, options = {}) => { const tenantId = getTenantId() - try { - let db - if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) { - db = getGlobalDB() - } else { - throw new Error(`Unrecognised migration db [${migrationDb}]`) - } + const migrationType = migration.type + const migrationName = migration.name - if (!DB_LOOKUP[migrationDb].includes(migrationName)) { - throw new Error( - `Unrecognised migration name [${migrationName}] for db [${migrationDb}]` - ) - } - - const doc = await exports.getMigrationsDoc(db) - // exit if the migration has been performed - if (doc[migrationName]) { - return - } - - console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`) - await migrateFn() - console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`) - - // mark as complete - doc[migrationName] = Date.now() - await db.put(doc) - } catch (err) { - console.error( - `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `, - err + // get the db to store the migration in + let dbNames + if (migrationType === exports.MIGRATION_TYPES.GLOBAL) { + dbNames = [getGlobalDBName()] + } else if (migrationType === exports.MIGRATION_TYPES.APP) { + const apps = await getAllApps(CouchDB, migration.opts) + dbNames = apps.map(app => app.appId) + } else { + throw new Error( + `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]` ) - throw err + } + + // run the migration against each db + for (const dbName of dbNames) { + const db = new CouchDB(dbName) + try { + const doc = await exports.getMigrationsDoc(db) + + // exit if the migration has been performed already + if (doc[migrationName]) { + if ( + options.force && + options.force[migrationType] && + options.force[migrationType].includes(migrationName) + ) { + console.log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` + ) + } else { + // the migration has already been performed + continue + } + } + + console.log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running` + ) + // run the migration with tenant context + await migration.fn(db) + console.log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` + ) + + // mark as complete + doc[migrationName] = Date.now() + await db.put(doc) + } catch (err) { + console.error( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, + err + ) + throw err + } } } + +exports.runMigrations = async (CouchDB, migrations, options = {}) => { + console.log("Running migrations") + let tenantIds + if (environment.MULTI_TENANCY) { + if (!options.tenantIds || !options.tenantIds.length) { + // run for all tenants + tenantIds = await getTenantIds() + } + } else { + // single tenancy + tenantIds = [DEFAULT_TENANT_ID] + } + + // for all tenants + for (const tenantId of tenantIds) { + // for all migrations + for (const migration of migrations) { + // run the migration + await doInTenant(tenantId, () => + runMigration(CouchDB, migration, options) + ) + } + } + console.log("Migrations complete") +} diff --git a/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap b/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap index e9a18eadde..222c3b1228 100644 --- a/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap +++ b/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap @@ -3,7 +3,7 @@ exports[`migrations should match snapshot 1`] = ` Object { "_id": "migrations", - "_rev": "1-af6c272fe081efafecd2ea49a8fcbb40", - "user_email_view_casing": 1487076708000, + "_rev": "1-6277abc4e3db950221768e5a2618a059", + "test": 1487076708000, } `; diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js index 0ed16fc184..12a2e54cb3 100644 --- a/packages/backend-core/src/migrations/tests/index.spec.js +++ b/packages/backend-core/src/migrations/tests/index.spec.js @@ -1,7 +1,7 @@ require("../../tests/utilities/dbConfig") -const { migrateIfRequired, MIGRATION_DBS, MIGRATIONS, getMigrationsDoc } = require("../index") -const database = require("../../db") +const { runMigrations, getMigrationsDoc } = require("../index") +const CouchDB = require("../../db").getCouch() const { StaticDatabases, } = require("../../db/utils") @@ -13,8 +13,14 @@ describe("migrations", () => { const migrationFunction = jest.fn() + const MIGRATIONS = [{ + type: "global", + name: "test", + fn: migrationFunction + }] + beforeEach(() => { - db = database.getDB(StaticDatabases.GLOBAL.name) + db = new CouchDB(StaticDatabases.GLOBAL.name) }) afterEach(async () => { @@ -22,39 +28,29 @@ describe("migrations", () => { await db.destroy() }) - const validMigration = () => { - return migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction) + const migrate = () => { + return runMigrations(CouchDB, MIGRATIONS) } it("should run a new migration", async () => { - await validMigration() + await migrate() expect(migrationFunction).toHaveBeenCalled() + const doc = await getMigrationsDoc(db) + expect(doc.test).toBeDefined() }) it("should match snapshot", async () => { - await validMigration() + await migrate() const doc = await getMigrationsDoc(db) expect(doc).toMatchSnapshot() }) it("should skip a previously run migration", async () => { - await validMigration() - await validMigration() + await migrate() + const previousMigrationTime = await getMigrationsDoc(db).test + await migrate() + const currentMigrationTime = await getMigrationsDoc(db).test expect(migrationFunction).toHaveBeenCalledTimes(1) + expect(currentMigrationTime).toBe(previousMigrationTime) }) - - it("should reject an unknown migration name", async () => { - expect(async () => { - await migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, "bogus_name", migrationFunction) - }).rejects.toThrow() - expect(migrationFunction).not.toHaveBeenCalled() - }) - - it("should reject an unknown database name", async () => { - expect(async () => { - await migrateIfRequired("bogus_db", MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction) - }).rejects.toThrow() - expect(migrationFunction).not.toHaveBeenCalled() - }) - }) \ No newline at end of file diff --git a/packages/backend-core/src/tenancy/tenancy.js b/packages/backend-core/src/tenancy/tenancy.js index 2cd05ea925..de597eac01 100644 --- a/packages/backend-core/src/tenancy/tenancy.js +++ b/packages/backend-core/src/tenancy/tenancy.js @@ -148,3 +148,15 @@ exports.isUserInAppTenant = (appId, user = null) => { const tenantId = exports.getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID return tenantId === userTenantId } + +exports.getTenantIds = async () => { + const db = getDB(PLATFORM_INFO_DB) + let tenants + try { + tenants = await db.get(TENANT_DOC) + } catch (err) { + // if theres an error the doc doesn't exist, no tenants exist + return [] + } + return (tenants && tenants.tenantIds) || [] +} diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js index 8c00f2a8b8..6c71c51b9d 100644 --- a/packages/backend-core/src/utils.js +++ b/packages/backend-core/src/utils.js @@ -20,9 +20,6 @@ const { hash } = require("./hashing") const userCache = require("./cache/user") const env = require("./environment") const { getUserSessions, invalidateSessions } = require("./security/sessions") -const { migrateIfRequired } = require("./migrations") -const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS -const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS const APP_PREFIX = DocumentTypes.APP + SEPARATOR @@ -144,11 +141,6 @@ exports.getGlobalUserByEmail = async email => { } const db = getGlobalDB() - await migrateIfRequired(GLOBAL_DB, USER_EMAIL_VIEW_CASING, async () => { - // re-create the view with latest changes - await createUserEmailView(db) - }) - try { let users = ( await db.query(`database/${ViewNames.USER_BY_EMAIL}`, { diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index db8f261aef..eb1f7bc5e6 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -75,7 +75,7 @@ function getUserRoleId(ctx) { : ctx.user.role._id } -async function getAppUrl(ctx) { +exports.getAppUrl = ctx => { // construct the url let url if (ctx.request.body.url) { @@ -219,7 +219,7 @@ exports.create = async ctx => { const apps = await getAllApps(CouchDB, { dev: true }) const name = ctx.request.body.name checkAppName(ctx, apps, name) - const url = await getAppUrl(ctx) + const url = exports.getAppUrl(ctx) checkAppUrl(ctx, apps, url) const { useTemplate, templateKey, templateString } = ctx.request.body @@ -286,7 +286,7 @@ exports.update = async ctx => { if (name) { checkAppName(ctx, apps, name, ctx.params.appId) } - const url = await getAppUrl(ctx) + const url = await exports.getAppUrl(ctx) if (url) { checkAppUrl(ctx, apps, url, ctx.params.appId) ctx.request.body.url = url diff --git a/packages/server/src/api/controllers/migrations.js b/packages/server/src/api/controllers/migrations.js new file mode 100644 index 0000000000..6a890349c3 --- /dev/null +++ b/packages/server/src/api/controllers/migrations.js @@ -0,0 +1,13 @@ +const { migrate, MIGRATIONS } = require("../../migrations") + +exports.migrate = async ctx => { + const options = ctx.request.body + // don't await as can take a while, just return + migrate(options) + ctx.status = 200 +} + +exports.fetchDefinitions = async ctx => { + ctx.body = MIGRATIONS + ctx.status = 200 +} diff --git a/packages/server/src/api/routes/index.js b/packages/server/src/api/routes/index.js index 44c5fbb7b0..8ded7104b0 100644 --- a/packages/server/src/api/routes/index.js +++ b/packages/server/src/api/routes/index.js @@ -24,6 +24,7 @@ const backupRoutes = require("./backup") const metadataRoutes = require("./metadata") const devRoutes = require("./dev") const cloudRoutes = require("./cloud") +const migrationRoutes = require("./migrations") exports.mainRoutes = [ authRoutes, @@ -53,6 +54,7 @@ exports.mainRoutes = [ // this could be breaking as koa may recognise other routes as this tableRoutes, rowRoutes, + migrationRoutes, ] exports.staticRoutes = staticRoutes diff --git a/packages/server/src/api/routes/migrations.js b/packages/server/src/api/routes/migrations.js new file mode 100644 index 0000000000..01e573edb3 --- /dev/null +++ b/packages/server/src/api/routes/migrations.js @@ -0,0 +1,14 @@ +const Router = require("@koa/router") +const migrationsController = require("../controllers/migrations") +const router = Router() +const { internalApi } = require("@budibase/backend-core/auth") + +router + .post("/api/migrations/run", internalApi, migrationsController.migrate) + .get( + "/api/migrations/definitions", + internalApi, + migrationsController.fetchDefinitions + ) + +module.exports = router diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 060169a777..0c0ef68ad9 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -1,7 +1,7 @@ // need to load environment first import { ExtendableContext } from "koa" -const env = require("./environment") +import * as env from "./environment" const CouchDB = require("./db") require("@budibase/backend-core").init(CouchDB) const Koa = require("koa") @@ -16,6 +16,7 @@ const Sentry = require("@sentry/node") const fileSystem = require("./utilities/fileSystem") const bullboard = require("./automations/bullboard") const redis = require("./utilities/redis") +import * as migrations from "./migrations" const app = new Koa() @@ -84,13 +85,25 @@ module.exports = server.listen(env.PORT || 0, async () => { await automations.init() }) -process.on("uncaughtException", err => { - console.error(err) +const shutdown = () => { server.close() server.destroy() +} + +process.on("uncaughtException", err => { + console.error(err) + shutdown() }) process.on("SIGTERM", () => { - server.close() - server.destroy() + shutdown() }) + +// run migrations on startup if not done via http +// not recommended in a clustered environment +if (!env.HTTP_MIGRATIONS) { + migrations.migrate().catch(err => { + console.error("Error performing migrations. Exiting.\n", err) + shutdown() + }) +} diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 614f41a29f..99343937d9 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -43,6 +43,7 @@ module.exports = { REDIS_PASSWORD: process.env.REDIS_PASSWORD, INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, MULTI_TENANCY: process.env.MULTI_TENANCY, + HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS, // environment NODE_ENV: process.env.NODE_ENV, JEST_WORKER_ID: process.env.JEST_WORKER_ID, diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 4bafa75132..2cd0836113 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -5,7 +5,6 @@ const { isExternalTable, isRowId: isExternalRowId, } = require("../integrations/utils") -const migration = require("../migrations/usageQuotas") // currently only counting new writes and deletes const METHOD_MAP = { @@ -74,7 +73,6 @@ module.exports = async (ctx, next) => { usage = files.map(file => file.size).reduce((total, size) => total + size) } try { - await migration.run() await performRequest(ctx, next, property, usage) } catch (err) { ctx.throw(400, err) diff --git a/packages/server/src/migrations/functions/appUrls.ts b/packages/server/src/migrations/functions/appUrls.ts new file mode 100644 index 0000000000..8852c27822 --- /dev/null +++ b/packages/server/src/migrations/functions/appUrls.ts @@ -0,0 +1,25 @@ +const { DocumentTypes } = require("@budibase/backend-core/db") +import { getAppUrl } from "../../api/controllers/application" + +/** + * Date: + * January 2022 + * + * Description: + * Add the url to the app metadata if it doesn't exist + */ +export const run = async (appDb: any) => { + const metadata = await appDb.get(DocumentTypes.APP_METADATA) + if (!metadata.url) { + const context = { + request: { + body: { + name: metadata.name, + }, + }, + } + metadata.url = getAppUrl(context) + console.log(`Adding url to app: ${metadata.url}`) + } + await appDb.put(metadata) +} diff --git a/packages/server/src/migrations/functions/quotas1.ts b/packages/server/src/migrations/functions/quotas1.ts new file mode 100644 index 0000000000..500aa68f51 --- /dev/null +++ b/packages/server/src/migrations/functions/quotas1.ts @@ -0,0 +1,20 @@ +import { runQuotaMigration } from "./usageQuotas" +import * as syncApps from "./usageQuotas/syncApps" +import * as syncRows from "./usageQuotas/syncRows" + +/** + * Date: + * January 2022 + * + * Description: + * Synchronise the app and row quotas to the state of the db after it was + * discovered that the quota resets were still in place and the row quotas + * weren't being decremented correctly. + */ + +export const run = async () => { + await runQuotaMigration(async () => { + await syncApps.run() + await syncRows.run() + }) +} diff --git a/packages/server/src/migrations/functions/tests/appUrls.spec.js b/packages/server/src/migrations/functions/tests/appUrls.spec.js new file mode 100644 index 0000000000..d3f080dfd4 --- /dev/null +++ b/packages/server/src/migrations/functions/tests/appUrls.spec.js @@ -0,0 +1,29 @@ +const { DocumentTypes } = require("@budibase/backend-core/db") +const env = require("../../../environment") +const TestConfig = require("../../../tests/utilities/TestConfiguration") + +const migration = require("../appUrls") + +describe("run", () => { + let config = new TestConfig(false) + const CouchDB = config.getCouch() + + beforeEach(async () => { + await config.init() + }) + + afterAll(config.end) + + it("runs successfully", async () => { + const app = await config.createApp("testApp") + const appDb = new CouchDB(app.appId) + let metadata = await appDb.get(DocumentTypes.APP_METADATA) + delete metadata.url + await appDb.put(metadata) + + await migration.run(appDb) + + metadata = await appDb.get(DocumentTypes.APP_METADATA) + expect(metadata.url).toEqual("/testapp") + }) +}) diff --git a/packages/server/src/migrations/tests/usageQuotas/index.spec.js b/packages/server/src/migrations/functions/tests/quotas1.spec.js similarity index 63% rename from packages/server/src/migrations/tests/usageQuotas/index.spec.js rename to packages/server/src/migrations/functions/tests/quotas1.spec.js index 0c5b982909..df8703e9a0 100644 --- a/packages/server/src/migrations/tests/usageQuotas/index.spec.js +++ b/packages/server/src/migrations/functions/tests/quotas1.spec.js @@ -4,10 +4,10 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration") const syncApps = jest.fn() const syncRows = jest.fn() -jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) -jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) +jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) ) +jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) ) -const migrations = require("../../usageQuotas") +const migration = require("../quotas1") describe("run", () => { let config = new TestConfig(false) @@ -19,8 +19,8 @@ describe("run", () => { afterAll(config.end) - it("runs the required migrations", async () => { - await migrations.run() + it("runs ", async () => { + await migration.run() expect(syncApps).toHaveBeenCalledTimes(1) expect(syncRows).toHaveBeenCalledTimes(1) }) diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js new file mode 100644 index 0000000000..c0d7823cbf --- /dev/null +++ b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js @@ -0,0 +1,25 @@ +const TestConfig = require("../../../tests/utilities/TestConfiguration") +const { getGlobalDB } = require("@budibase/backend-core/tenancy") + +// mock email view creation +const coreDb = require("@budibase/backend-core/db") +const createUserEmailView = jest.fn() +coreDb.createUserEmailView = createUserEmailView + +const migration = require("../userEmailViewCasing") + +describe("run", () => { + let config = new TestConfig(false) + const globalDb = getGlobalDB() + + beforeEach(async () => { + await config.init() + }) + + afterAll(config.end) + + it("runs successfully", async () => { + await migration.run(globalDb) + expect(createUserEmailView).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/server/src/migrations/functions/usageQuotas/index.ts b/packages/server/src/migrations/functions/usageQuotas/index.ts new file mode 100644 index 0000000000..16c4bf1d89 --- /dev/null +++ b/packages/server/src/migrations/functions/usageQuotas/index.ts @@ -0,0 +1,8 @@ +const { useQuotas } = require("../../../utilities/usageQuota") + +export const runQuotaMigration = async (migration: Function) => { + if (!useQuotas()) { + return + } + await migration() +} diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts similarity index 80% rename from packages/server/src/migrations/usageQuotas/syncApps.js rename to packages/server/src/migrations/functions/usageQuotas/syncApps.ts index ee106129e6..0fba4f0f7f 100644 --- a/packages/server/src/migrations/usageQuotas/syncApps.js +++ b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts @@ -1,9 +1,9 @@ const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") const { getAllApps } = require("@budibase/backend-core/db") -const CouchDB = require("../../db") -const { getUsageQuotaDoc } = require("../../utilities/usageQuota") +import CouchDB from "../../../db" +import { getUsageQuotaDoc } from "../../../utilities/usageQuota" -exports.run = async () => { +export const run = async () => { const db = getGlobalDB() // get app count const devApps = await getAllApps(CouchDB, { dev: true }) diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts similarity index 67% rename from packages/server/src/migrations/usageQuotas/syncRows.js rename to packages/server/src/migrations/functions/usageQuotas/syncRows.ts index 7990f405de..58767d0c0a 100644 --- a/packages/server/src/migrations/usageQuotas/syncRows.js +++ b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts @@ -1,14 +1,14 @@ const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") const { getAllApps } = require("@budibase/backend-core/db") -const CouchDB = require("../../db") -const { getUsageQuotaDoc } = require("../../utilities/usageQuota") -const { getUniqueRows } = require("../../utilities/usageQuota/rows") +import CouchDB from "../../../db" +import { getUsageQuotaDoc } from "../../../utilities/usageQuota" +import { getUniqueRows } from "../../../utilities/usageQuota/rows" -exports.run = async () => { +export const run = async () => { const db = getGlobalDB() // get all rows in all apps const allApps = await getAllApps(CouchDB, { all: true }) - const appIds = allApps ? allApps.map(app => app.appId) : [] + const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : [] const rows = await getUniqueRows(appIds) const rowCount = rows ? rows.length : 0 diff --git a/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js similarity index 81% rename from packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js rename to packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js index 160319a31b..7c74cbfe9a 100644 --- a/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js +++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js @@ -1,8 +1,8 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const TestConfig = require("../../../tests/utilities/TestConfiguration") -const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") -const syncApps = require("../../usageQuotas/syncApps") -const env = require("../../../environment") +const TestConfig = require("../../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota") +const syncApps = require("../syncApps") +const env = require("../../../../environment") describe("syncApps", () => { let config = new TestConfig(false) diff --git a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js similarity index 83% rename from packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js rename to packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js index 2e2ea7b291..034d0eb067 100644 --- a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js +++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js @@ -1,8 +1,8 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const TestConfig = require("../../../tests/utilities/TestConfiguration") -const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") -const syncRows = require("../../usageQuotas/syncRows") -const env = require("../../../environment") +const TestConfig = require("../../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota") +const syncRows = require("../syncRows") +const env = require("../../../../environment") describe("syncRows", () => { let config = new TestConfig(false) diff --git a/packages/server/src/migrations/functions/userEmailViewCasing.ts b/packages/server/src/migrations/functions/userEmailViewCasing.ts new file mode 100644 index 0000000000..16f55655ab --- /dev/null +++ b/packages/server/src/migrations/functions/userEmailViewCasing.ts @@ -0,0 +1,13 @@ +const { createUserEmailView } = require("@budibase/backend-core/db") + +/** + * Date: + * October 2021 + * + * Description: + * Recreate the user email view to include latest changes i.e. lower casing the email address + */ + +export const run = async (db: any) => { + await createUserEmailView(db) +} diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts new file mode 100644 index 0000000000..966041e0c9 --- /dev/null +++ b/packages/server/src/migrations/index.ts @@ -0,0 +1,56 @@ +import CouchDB from "../db" +const { + MIGRATION_TYPES, + runMigrations, +} = require("@budibase/backend-core/migrations") + +// migration functions +import * as userEmailViewCasing from "./functions/userEmailViewCasing" +import * as quota1 from "./functions/quotas1" +import * as appUrls from "./functions/appUrls" + +export interface Migration { + type: string + name: string + opts?: object + fn: Function +} + +/** + * e.g. + * { + * tenantIds: ['bb'], + * force: { + * global: ['quota_1'] + * } + * } + */ +export interface MigrationOptions { + tenantIds?: string[] + forced?: { + [type: string]: string[] + } +} + +export const MIGRATIONS: Migration[] = [ + { + type: MIGRATION_TYPES.GLOBAL, + name: "user_email_view_casing", + fn: userEmailViewCasing.run, + }, + { + type: MIGRATION_TYPES.GLOBAL, + name: "quotas_1", + fn: quota1.run, + }, + { + type: MIGRATION_TYPES.APP, + name: "app_urls", + opts: { all: true }, + fn: appUrls.run, + }, +] + +export const migrate = async (options?: MigrationOptions) => { + await runMigrations(CouchDB, MIGRATIONS, options) +} diff --git a/packages/server/src/migrations/usageQuotas/index.js b/packages/server/src/migrations/usageQuotas/index.js deleted file mode 100644 index 39744093c2..0000000000 --- a/packages/server/src/migrations/usageQuotas/index.js +++ /dev/null @@ -1,24 +0,0 @@ -const { - MIGRATIONS, - MIGRATION_DBS, - migrateIfRequired, -} = require("@budibase/backend-core/migrations") -const { useQuotas } = require("../../utilities/usageQuota") -const syncApps = require("./syncApps") -const syncRows = require("./syncRows") - -exports.run = async () => { - if (!useQuotas()) { - return - } - - // Jan 2022 - await migrateIfRequired( - MIGRATION_DBS.GLOBAL_DB, - MIGRATIONS.QUOTAS_1, - async () => { - await syncApps.run() - await syncRows.run() - } - ) -} diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index 6fd13fc229..68aa68dc66 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -50,6 +50,10 @@ class TestConfiguration { return this.appId } + getCouch() { + return CouchDB + } + async _req(config, params, controlFunc) { const request = {} // fake cookies, we don't need them