From 86d094dda4e2dd999d988d058bfba969b9e387c9 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 1 Jun 2022 17:52:41 +0100 Subject: [PATCH] Migration locks and add optional preventRetry option --- packages/backend-core/package.json | 2 + packages/backend-core/redis.js | 1 + packages/backend-core/src/migrations/index.js | 9 ++++ packages/backend-core/src/redis/authRedis.js | 16 +++++++ packages/backend-core/src/redis/index.js | 4 ++ packages/backend-core/src/redis/redlock.ts | 21 +++++++++ packages/backend-core/yarn.lock | 39 ++++++++++++--- packages/server/src/migrations/index.ts | 47 ++++++++++++++++++- 8 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 packages/backend-core/src/redis/redlock.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 6c5063eb01..2cd53537cd 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -19,6 +19,7 @@ "dotenv": "^16.0.1", "emitter-listener": "^1.1.2", "ioredis": "^4.27.1", + "redlock": "^4.0.0", "jsonwebtoken": "^8.5.1", "koa-passport": "^4.1.4", "lodash": "^4.17.21", @@ -55,6 +56,7 @@ "@types/tar-fs": "^2.0.1", "@types/uuid": "^8.3.4", "@types/semver": "^7.0.0", + "@types/redlock": "^4.0.0", "ioredis-mock": "^5.5.5", "jest": "^27.0.3", "koa": "2.7.0", diff --git a/packages/backend-core/redis.js b/packages/backend-core/redis.js index 0a9dc91881..badce3be20 100644 --- a/packages/backend-core/redis.js +++ b/packages/backend-core/redis.js @@ -1,4 +1,5 @@ module.exports = { Client: require("./src/redis"), utils: require("./src/redis/utils"), + clients: require("./src/redis/authRedis"), } diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js index 11067d4cb0..5c3ab4f4a5 100644 --- a/packages/backend-core/src/migrations/index.js +++ b/packages/backend-core/src/migrations/index.js @@ -90,6 +90,15 @@ exports.runMigration = async (migration, options = {}) => { log( `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` ) + + if (migration.preventRetry) { + // eagerly set the completion date + // so that we never run this migration twice even upon failure + doc[migrationName] = Date.now() + const response = await db.put(doc) + doc._rev = response.rev + } + // run the migration with tenant context if (migrationType === exports.MIGRATION_TYPES.APP) { await context.doInAppContext(db.name, async () => { diff --git a/packages/backend-core/src/redis/authRedis.js b/packages/backend-core/src/redis/authRedis.js index b9f6d8d0b0..b2f686b801 100644 --- a/packages/backend-core/src/redis/authRedis.js +++ b/packages/backend-core/src/redis/authRedis.js @@ -1,13 +1,23 @@ const Client = require("./index") const utils = require("./utils") +const { getRedlock } = require("./redlock") let userClient, sessionClient, appClient, cacheClient +let migrationsRedlock + +// turn retry off so that only one instance can ever hold the lock +const migrationsRedlockConfig = { retryCount: 0 } async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() + // pass the underlying ioredis client to redlock + migrationsRedlock = getRedlock( + cacheClient.getClient(), + migrationsRedlockConfig + ) } process.on("exit", async () => { @@ -42,4 +52,10 @@ module.exports = { } return cacheClient }, + getMigrationsRedlock: async () => { + if (!migrationsRedlock) { + await init() + } + return migrationsRedlock + }, } diff --git a/packages/backend-core/src/redis/index.js b/packages/backend-core/src/redis/index.js index 38f7130a4c..32fce94ba5 100644 --- a/packages/backend-core/src/redis/index.js +++ b/packages/backend-core/src/redis/index.js @@ -139,6 +139,10 @@ class RedisWrapper { this._db = db } + getClient() { + return CLIENT + } + async init() { CLOSED = false init() diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlock.ts new file mode 100644 index 0000000000..beef375b55 --- /dev/null +++ b/packages/backend-core/src/redis/redlock.ts @@ -0,0 +1,21 @@ +import Redlock from "redlock" + +export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { + return new Redlock([redisClient], { + // the expected clock drift; for more details + // see http://redis.io/topics/distlock + driftFactor: 0.01, // multiplied by lock ttl to determine drift time + + // the max number of times Redlock will attempt + // to lock a resource before erroring + retryCount: opts.retryCount, + + // the time in ms between attempts + retryDelay: 200, // time in ms + + // the max time in ms randomly added to retries + // to improve performance under high contention + // see https://www.awsarchitectureblog.com/2015/03/backoff.html + retryJitter: 200, // time in ms + }) +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 98c82da865..013bd6062f 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -829,6 +829,11 @@ dependencies: "@babel/types" "^7.3.0" +"@types/bluebird@*": + version "3.5.36" + resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.36.tgz#00d9301d4dc35c2f6465a8aec634bb533674c652" + integrity sha512-HBNx4lhkxN7bx6P0++W8E289foSu8kO8GCk2unhuVggO+cE7rh9DhZUyPhUxNRG9m+5B5BTKxZQ5ZP92x/mx9Q== + "@types/body-parser@*": version "1.19.2" resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0" @@ -895,13 +900,6 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1" integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w== -"@types/ioredis@^4.27.1": - version "4.28.10" - resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff" - integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ== - dependencies: - "@types/node" "*" - "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.3" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762" @@ -993,6 +991,21 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== +"@types/redis@^2.8.0": + version "2.8.32" + resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11" + integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w== + dependencies: + "@types/node" "*" + +"@types/redlock@^4.0.0": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@types/redlock/-/redlock-4.0.3.tgz#aeab5fe5f0d433a125f6dcf9a884372ac0cddd4b" + integrity sha512-mcvvrquwREbAqyZALNBIlf49AL9Aa324BG+J/Dv4TAP8g+nxQMBI4/APNqqS99QEY7VTNT9XvsaczCVGK8uNnQ== + dependencies: + "@types/bluebird" "*" + "@types/redis" "^2.8.0" + "@types/semver@^7.0.0": version "7.3.9" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc" @@ -1397,6 +1410,11 @@ bl@^4.0.3: inherits "^2.0.4" readable-stream "^3.4.0" +bluebird@^3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" + integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== + boom@2.x.x: version "2.10.1" resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f" @@ -4835,6 +4853,13 @@ redis-parser@^3.0.0: dependencies: redis-errors "^1.0.0" +redlock@^4.0.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/redlock/-/redlock-4.2.0.tgz#c26590768559afd5fff76aa1133c94b411ff4f5f" + integrity sha512-j+oQlG+dOwcetUt2WJWttu4CZVeRzUrcVcISFmEmfyuwCVSJ93rDT7YSgg7H7rnxwoRyk/jU46kycVka5tW7jA== + dependencies: + bluebird "^3.7.2" + regenerator-runtime@^0.13.4: version "0.13.9" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52" diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index 1418a05f18..c93f628c71 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -1,4 +1,4 @@ -import { migrations } from "@budibase/backend-core" +import { migrations, redis } from "@budibase/backend-core" // migration functions import * as userEmailViewCasing from "./functions/userEmailViewCasing" @@ -15,6 +15,7 @@ export interface Migration { opts?: object fn: Function silent?: boolean + preventRetry?: boolean } /** @@ -66,21 +67,63 @@ export const MIGRATIONS: Migration[] = [ opts: { all: true }, fn: backfill.app.run, silent: !!env.SELF_HOSTED, // reduce noisy logging + preventRetry: !!env.SELF_HOSTED, // only ever run once }, { type: migrations.MIGRATION_TYPES.GLOBAL, name: "event_global_backfill", fn: backfill.global.run, silent: !!env.SELF_HOSTED, // reduce noisy logging + preventRetry: !!env.SELF_HOSTED, // only ever run once }, { type: migrations.MIGRATION_TYPES.INSTALLATION, name: "event_installation_backfill", fn: backfill.installation.run, silent: !!env.SELF_HOSTED, // reduce noisy logging + preventRetry: !!env.SELF_HOSTED, // only ever run once }, ] export const migrate = async (options?: MigrationOptions) => { - await migrations.runMigrations(MIGRATIONS, options) + if (env.SELF_HOSTED) { + // self host runs migrations on startup + // make sure only a single instance runs them + await migrateWithLock(options) + } else { + await migrations.runMigrations(MIGRATIONS, options) + } +} + +const migrateWithLock = async (options?: MigrationOptions) => { + // get a new lock client + + const redlock = await redis.clients.getMigrationsRedlock() + // lock for 15 minutes + const ttl = 1000 * 60 * 15 + + let migrationLock + + // acquire lock + try { + migrationLock = await redlock.lock("migrations", ttl) + } catch (e: any) { + if (e.name === "LockError") { + return + } else { + throw e + } + } + + // run migrations + try { + await migrations.runMigrations(MIGRATIONS, options) + } finally { + // release lock + try { + await migrationLock.unlock() + } catch (e) { + console.error("unable to release migration lock") + } + } }