diff --git a/hosting/proxy/10-listen-on-ipv6-by-default.sh b/hosting/proxy/10-listen-on-ipv6-by-default.sh new file mode 100644 index 0000000000..e2e89388a9 --- /dev/null +++ b/hosting/proxy/10-listen-on-ipv6-by-default.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# vim:sw=4:ts=4:et + +set -e + +ME=$(basename $0) +NGINX_CONF_FILE="/etc/nginx/nginx.conf" +DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf" + +# check if we have ipv6 available +if [ ! -f "/proc/net/if_inet6" ]; then + # ipv6 not available so delete lines from nginx conf + if [ -f "$NGINX_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $NGINX_CONF_FILE + fi + if [ -f "$DEFAULT_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE + fi + echo "$ME: info: ipv6 not available so delete lines from nginx conf" +else + echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf" +fi + +exit 0 diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index 298762aaf1..5fd0dc7d11 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -5,7 +5,7 @@ FROM nginx:latest # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template - +COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh # Error handling COPY error.html /usr/share/nginx/html/error.html diff --git a/lerna.json b/lerna.json index f5ecd09c34..c17905e010 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.0.29", + "version": "2.0.30-alpha.1", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 05606920a3..e89ff61325 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.0.29", + "@budibase/types": "2.0.30-alpha.1", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", @@ -62,6 +62,7 @@ ] }, "devDependencies": { + "@types/chance": "1.1.3", "@types/jest": "27.5.1", "@types/koa": "2.0.52", "@types/lodash": "4.14.180", @@ -72,6 +73,7 @@ "@types/semver": "7.3.7", "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", + "chance": "1.1.3", "ioredis-mock": "5.8.0", "jest": "27.5.1", "koa": "2.7.0", diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 83b23b479d..42cad17620 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -37,6 +37,7 @@ const core = { db, ...dbConstants, redis, + locks: redis.redlock, objectStore, utils, users, diff --git a/packages/backend-core/src/pkg/redis.ts b/packages/backend-core/src/pkg/redis.ts index 65ab186d9a..297c2b54f4 100644 --- a/packages/backend-core/src/pkg/redis.ts +++ b/packages/backend-core/src/pkg/redis.ts @@ -3,9 +3,11 @@ import Client from "../redis" import utils from "../redis/utils" import clients from "../redis/init" +import * as redlock from "../redis/redlock" export = { Client, utils, clients, + redlock, } diff --git a/packages/backend-core/src/redis/init.js b/packages/backend-core/src/redis/init.js index 8e5d10f838..3150ef2c1c 100644 --- a/packages/backend-core/src/redis/init.js +++ b/packages/backend-core/src/redis/init.js @@ -1,27 +1,23 @@ const Client = require("./index") const utils = require("./utils") -const { getRedlock } = require("./redlock") -let userClient, sessionClient, appClient, cacheClient, writethroughClient -let migrationsRedlock - -// turn retry off so that only one instance can ever hold the lock -const migrationsRedlockConfig = { retryCount: 0 } +let userClient, + sessionClient, + appClient, + cacheClient, + writethroughClient, + lockClient async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() + lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client( utils.Databases.WRITE_THROUGH, utils.SelectableDatabases.WRITE_THROUGH ).init() - // pass the underlying ioredis client to redlock - migrationsRedlock = getRedlock( - cacheClient.getClient(), - migrationsRedlockConfig - ) } process.on("exit", async () => { @@ -30,6 +26,7 @@ process.on("exit", async () => { if (appClient) await appClient.finish() if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() + if (lockClient) await lockClient.finish() }) module.exports = { @@ -63,10 +60,10 @@ module.exports = { } return writethroughClient }, - getMigrationsRedlock: async () => { - if (!migrationsRedlock) { + getLockClient: async () => { + if (!lockClient) { await init() } - return migrationsRedlock + return lockClient }, } diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlock.ts index beef375b55..abb13b2534 100644 --- a/packages/backend-core/src/redis/redlock.ts +++ b/packages/backend-core/src/redis/redlock.ts @@ -1,14 +1,37 @@ -import Redlock from "redlock" +import Redlock, { Options } from "redlock" +import { getLockClient } from "./init" +import { LockOptions, LockType } from "@budibase/types" +import * as tenancy from "../tenancy" -export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { - return new Redlock([redisClient], { +let noRetryRedlock: Redlock | undefined + +const getClient = async (type: LockType): Promise => { + switch (type) { + case LockType.TRY_ONCE: { + if (!noRetryRedlock) { + noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE) + } + return noRetryRedlock + } + default: { + throw new Error(`Could not get redlock client: ${type}`) + } + } +} + +export const OPTIONS = { + TRY_ONCE: { + // immediately throws an error if the lock is already held + retryCount: 0, + }, + DEFAULT: { // the expected clock drift; for more details // see http://redis.io/topics/distlock driftFactor: 0.01, // multiplied by lock ttl to determine drift time // the max number of times Redlock will attempt // to lock a resource before erroring - retryCount: opts.retryCount, + retryCount: 10, // the time in ms between attempts retryDelay: 200, // time in ms @@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { // the max time in ms randomly added to retries // to improve performance under high contention // see https://www.awsarchitectureblog.com/2015/03/backoff.html - retryJitter: 200, // time in ms - }) + retryJitter: 100, // time in ms + }, +} + +export const newRedlock = async (opts: Options = {}) => { + let options = { ...OPTIONS.DEFAULT, ...opts } + const redisWrapper = await getLockClient() + const client = redisWrapper.getClient() + return new Redlock([client], options) +} + +export const doWithLock = async (opts: LockOptions, task: any) => { + const redlock = await getClient(opts.type) + let lock + try { + // aquire lock + let name: string = `${tenancy.getTenantId()}_${opts.name}` + if (opts.nameSuffix) { + name = name + `_${opts.nameSuffix}` + } + lock = await redlock.lock(name, opts.ttl) + // perform locked task + return task() + } catch (e: any) { + // lock limit exceeded + if (e.name === "LockError") { + if (opts.type === LockType.TRY_ONCE) { + // don't throw for try-once locks, they will always error + // due to retry count (0) exceeded + return + } else { + throw e + } + } else { + throw e + } + } finally { + if (lock) { + await lock.unlock() + } + } } diff --git a/packages/backend-core/src/redis/utils.js b/packages/backend-core/src/redis/utils.js index 90b3561f31..af719197b5 100644 --- a/packages/backend-core/src/redis/utils.js +++ b/packages/backend-core/src/redis/utils.js @@ -28,6 +28,7 @@ exports.Databases = { LICENSES: "license", GENERIC_CACHE: "data_cache", WRITE_THROUGH: "writeThrough", + LOCKS: "locks", } /** diff --git a/packages/backend-core/tests/utilities/structures/accounts.ts b/packages/backend-core/tests/utilities/structures/accounts.ts new file mode 100644 index 0000000000..5d23962575 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/accounts.ts @@ -0,0 +1,23 @@ +import { generator, uuid } from "." +import { AuthType, CloudAccount, Hosting } from "@budibase/types" +import * as db from "../../../src/db/utils" + +export const cloudAccount = (): CloudAccount => { + return { + accountId: uuid(), + createdAt: Date.now(), + verified: true, + verificationSent: true, + tier: "", + email: generator.email(), + tenantId: generator.word(), + hosting: Hosting.CLOUD, + authType: AuthType.PASSWORD, + password: generator.word(), + tenantName: generator.word(), + name: generator.name(), + size: "10+", + profession: "Software Engineer", + budibaseUserId: db.generateGlobalUserID(), + } +} diff --git a/packages/backend-core/tests/utilities/structures/common.ts b/packages/backend-core/tests/utilities/structures/common.ts new file mode 100644 index 0000000000..51ae220254 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/common.ts @@ -0,0 +1 @@ +export { v4 as uuid } from "uuid" diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index 12b6ab7ad6..68064b9715 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -1 +1,8 @@ +export * from "./common" + +import Chance from "chance" +export const generator = new Chance() + export * as koa from "./koa" +export * as accounts from "./accounts" +export * as licenses from "./licenses" diff --git a/packages/backend-core/tests/utilities/structures/licenses.ts b/packages/backend-core/tests/utilities/structures/licenses.ts new file mode 100644 index 0000000000..a541e91860 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/licenses.ts @@ -0,0 +1,18 @@ +import { AccountPlan, License, PlanType, Quotas } from "@budibase/types" + +const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => { + return { + type, + } +} + +export const newLicense = (opts: { + quotas: Quotas + planType?: PlanType +}): License => { + return { + features: [], + quotas: opts.quotas, + plan: newPlan(opts.planType), + } +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 2e62aea734..6bc9b63728 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -663,6 +663,11 @@ "@types/connect" "*" "@types/node" "*" +"@types/chance@1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea" + integrity sha512-X6c6ghhe4/sQh4XzcZWSFaTAUOda38GQHmq9BUanYkOE/EO7ZrkazwKmtsj3xzTjkLWmwULE++23g3d3CCWaWw== + "@types/connect@*": version "3.4.35" resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1" @@ -1555,6 +1560,11 @@ chalk@^4.0.0, chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chance@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3" + integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg== + char-regex@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index d453fcf3b1..2b1b50485e 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.0.29", + "@budibase/string-templates": "2.0.30-alpha.1", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 7fcfa6221f..318309653f 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.0.29", - "@budibase/client": "^2.0.29", - "@budibase/frontend-core": "^2.0.29", - "@budibase/string-templates": "^2.0.29", + "@budibase/bbui": "2.0.30-alpha.1", + "@budibase/client": "2.0.30-alpha.1", + "@budibase/frontend-core": "2.0.30-alpha.1", + "@budibase/string-templates": "2.0.30-alpha.1", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 0cf8656079..5c17ec8abf 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.0.29", - "@budibase/string-templates": "^2.0.29", - "@budibase/types": "^2.0.29", + "@budibase/backend-core": "2.0.30-alpha.1", + "@budibase/string-templates": "2.0.30-alpha.1", + "@budibase/types": "2.0.30-alpha.1", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index 367c8b2a87..d8c611b2bb 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.0.29", - "@budibase/frontend-core": "^2.0.29", - "@budibase/string-templates": "^2.0.29", + "@budibase/bbui": "2.0.30-alpha.1", + "@budibase/frontend-core": "2.0.30-alpha.1", + "@budibase/string-templates": "2.0.30-alpha.1", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index daafebee22..2f7fdd41e8 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.0.29", + "@budibase/bbui": "2.0.30-alpha.1", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index fbe5b7b637..9153383675 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index 76eef3680f..1534bd13ef 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.0.29", - "@budibase/client": "^2.0.29", - "@budibase/pro": "2.0.29", - "@budibase/string-templates": "^2.0.29", - "@budibase/types": "^2.0.29", + "@budibase/backend-core": "2.0.30-alpha.1", + "@budibase/client": "2.0.30-alpha.1", + "@budibase/pro": "2.0.30-alpha.1", + "@budibase/string-templates": "2.0.30-alpha.1", + "@budibase/types": "2.0.30-alpha.1", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 5edf862706..a51e7ad6ec 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -17,7 +17,6 @@ import { getProdAppDB, getDevAppDB, } from "@budibase/backend-core/context" -import { quotas } from "@budibase/pro" import { events } from "@budibase/backend-core" // the max time we can wait for an invalidation to complete before considering it failed diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index cb1e6d1c82..275a954a78 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { locks, migrations } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -86,33 +92,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 12fd228281..0fe33c8f68 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.29.tgz#d5856d49d8cc64790961631dfe0fface7f7be4e4" - integrity sha512-05mnl6YcucWrO1X6bVBYG6r7Yig/fIHbokLRfEvFFrZNe/EcRB3iLeOG1+2190dv5TbO/jhabS3kcrbDs54HHw== +"@budibase/backend-core@2.0.30-alpha.1": + version "2.0.30-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.1.tgz#6e5cbc448cd819f67c93451d35c8793748380649" + integrity sha512-AuXk5IZBQ9zFTjTL2FsqqYdqqVjrSv19ozXSsb1w2z8Pyymz57WgzUiZ6uzHbHzUXNVD4Nei7cgfgSyCJ27Phg== dependencies: - "@budibase/types" "^2.0.29" + "@budibase/types" "2.0.30-alpha.1" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1180,13 +1180,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.29.tgz#169055bc39894f90341226fbff4a1601418d0b42" - integrity sha512-ELBoQ7/MXlgatCJNvTNXgF7DK02pfYx5Yy1s/2BJr4iGe26+5Q65ztiC7Jp+d/owese+f5kqKJRNuU1KINUfjQ== +"@budibase/pro@2.0.30-alpha.1": + version "2.0.30-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.1.tgz#12e4c2115d3e647d365b46e6b5df59746148d4ea" + integrity sha512-K/iy3Z3f9AVodxtuAk8yemSQ2R08tjoot4wogDCwsTMRlXxq/WChCcm9prkb2DzwOzQQSsJ3bLgL1p/9sGzagg== dependencies: - "@budibase/backend-core" "2.0.29" - "@budibase/types" "2.0.29" + "@budibase/backend-core" "2.0.30-alpha.1" + "@budibase/types" "2.0.30-alpha.1" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" @@ -1209,10 +1209,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.0.29", "@budibase/types@^2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.29.tgz#8b27f695aded7ad7523c4943deb556eadfb66c3c" - integrity sha512-wwpHgDwKff2UhNmKAdrzIxmDQ/crY77AZdFyWNpPvrHYIetyh2Kp5ikEKyZlYHTEpS2IPDE8EKn4coDeu+mGlQ== +"@budibase/types@2.0.30-alpha.1": + version "2.0.30-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.1.tgz#1320eb9c2996e371b47b2a845ce1b81e1b21b67c" + integrity sha512-tw49Jwo7Dnhv9IuZq68DVvHsKcaVpggZ/1BYkQsdzUw+0yoIe1k8t+4XPbHqBP328mpp8oPx4rUQF3yFpJv41g== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 28da60a3e3..a453583844 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index d6f467ef88..b7c7a3f540 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/types/src/api/account/index.ts b/packages/types/src/api/account/index.ts index 50c6bf22c6..0cbc487bcc 100644 --- a/packages/types/src/api/account/index.ts +++ b/packages/types/src/api/account/index.ts @@ -1 +1,2 @@ export * from "./user" +export * from "./license" diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts new file mode 100644 index 0000000000..40ee79c3e3 --- /dev/null +++ b/packages/types/src/api/account/license.ts @@ -0,0 +1,11 @@ +import { QuotaUsage } from "../../documents" + +export interface GetLicenseRequest { + quotaUsage: QuotaUsage +} + +export interface QuotaTriggeredRequest { + percentage: number + name: string + resetDate?: string +} diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index e7dcf2d89f..70c3061c3f 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -1,4 +1,12 @@ -import { Feature, Hosting, PlanType, Quotas } from "../../sdk" +import { + Feature, + Hosting, + MonthlyQuotaName, + PlanType, + Quotas, + StaticQuotaName, +} from "../../sdk" +import { MonthlyUsage, QuotaUsage, StaticUsage } from "../global" export interface CreateAccount { email: string @@ -42,6 +50,7 @@ export interface Account extends CreateAccount { licenseKey?: string licenseKeyActivatedAt?: number licenseOverrides?: LicenseOverrides + quotaUsage?: QuotaUsage } export interface PasswordAccount extends Account { diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts index eb1d77c228..84e5af3996 100644 --- a/packages/types/src/documents/global/quotas.ts +++ b/packages/types/src/documents/global/quotas.ts @@ -24,19 +24,34 @@ export interface UsageBreakdown { } } -export type MonthlyUsage = { +export type QuotaTriggers = { + [key: string]: string | undefined +} + +export interface StaticUsage { + [StaticQuotaName.APPS]: number + [StaticQuotaName.PLUGINS]: number + [StaticQuotaName.USER_GROUPS]: number + [StaticQuotaName.ROWS]: number + triggers: { + [key in StaticQuotaName]?: QuotaTriggers + } +} + +export interface MonthlyUsage { [MonthlyQuotaName.QUERIES]: number [MonthlyQuotaName.AUTOMATIONS]: number [MonthlyQuotaName.DAY_PASSES]: number + triggers: { + [key in MonthlyQuotaName]?: QuotaTriggers + } breakdown?: { [key in BreakdownQuotaName]?: UsageBreakdown } } export interface BaseQuotaUsage { - usageQuota: { - [key in StaticQuotaName]: number - } + usageQuota: StaticUsage monthly: { [key: string]: MonthlyUsage } @@ -51,6 +66,13 @@ export interface QuotaUsage extends BaseQuotaUsage { } } +export type SetUsageValues = { + total: number + app?: number + breakdown?: number + triggers?: QuotaTriggers +} + export type UsageValues = { total: number app?: number diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index bae566b42e..0c374dd105 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -7,3 +7,4 @@ export * from "./datasources" export * from "./search" export * from "./koa" export * from "./auth" +export * from "./locks" diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 49dd561db0..74777d4590 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -61,26 +61,40 @@ export type PlanQuotas = { [PlanType.ENTERPRISE]: Quotas } +export type MonthlyQuotas = { + [MonthlyQuotaName.QUERIES]: Quota + [MonthlyQuotaName.AUTOMATIONS]: Quota + [MonthlyQuotaName.DAY_PASSES]: Quota +} + +export type StaticQuotas = { + [StaticQuotaName.ROWS]: Quota + [StaticQuotaName.APPS]: Quota + [StaticQuotaName.USER_GROUPS]: Quota + [StaticQuotaName.PLUGINS]: Quota +} + +export type ConstantQuotas = { + [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota +} + export type Quotas = { [QuotaType.USAGE]: { - [QuotaUsageType.MONTHLY]: { - [MonthlyQuotaName.QUERIES]: Quota - [MonthlyQuotaName.AUTOMATIONS]: Quota - [MonthlyQuotaName.DAY_PASSES]: Quota - } - [QuotaUsageType.STATIC]: { - [StaticQuotaName.ROWS]: Quota - [StaticQuotaName.APPS]: Quota - [StaticQuotaName.USER_GROUPS]: Quota - [StaticQuotaName.PLUGINS]: Quota - } - } - [QuotaType.CONSTANT]: { - [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota + [QuotaUsageType.MONTHLY]: MonthlyQuotas + [QuotaUsageType.STATIC]: StaticQuotas } + [QuotaType.CONSTANT]: ConstantQuotas } export interface Quota { name: string value: number + /** + * Array of whole numbers (1-100) that dictate the percentage that this quota should trigger + * at in relation to the corresponding usage inside budibase. + * + * Triggering results in a budibase installation sending a request to account-portal, + * which can have subsequent effects such as sending emails to users. + */ + triggers: number[] } diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts new file mode 100644 index 0000000000..3aa067bea1 --- /dev/null +++ b/packages/types/src/sdk/locks.ts @@ -0,0 +1,31 @@ +export enum LockType { + /** + * If this lock is already held the attempted operation will not be performed. + * No retries will take place and no error will be thrown. + */ + TRY_ONCE = "try_once", +} + +export enum LockName { + MIGRATIONS = "migrations", + TRIGGER_QUOTA = "trigger_quota", +} + +export interface LockOptions { + /** + * The lock type determines which client to use + */ + type: LockType + /** + * The name for the lock + */ + name: LockName + /** + * The ttl to auto-expire the lock if not unlocked manually + */ + ttl: number + /** + * The suffix to add to the lock name for additional uniqueness + */ + nameSuffix?: string +} diff --git a/packages/worker/package.json b/packages/worker/package.json index 64910f3708..e87be9afb2 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.0.29", + "version": "2.0.30-alpha.1", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.0.29", - "@budibase/pro": "2.0.29", - "@budibase/string-templates": "^2.0.29", - "@budibase/types": "^2.0.29", + "@budibase/backend-core": "2.0.30-alpha.1", + "@budibase/pro": "2.0.30-alpha.1", + "@budibase/string-templates": "2.0.30-alpha.1", + "@budibase/types": "2.0.30-alpha.1", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", diff --git a/packages/worker/src/api/controllers/system/tenants.js b/packages/worker/src/api/controllers/system/tenants.js deleted file mode 100644 index c54a3d9834..0000000000 --- a/packages/worker/src/api/controllers/system/tenants.js +++ /dev/null @@ -1,58 +0,0 @@ -const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") -const { getTenantId } = require("@budibase/backend-core/tenancy") -const { deleteTenant } = require("@budibase/backend-core/deprovision") -const { quotas } = require("@budibase/pro") - -exports.exists = async ctx => { - const tenantId = ctx.request.params - ctx.body = { - exists: await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let exists = false - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 - } - } catch (err) { - // if error it doesn't exist - } - return exists - }), - } -} - -exports.fetch = async ctx => { - ctx.body = await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let tenants = [] - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - tenants = tenantsDoc.tenantIds - } - } catch (err) { - // if error it doesn't exist - } - return tenants - }) -} - -exports.delete = async ctx => { - const tenantId = getTenantId() - - if (ctx.params.tenantId !== tenantId) { - ctx.throw(403, "Unauthorized") - } - - try { - await deleteTenant(tenantId) - await quotas.bustCache() - ctx.status = 204 - } catch (err) { - ctx.log.error(err) - throw err - } -} diff --git a/packages/worker/src/api/controllers/system/tenants.ts b/packages/worker/src/api/controllers/system/tenants.ts new file mode 100644 index 0000000000..d6e6261c22 --- /dev/null +++ b/packages/worker/src/api/controllers/system/tenants.ts @@ -0,0 +1,66 @@ +const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") +const { getTenantId } = require("@budibase/backend-core/tenancy") +const { deleteTenant } = require("@budibase/backend-core/deprovision") +import { quotas } from "@budibase/pro" + +export const exists = async (ctx: any) => { + const tenantId = ctx.request.params + ctx.body = { + exists: await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let exists = false + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 + } + } catch (err) { + // if error it doesn't exist + } + return exists + } + ), + } +} + +export const fetch = async (ctx: any) => { + ctx.body = await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let tenants = [] + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + tenants = tenantsDoc.tenantIds + } + } catch (err) { + // if error it doesn't exist + } + return tenants + } + ) +} + +const _delete = async (ctx: any) => { + const tenantId = getTenantId() + + if (ctx.params.tenantId !== tenantId) { + ctx.throw(403, "Unauthorized") + } + + try { + await deleteTenant(tenantId) + await quotas.bustCache() + ctx.status = 204 + } catch (err) { + ctx.log.error(err) + throw err + } +} + +export { _delete as delete } diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts index 6900596216..19ef076a52 100644 --- a/packages/worker/src/migrations/index.ts +++ b/packages/worker/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { migrations, locks } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -42,33 +48,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } diff --git a/packages/worker/src/sdk/users/events.ts b/packages/worker/src/sdk/users/events.ts index 0094c6fd84..3046442393 100644 --- a/packages/worker/src/sdk/users/events.ts +++ b/packages/worker/src/sdk/users/events.ts @@ -1,7 +1,6 @@ import env from "../../environment" import { events, accounts, tenancy } from "@budibase/backend-core" import { User, UserRoles, CloudAccount } from "@budibase/types" -import { users as pro } from "@budibase/pro" export const handleDeleteEvents = async (user: any) => { await events.user.deleted(user) diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 3ecc2a4f02..68188cf827 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.29.tgz#d5856d49d8cc64790961631dfe0fface7f7be4e4" - integrity sha512-05mnl6YcucWrO1X6bVBYG6r7Yig/fIHbokLRfEvFFrZNe/EcRB3iLeOG1+2190dv5TbO/jhabS3kcrbDs54HHw== +"@budibase/backend-core@2.0.30-alpha.1": + version "2.0.30-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.1.tgz#6e5cbc448cd819f67c93451d35c8793748380649" + integrity sha512-AuXk5IZBQ9zFTjTL2FsqqYdqqVjrSv19ozXSsb1w2z8Pyymz57WgzUiZ6uzHbHzUXNVD4Nei7cgfgSyCJ27Phg== dependencies: - "@budibase/types" "^2.0.29" + "@budibase/types" "2.0.30-alpha.1" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -327,21 +327,21 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.29.tgz#169055bc39894f90341226fbff4a1601418d0b42" - integrity sha512-ELBoQ7/MXlgatCJNvTNXgF7DK02pfYx5Yy1s/2BJr4iGe26+5Q65ztiC7Jp+d/owese+f5kqKJRNuU1KINUfjQ== +"@budibase/pro@2.0.30-alpha.1": + version "2.0.30-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.1.tgz#12e4c2115d3e647d365b46e6b5df59746148d4ea" + integrity sha512-K/iy3Z3f9AVodxtuAk8yemSQ2R08tjoot4wogDCwsTMRlXxq/WChCcm9prkb2DzwOzQQSsJ3bLgL1p/9sGzagg== dependencies: - "@budibase/backend-core" "2.0.29" - "@budibase/types" "2.0.29" + "@budibase/backend-core" "2.0.30-alpha.1" + "@budibase/types" "2.0.30-alpha.1" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.0.29", "@budibase/types@^2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.29.tgz#8b27f695aded7ad7523c4943deb556eadfb66c3c" - integrity sha512-wwpHgDwKff2UhNmKAdrzIxmDQ/crY77AZdFyWNpPvrHYIetyh2Kp5ikEKyZlYHTEpS2IPDE8EKn4coDeu+mGlQ== +"@budibase/types@2.0.30-alpha.1": + version "2.0.30-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.1.tgz#1320eb9c2996e371b47b2a845ce1b81e1b21b67c" + integrity sha512-tw49Jwo7Dnhv9IuZq68DVvHsKcaVpggZ/1BYkQsdzUw+0yoIe1k8t+4XPbHqBP328mpp8oPx4rUQF3yFpJv41g== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0"