diff --git a/hosting/proxy/10-listen-on-ipv6-by-default.sh b/hosting/proxy/10-listen-on-ipv6-by-default.sh new file mode 100644 index 0000000000..e2e89388a9 --- /dev/null +++ b/hosting/proxy/10-listen-on-ipv6-by-default.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# vim:sw=4:ts=4:et + +set -e + +ME=$(basename $0) +NGINX_CONF_FILE="/etc/nginx/nginx.conf" +DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf" + +# check if we have ipv6 available +if [ ! -f "/proc/net/if_inet6" ]; then + # ipv6 not available so delete lines from nginx conf + if [ -f "$NGINX_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $NGINX_CONF_FILE + fi + if [ -f "$DEFAULT_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE + fi + echo "$ME: info: ipv6 not available so delete lines from nginx conf" +else + echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf" +fi + +exit 0 diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index 298762aaf1..5fd0dc7d11 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -5,7 +5,7 @@ FROM nginx:latest # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template - +COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh # Error handling COPY error.html /usr/share/nginx/html/error.html diff --git a/lerna.json b/lerna.json index df9302103d..1d092b043c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.0.30-alpha.0", + "version": "2.0.30-alpha.3", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 1f8da2b010..abcf72491a 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.0.30-alpha.0", + "version": "2.0.30-alpha.3", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "2.0.30-alpha.0", + "@budibase/types": "2.0.30-alpha.3", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", @@ -62,6 +62,7 @@ ] }, "devDependencies": { + "@types/chance": "1.1.3", "@types/jest": "27.5.1", "@types/koa": "2.0.52", "@types/lodash": "4.14.180", @@ -72,6 +73,7 @@ "@types/semver": "7.3.7", "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", + "chance": "1.1.3", "ioredis-mock": "5.8.0", "jest": "27.5.1", "koa": "2.7.0", diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 83b23b479d..42cad17620 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -37,6 +37,7 @@ const core = { db, ...dbConstants, redis, + locks: redis.redlock, objectStore, utils, users, diff --git a/packages/backend-core/src/pkg/redis.ts b/packages/backend-core/src/pkg/redis.ts index 65ab186d9a..297c2b54f4 100644 --- a/packages/backend-core/src/pkg/redis.ts +++ b/packages/backend-core/src/pkg/redis.ts @@ -3,9 +3,11 @@ import Client from "../redis" import utils from "../redis/utils" import clients from "../redis/init" +import * as redlock from "../redis/redlock" export = { Client, utils, clients, + redlock, } diff --git a/packages/backend-core/src/redis/init.js b/packages/backend-core/src/redis/init.js index 8e5d10f838..3150ef2c1c 100644 --- a/packages/backend-core/src/redis/init.js +++ b/packages/backend-core/src/redis/init.js @@ -1,27 +1,23 @@ const Client = require("./index") const utils = require("./utils") -const { getRedlock } = require("./redlock") -let userClient, sessionClient, appClient, cacheClient, writethroughClient -let migrationsRedlock - -// turn retry off so that only one instance can ever hold the lock -const migrationsRedlockConfig = { retryCount: 0 } +let userClient, + sessionClient, + appClient, + cacheClient, + writethroughClient, + lockClient async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() + lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client( utils.Databases.WRITE_THROUGH, utils.SelectableDatabases.WRITE_THROUGH ).init() - // pass the underlying ioredis client to redlock - migrationsRedlock = getRedlock( - cacheClient.getClient(), - migrationsRedlockConfig - ) } process.on("exit", async () => { @@ -30,6 +26,7 @@ process.on("exit", async () => { if (appClient) await appClient.finish() if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() + if (lockClient) await lockClient.finish() }) module.exports = { @@ -63,10 +60,10 @@ module.exports = { } return writethroughClient }, - getMigrationsRedlock: async () => { - if (!migrationsRedlock) { + getLockClient: async () => { + if (!lockClient) { await init() } - return migrationsRedlock + return lockClient }, } diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlock.ts index beef375b55..abb13b2534 100644 --- a/packages/backend-core/src/redis/redlock.ts +++ b/packages/backend-core/src/redis/redlock.ts @@ -1,14 +1,37 @@ -import Redlock from "redlock" +import Redlock, { Options } from "redlock" +import { getLockClient } from "./init" +import { LockOptions, LockType } from "@budibase/types" +import * as tenancy from "../tenancy" -export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { - return new Redlock([redisClient], { +let noRetryRedlock: Redlock | undefined + +const getClient = async (type: LockType): Promise => { + switch (type) { + case LockType.TRY_ONCE: { + if (!noRetryRedlock) { + noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE) + } + return noRetryRedlock + } + default: { + throw new Error(`Could not get redlock client: ${type}`) + } + } +} + +export const OPTIONS = { + TRY_ONCE: { + // immediately throws an error if the lock is already held + retryCount: 0, + }, + DEFAULT: { // the expected clock drift; for more details // see http://redis.io/topics/distlock driftFactor: 0.01, // multiplied by lock ttl to determine drift time // the max number of times Redlock will attempt // to lock a resource before erroring - retryCount: opts.retryCount, + retryCount: 10, // the time in ms between attempts retryDelay: 200, // time in ms @@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { // the max time in ms randomly added to retries // to improve performance under high contention // see https://www.awsarchitectureblog.com/2015/03/backoff.html - retryJitter: 200, // time in ms - }) + retryJitter: 100, // time in ms + }, +} + +export const newRedlock = async (opts: Options = {}) => { + let options = { ...OPTIONS.DEFAULT, ...opts } + const redisWrapper = await getLockClient() + const client = redisWrapper.getClient() + return new Redlock([client], options) +} + +export const doWithLock = async (opts: LockOptions, task: any) => { + const redlock = await getClient(opts.type) + let lock + try { + // aquire lock + let name: string = `${tenancy.getTenantId()}_${opts.name}` + if (opts.nameSuffix) { + name = name + `_${opts.nameSuffix}` + } + lock = await redlock.lock(name, opts.ttl) + // perform locked task + return task() + } catch (e: any) { + // lock limit exceeded + if (e.name === "LockError") { + if (opts.type === LockType.TRY_ONCE) { + // don't throw for try-once locks, they will always error + // due to retry count (0) exceeded + return + } else { + throw e + } + } else { + throw e + } + } finally { + if (lock) { + await lock.unlock() + } + } } diff --git a/packages/backend-core/src/redis/utils.js b/packages/backend-core/src/redis/utils.js index 90b3561f31..af719197b5 100644 --- a/packages/backend-core/src/redis/utils.js +++ b/packages/backend-core/src/redis/utils.js @@ -28,6 +28,7 @@ exports.Databases = { LICENSES: "license", GENERIC_CACHE: "data_cache", WRITE_THROUGH: "writeThrough", + LOCKS: "locks", } /** diff --git a/packages/backend-core/tests/utilities/structures/accounts.ts b/packages/backend-core/tests/utilities/structures/accounts.ts new file mode 100644 index 0000000000..5d23962575 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/accounts.ts @@ -0,0 +1,23 @@ +import { generator, uuid } from "." +import { AuthType, CloudAccount, Hosting } from "@budibase/types" +import * as db from "../../../src/db/utils" + +export const cloudAccount = (): CloudAccount => { + return { + accountId: uuid(), + createdAt: Date.now(), + verified: true, + verificationSent: true, + tier: "", + email: generator.email(), + tenantId: generator.word(), + hosting: Hosting.CLOUD, + authType: AuthType.PASSWORD, + password: generator.word(), + tenantName: generator.word(), + name: generator.name(), + size: "10+", + profession: "Software Engineer", + budibaseUserId: db.generateGlobalUserID(), + } +} diff --git a/packages/backend-core/tests/utilities/structures/common.ts b/packages/backend-core/tests/utilities/structures/common.ts new file mode 100644 index 0000000000..51ae220254 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/common.ts @@ -0,0 +1 @@ +export { v4 as uuid } from "uuid" diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index 12b6ab7ad6..68064b9715 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -1 +1,8 @@ +export * from "./common" + +import Chance from "chance" +export const generator = new Chance() + export * as koa from "./koa" +export * as accounts from "./accounts" +export * as licenses from "./licenses" diff --git a/packages/backend-core/tests/utilities/structures/licenses.ts b/packages/backend-core/tests/utilities/structures/licenses.ts new file mode 100644 index 0000000000..a541e91860 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/licenses.ts @@ -0,0 +1,18 @@ +import { AccountPlan, License, PlanType, Quotas } from "@budibase/types" + +const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => { + return { + type, + } +} + +export const newLicense = (opts: { + quotas: Quotas + planType?: PlanType +}): License => { + return { + features: [], + quotas: opts.quotas, + plan: newPlan(opts.planType), + } +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 2e62aea734..6bc9b63728 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -663,6 +663,11 @@ "@types/connect" "*" "@types/node" "*" +"@types/chance@1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea" + integrity sha512-X6c6ghhe4/sQh4XzcZWSFaTAUOda38GQHmq9BUanYkOE/EO7ZrkazwKmtsj3xzTjkLWmwULE++23g3d3CCWaWw== + "@types/connect@*": version "3.4.35" resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1" @@ -1555,6 +1560,11 @@ chalk@^4.0.0, chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chance@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3" + integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg== + char-regex@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 5bf0e38e3c..147a1b3785 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.0.30-alpha.0", + "version": "2.0.30-alpha.3", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "2.0.30-alpha.0", + "@budibase/string-templates": "2.0.30-alpha.3", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index e118b66a5d..8ced899d20 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.0.30-alpha.0", + "version": "2.0.30-alpha.3", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.0.30-alpha.0", - "@budibase/client": "2.0.30-alpha.0", - "@budibase/frontend-core": "2.0.30-alpha.0", - "@budibase/string-templates": "2.0.30-alpha.0", + "@budibase/bbui": "2.0.30-alpha.3", + "@budibase/client": "2.0.30-alpha.3", + "@budibase/frontend-core": "2.0.30-alpha.3", + "@budibase/string-templates": "2.0.30-alpha.3", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/pages/builder/portal/settings/upgrade.svelte b/packages/builder/src/pages/builder/portal/settings/upgrade.svelte index 16bbc0b03d..5d10f22396 100644 --- a/packages/builder/src/pages/builder/portal/settings/upgrade.svelte +++ b/packages/builder/src/pages/builder/portal/settings/upgrade.svelte @@ -91,7 +91,6 @@ }) -{"is adming" + $auth.isAdmin} {#if $auth.isAdmin} { config: { url: info.url, defaultHeaders: [], + rejectUnauthorized: true, }, name: info.name, } diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 71294b9cb0..08d2337593 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -78,16 +78,11 @@ const SCHEMA: Integration = { default: {}, }, rejectUnauthorized: { + display: "Reject Unauthorized", type: DatasourceFieldType.BOOLEAN, default: true, required: false, }, - legacyHttpParser: { - display: "Legacy HTTP Support", - type: DatasourceFieldType.BOOLEAN, - required: false, - default: false, - }, }, query: { create: { @@ -397,6 +392,7 @@ class RestIntegration implements IntegrationBase { }) } + // Deprecated by rejectUnauthorized if (this.config.legacyHttpParser) { // https://github.com/nodejs/node/issues/43798 input.extraHttpOptions = { insecureHTTPParser: true } diff --git a/packages/server/src/integrations/tests/rest.spec.ts b/packages/server/src/integrations/tests/rest.spec.ts index d451282a86..d53f9c7ce3 100644 --- a/packages/server/src/integrations/tests/rest.spec.ts +++ b/packages/server/src/integrations/tests/rest.spec.ts @@ -1,17 +1,16 @@ -jest.mock("node-fetch", () => - jest.fn(() => ({ - headers: { - raw: () => { - return { "content-type": ["application/json"] } - }, - get: () => ["application/json"], +const mockFetch = jest.fn(() => ({ + headers: { + raw: () => { + return { "content-type": ["application/json"] } }, - json: jest.fn(() => ({ - my_next_cursor: 123, - })), - text: jest.fn(), - })) -) + get: () => ["application/json"], + }, + json: jest.fn(() => ({ + my_next_cursor: 123, + })), + text: jest.fn(), +})) +jest.mock("node-fetch", () => mockFetch) import fetch from "node-fetch" import { default as RestIntegration } from "../rest" const FormData = require("form-data") @@ -572,4 +571,21 @@ describe("REST Integration", () => { }) }) }) + + it("Attaches custom agent when Reject Unauthorized option is false", async () => { + config = new TestConfiguration({ + url: BASE_URL, + rejectUnauthorized: false, + }) + await config.integration.read({}) + + const calls: any = mockFetch.mock.calls[0] + const url = calls[0] + expect(url).toBe(`${BASE_URL}/`) + + const calledConfig = calls[1] + expect(calledConfig.method).toBe("GET") + expect(calledConfig.headers).toEqual({}) + expect(calledConfig.agent.options.rejectUnauthorized).toBe(false) + }) }) diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index cb1e6d1c82..275a954a78 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { locks, migrations } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -86,33 +92,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 1e3a9c6738..1a8f5a6cd5 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.30-alpha.0": - version "2.0.30-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.0.tgz#aa9277ec26da51f8396fe047ebab39a0dff69c93" - integrity sha512-6+KHAceEZv1SC0qSA4GTNQBAlF1wnr9jfl85D5Tq1O2caD4J7drkIz8D6e0/DVv8zccHT+aUA7hy7YhKxbHAUA== +"@budibase/backend-core@2.0.30-alpha.3": + version "2.0.30-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.3.tgz#42235b33676c3357d8cf8e181e619c4c92065fd0" + integrity sha512-ixE28ddY8FqvCcaCkqsxUI5UjkFKxFu23MgNARJ8VSpj5NLq6d8qAR+xE0E2pJ58hm5r1KLn2yAZiNXZzu0dzg== dependencies: - "@budibase/types" "2.0.30-alpha.0" + "@budibase/types" "2.0.30-alpha.3" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1180,13 +1180,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.0.30-alpha.0": - version "2.0.30-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.0.tgz#6f0d9e45bc458e0e423280355c1a339f3143da47" - integrity sha512-8PFdiIHT7wUusa1Gi+KvcpLaJdBAtnAK6bfYGayasnrS9OIasfIBAMgzXRONMAGMvY7PZz84psZVimXSkW7EEA== +"@budibase/pro@2.0.30-alpha.3": + version "2.0.30-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.3.tgz#ad8f8aa5ced2b5200e05b98e392267b9f84c8177" + integrity sha512-Cw7XUGVVbHCxKo3tJFvG5EKUvO0zdOoU/baFArjMiNHAivlZvFVWCsxptIua+qtFkaUpjQneTHihFiQnY9g83A== dependencies: - "@budibase/backend-core" "2.0.30-alpha.0" - "@budibase/types" "2.0.30-alpha.0" + "@budibase/backend-core" "2.0.30-alpha.3" + "@budibase/types" "2.0.30-alpha.3" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" @@ -1209,10 +1209,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.0.30-alpha.0": - version "2.0.30-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.0.tgz#fb5e80870b00850601785f1102f64bd22339f4e9" - integrity sha512-i9c02crPzZPRzn+RXwn2f4H4WCLWhC7HlsiXH2w4Ngk7PJvtcR9Jn7mKBKVcXM/9PAK6d0vWIQidA8XuU0CpZA== +"@budibase/types@2.0.30-alpha.3": + version "2.0.30-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.3.tgz#cb55bcced75b711cc8a675284fbacaa8ebf1c0f2" + integrity sha512-rHeFVuNbSSE4fMnX6uyrM2r47m+neqFXlVNOkhHU9i7KoIcIZbEYInU8CjUFR2da3ruST9ajXjJ5UenX2+MnTg== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index e8df4e2bba..d8117e19f4 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.0.30-alpha.0", + "version": "2.0.30-alpha.3", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 0787093e46..c29f53d997 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.0.30-alpha.0", + "version": "2.0.30-alpha.3", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/types/src/api/account/index.ts b/packages/types/src/api/account/index.ts index 50c6bf22c6..0cbc487bcc 100644 --- a/packages/types/src/api/account/index.ts +++ b/packages/types/src/api/account/index.ts @@ -1 +1,2 @@ export * from "./user" +export * from "./license" diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts new file mode 100644 index 0000000000..40ee79c3e3 --- /dev/null +++ b/packages/types/src/api/account/license.ts @@ -0,0 +1,11 @@ +import { QuotaUsage } from "../../documents" + +export interface GetLicenseRequest { + quotaUsage: QuotaUsage +} + +export interface QuotaTriggeredRequest { + percentage: number + name: string + resetDate?: string +} diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index e7dcf2d89f..70c3061c3f 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -1,4 +1,12 @@ -import { Feature, Hosting, PlanType, Quotas } from "../../sdk" +import { + Feature, + Hosting, + MonthlyQuotaName, + PlanType, + Quotas, + StaticQuotaName, +} from "../../sdk" +import { MonthlyUsage, QuotaUsage, StaticUsage } from "../global" export interface CreateAccount { email: string @@ -42,6 +50,7 @@ export interface Account extends CreateAccount { licenseKey?: string licenseKeyActivatedAt?: number licenseOverrides?: LicenseOverrides + quotaUsage?: QuotaUsage } export interface PasswordAccount extends Account { diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts index eb1d77c228..84e5af3996 100644 --- a/packages/types/src/documents/global/quotas.ts +++ b/packages/types/src/documents/global/quotas.ts @@ -24,19 +24,34 @@ export interface UsageBreakdown { } } -export type MonthlyUsage = { +export type QuotaTriggers = { + [key: string]: string | undefined +} + +export interface StaticUsage { + [StaticQuotaName.APPS]: number + [StaticQuotaName.PLUGINS]: number + [StaticQuotaName.USER_GROUPS]: number + [StaticQuotaName.ROWS]: number + triggers: { + [key in StaticQuotaName]?: QuotaTriggers + } +} + +export interface MonthlyUsage { [MonthlyQuotaName.QUERIES]: number [MonthlyQuotaName.AUTOMATIONS]: number [MonthlyQuotaName.DAY_PASSES]: number + triggers: { + [key in MonthlyQuotaName]?: QuotaTriggers + } breakdown?: { [key in BreakdownQuotaName]?: UsageBreakdown } } export interface BaseQuotaUsage { - usageQuota: { - [key in StaticQuotaName]: number - } + usageQuota: StaticUsage monthly: { [key: string]: MonthlyUsage } @@ -51,6 +66,13 @@ export interface QuotaUsage extends BaseQuotaUsage { } } +export type SetUsageValues = { + total: number + app?: number + breakdown?: number + triggers?: QuotaTriggers +} + export type UsageValues = { total: number app?: number diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index bae566b42e..0c374dd105 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -7,3 +7,4 @@ export * from "./datasources" export * from "./search" export * from "./koa" export * from "./auth" +export * from "./locks" diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 49dd561db0..74777d4590 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -61,26 +61,40 @@ export type PlanQuotas = { [PlanType.ENTERPRISE]: Quotas } +export type MonthlyQuotas = { + [MonthlyQuotaName.QUERIES]: Quota + [MonthlyQuotaName.AUTOMATIONS]: Quota + [MonthlyQuotaName.DAY_PASSES]: Quota +} + +export type StaticQuotas = { + [StaticQuotaName.ROWS]: Quota + [StaticQuotaName.APPS]: Quota + [StaticQuotaName.USER_GROUPS]: Quota + [StaticQuotaName.PLUGINS]: Quota +} + +export type ConstantQuotas = { + [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota +} + export type Quotas = { [QuotaType.USAGE]: { - [QuotaUsageType.MONTHLY]: { - [MonthlyQuotaName.QUERIES]: Quota - [MonthlyQuotaName.AUTOMATIONS]: Quota - [MonthlyQuotaName.DAY_PASSES]: Quota - } - [QuotaUsageType.STATIC]: { - [StaticQuotaName.ROWS]: Quota - [StaticQuotaName.APPS]: Quota - [StaticQuotaName.USER_GROUPS]: Quota - [StaticQuotaName.PLUGINS]: Quota - } - } - [QuotaType.CONSTANT]: { - [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota + [QuotaUsageType.MONTHLY]: MonthlyQuotas + [QuotaUsageType.STATIC]: StaticQuotas } + [QuotaType.CONSTANT]: ConstantQuotas } export interface Quota { name: string value: number + /** + * Array of whole numbers (1-100) that dictate the percentage that this quota should trigger + * at in relation to the corresponding usage inside budibase. + * + * Triggering results in a budibase installation sending a request to account-portal, + * which can have subsequent effects such as sending emails to users. + */ + triggers: number[] } diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts new file mode 100644 index 0000000000..3aa067bea1 --- /dev/null +++ b/packages/types/src/sdk/locks.ts @@ -0,0 +1,31 @@ +export enum LockType { + /** + * If this lock is already held the attempted operation will not be performed. + * No retries will take place and no error will be thrown. + */ + TRY_ONCE = "try_once", +} + +export enum LockName { + MIGRATIONS = "migrations", + TRIGGER_QUOTA = "trigger_quota", +} + +export interface LockOptions { + /** + * The lock type determines which client to use + */ + type: LockType + /** + * The name for the lock + */ + name: LockName + /** + * The ttl to auto-expire the lock if not unlocked manually + */ + ttl: number + /** + * The suffix to add to the lock name for additional uniqueness + */ + nameSuffix?: string +} diff --git a/packages/worker/package.json b/packages/worker/package.json index 838077e657..34de6d0924 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.0.30-alpha.0", + "version": "2.0.30-alpha.3", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "2.0.30-alpha.0", - "@budibase/pro": "2.0.30-alpha.0", - "@budibase/string-templates": "2.0.30-alpha.0", - "@budibase/types": "2.0.30-alpha.0", + "@budibase/backend-core": "2.0.30-alpha.3", + "@budibase/pro": "2.0.30-alpha.3", + "@budibase/string-templates": "2.0.30-alpha.3", + "@budibase/types": "2.0.30-alpha.3", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", diff --git a/packages/worker/src/api/controllers/system/tenants.js b/packages/worker/src/api/controllers/system/tenants.js deleted file mode 100644 index c54a3d9834..0000000000 --- a/packages/worker/src/api/controllers/system/tenants.js +++ /dev/null @@ -1,58 +0,0 @@ -const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") -const { getTenantId } = require("@budibase/backend-core/tenancy") -const { deleteTenant } = require("@budibase/backend-core/deprovision") -const { quotas } = require("@budibase/pro") - -exports.exists = async ctx => { - const tenantId = ctx.request.params - ctx.body = { - exists: await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let exists = false - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 - } - } catch (err) { - // if error it doesn't exist - } - return exists - }), - } -} - -exports.fetch = async ctx => { - ctx.body = await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let tenants = [] - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - tenants = tenantsDoc.tenantIds - } - } catch (err) { - // if error it doesn't exist - } - return tenants - }) -} - -exports.delete = async ctx => { - const tenantId = getTenantId() - - if (ctx.params.tenantId !== tenantId) { - ctx.throw(403, "Unauthorized") - } - - try { - await deleteTenant(tenantId) - await quotas.bustCache() - ctx.status = 204 - } catch (err) { - ctx.log.error(err) - throw err - } -} diff --git a/packages/worker/src/api/controllers/system/tenants.ts b/packages/worker/src/api/controllers/system/tenants.ts new file mode 100644 index 0000000000..d6e6261c22 --- /dev/null +++ b/packages/worker/src/api/controllers/system/tenants.ts @@ -0,0 +1,66 @@ +const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") +const { getTenantId } = require("@budibase/backend-core/tenancy") +const { deleteTenant } = require("@budibase/backend-core/deprovision") +import { quotas } from "@budibase/pro" + +export const exists = async (ctx: any) => { + const tenantId = ctx.request.params + ctx.body = { + exists: await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let exists = false + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 + } + } catch (err) { + // if error it doesn't exist + } + return exists + } + ), + } +} + +export const fetch = async (ctx: any) => { + ctx.body = await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let tenants = [] + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + tenants = tenantsDoc.tenantIds + } + } catch (err) { + // if error it doesn't exist + } + return tenants + } + ) +} + +const _delete = async (ctx: any) => { + const tenantId = getTenantId() + + if (ctx.params.tenantId !== tenantId) { + ctx.throw(403, "Unauthorized") + } + + try { + await deleteTenant(tenantId) + await quotas.bustCache() + ctx.status = 204 + } catch (err) { + ctx.log.error(err) + throw err + } +} + +export { _delete as delete } diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts index 6900596216..19ef076a52 100644 --- a/packages/worker/src/migrations/index.ts +++ b/packages/worker/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { migrations, locks } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -42,33 +48,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } diff --git a/packages/worker/src/sdk/users/events.ts b/packages/worker/src/sdk/users/events.ts index 0094c6fd84..3046442393 100644 --- a/packages/worker/src/sdk/users/events.ts +++ b/packages/worker/src/sdk/users/events.ts @@ -1,7 +1,6 @@ import env from "../../environment" import { events, accounts, tenancy } from "@budibase/backend-core" import { User, UserRoles, CloudAccount } from "@budibase/types" -import { users as pro } from "@budibase/pro" export const handleDeleteEvents = async (user: any) => { await events.user.deleted(user) diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 6cf3a6556d..92a8ef024d 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.30-alpha.0": - version "2.0.30-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.0.tgz#aa9277ec26da51f8396fe047ebab39a0dff69c93" - integrity sha512-6+KHAceEZv1SC0qSA4GTNQBAlF1wnr9jfl85D5Tq1O2caD4J7drkIz8D6e0/DVv8zccHT+aUA7hy7YhKxbHAUA== +"@budibase/backend-core@2.0.30-alpha.3": + version "2.0.30-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.3.tgz#42235b33676c3357d8cf8e181e619c4c92065fd0" + integrity sha512-ixE28ddY8FqvCcaCkqsxUI5UjkFKxFu23MgNARJ8VSpj5NLq6d8qAR+xE0E2pJ58hm5r1KLn2yAZiNXZzu0dzg== dependencies: - "@budibase/types" "2.0.30-alpha.0" + "@budibase/types" "2.0.30-alpha.3" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -327,21 +327,21 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.0.30-alpha.0": - version "2.0.30-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.0.tgz#6f0d9e45bc458e0e423280355c1a339f3143da47" - integrity sha512-8PFdiIHT7wUusa1Gi+KvcpLaJdBAtnAK6bfYGayasnrS9OIasfIBAMgzXRONMAGMvY7PZz84psZVimXSkW7EEA== +"@budibase/pro@2.0.30-alpha.3": + version "2.0.30-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.3.tgz#ad8f8aa5ced2b5200e05b98e392267b9f84c8177" + integrity sha512-Cw7XUGVVbHCxKo3tJFvG5EKUvO0zdOoU/baFArjMiNHAivlZvFVWCsxptIua+qtFkaUpjQneTHihFiQnY9g83A== dependencies: - "@budibase/backend-core" "2.0.30-alpha.0" - "@budibase/types" "2.0.30-alpha.0" + "@budibase/backend-core" "2.0.30-alpha.3" + "@budibase/types" "2.0.30-alpha.3" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.0.30-alpha.0": - version "2.0.30-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.0.tgz#fb5e80870b00850601785f1102f64bd22339f4e9" - integrity sha512-i9c02crPzZPRzn+RXwn2f4H4WCLWhC7HlsiXH2w4Ngk7PJvtcR9Jn7mKBKVcXM/9PAK6d0vWIQidA8XuU0CpZA== +"@budibase/types@2.0.30-alpha.3": + version "2.0.30-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.3.tgz#cb55bcced75b711cc8a675284fbacaa8ebf1c0f2" + integrity sha512-rHeFVuNbSSE4fMnX6uyrM2r47m+neqFXlVNOkhHU9i7KoIcIZbEYInU8CjUFR2da3ruST9ajXjJ5UenX2+MnTg== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0"