CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-06-16 09:25:12 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'develop' of github.com:Budibase/budibase into feature/app-backups

Browse source
This commit is contained in:
mike12345567 2022-10-13 15:59:31 +01:00
parent 488e0c44c0 67ab376e63
commit 9242f695ab
43 changed files with 473 additions and 244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
hosting/proxy/10-listen-on-ipv6-by-default.sh Normal file
View file

@ -0,0 +1,24 @@
#!/bin/sh
# vim:sw=4:ts=4:et
set -e
ME=$(basename $0)
NGINX_CONF_FILE="/etc/nginx/nginx.conf"
DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf"
# check if we have ipv6 available
if [ ! -f "/proc/net/if_inet6" ]; then
# ipv6 not available so delete lines from nginx conf
if [ -f "$NGINX_CONF_FILE" ]; then
sed -i '/listen \[::\]/d' $NGINX_CONF_FILE
fi
if [ -f "$DEFAULT_CONF_FILE" ]; then
sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE
fi
echo "$ME: info: ipv6 not available so delete lines from nginx conf"
else
echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf"
fi
exit 0

2
hosting/proxy/Dockerfile
View file

@ -5,7 +5,7 @@ FROM nginx:latest
# override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
# Error handling
COPY error.html /usr/share/nginx/html/error.html

2
lerna.json
View file

@ -1,5 +1,5 @@
{
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"npmClient": "yarn",
"packages": [
"packages/*"

6
packages/backend-core/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"description": "Budibase backend core libraries used in server and worker",
"main": "dist/src/index.js",
"types": "dist/src/index.d.ts",
@ -20,7 +20,7 @@
"test:watch": "jest --watchAll"
},
"dependencies": {
"@budibase/types": "2.0.30-alpha.0",
"@budibase/types": "2.0.30-alpha.3",
"@shopify/jest-koa-mocks": "5.0.1",
"@techpass/passport-openidconnect": "0.3.2",
"aws-sdk": "2.1030.0",
@ -62,6 +62,7 @@
]
},
"devDependencies": {
"@types/chance": "1.1.3",
"@types/jest": "27.5.1",
"@types/koa": "2.0.52",
"@types/lodash": "4.14.180",
@ -72,6 +73,7 @@
"@types/semver": "7.3.7",
"@types/tar-fs": "2.0.1",
"@types/uuid": "8.3.4",
"chance": "1.1.3",
"ioredis-mock": "5.8.0",
"jest": "27.5.1",
"koa": "2.7.0",

1
packages/backend-core/src/index.ts
View file

@ -37,6 +37,7 @@ const core = {
db,
...dbConstants,
redis,
locks: redis.redlock,
objectStore,
utils,
users,

2
packages/backend-core/src/pkg/redis.ts
View file

@ -3,9 +3,11 @@
import Client from "../redis"
import utils from "../redis/utils"
import clients from "../redis/init"
import * as redlock from "../redis/redlock"
export = {
Client,
utils,
clients,
redlock,
}

25
packages/backend-core/src/redis/init.js
View file

@ -1,27 +1,23 @@
const Client = require("./index")
const utils = require("./utils")
const { getRedlock } = require("./redlock")
let userClient, sessionClient, appClient, cacheClient, writethroughClient
let migrationsRedlock
// turn retry off so that only one instance can ever hold the lock
const migrationsRedlockConfig = { retryCount: 0 }
let userClient,
sessionClient,
appClient,
cacheClient,
writethroughClient,
lockClient
async function init() {
userClient = await new Client(utils.Databases.USER_CACHE).init()
sessionClient = await new Client(utils.Databases.SESSIONS).init()
appClient = await new Client(utils.Databases.APP_METADATA).init()
cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
lockClient = await new Client(utils.Databases.LOCKS).init()
writethroughClient = await new Client(
utils.Databases.WRITE_THROUGH,
utils.SelectableDatabases.WRITE_THROUGH
).init()
// pass the underlying ioredis client to redlock
migrationsRedlock = getRedlock(
cacheClient.getClient(),
migrationsRedlockConfig
)
}
process.on("exit", async () => {
@ -30,6 +26,7 @@ process.on("exit", async () => {
if (appClient) await appClient.finish()
if (cacheClient) await cacheClient.finish()
if (writethroughClient) await writethroughClient.finish()
if (lockClient) await lockClient.finish()
})
module.exports = {
@ -63,10 +60,10 @@ module.exports = {
}
return writethroughClient
},
getMigrationsRedlock: async () => {
if (!migrationsRedlock) {
getLockClient: async () => {
if (!lockClient) {
await init()
}
return migrationsRedlock
return lockClient
},
}

74
packages/backend-core/src/redis/redlock.ts
View file

@ -1,14 +1,37 @@
import Redlock from "redlock"
import Redlock, { Options } from "redlock"
import { getLockClient } from "./init"
import { LockOptions, LockType } from "@budibase/types"
import * as tenancy from "../tenancy"
export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
return new Redlock([redisClient], {
let noRetryRedlock: Redlock | undefined
const getClient = async (type: LockType): Promise<Redlock> => {
switch (type) {
case LockType.TRY_ONCE: {
if (!noRetryRedlock) {
noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE)
}
return noRetryRedlock
}
default: {
throw new Error(`Could not get redlock client: ${type}`)
}
}
}
export const OPTIONS = {
TRY_ONCE: {
// immediately throws an error if the lock is already held
retryCount: 0,
},
DEFAULT: {
// the expected clock drift; for more details
// see http://redis.io/topics/distlock
driftFactor: 0.01, // multiplied by lock ttl to determine drift time
// the max number of times Redlock will attempt
// to lock a resource before erroring
retryCount: opts.retryCount,
retryCount: 10,
// the time in ms between attempts
retryDelay: 200, // time in ms
@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
// the max time in ms randomly added to retries
// to improve performance under high contention
// see https://www.awsarchitectureblog.com/2015/03/backoff.html
retryJitter: 200, // time in ms
})
retryJitter: 100, // time in ms
},
}
export const newRedlock = async (opts: Options = {}) => {
let options = { ...OPTIONS.DEFAULT, ...opts }
const redisWrapper = await getLockClient()
const client = redisWrapper.getClient()
return new Redlock([client], options)
}
export const doWithLock = async (opts: LockOptions, task: any) => {
const redlock = await getClient(opts.type)
let lock
try {
// aquire lock
let name: string = `${tenancy.getTenantId()}_${opts.name}`
if (opts.nameSuffix) {
name = name + `_${opts.nameSuffix}`
}
lock = await redlock.lock(name, opts.ttl)
// perform locked task
return task()
} catch (e: any) {
// lock limit exceeded
if (e.name === "LockError") {
if (opts.type === LockType.TRY_ONCE) {
// don't throw for try-once locks, they will always error
// due to retry count (0) exceeded
return
} else {
throw e
}
} else {
throw e
}
} finally {
if (lock) {
await lock.unlock()
}
}
}

1
packages/backend-core/src/redis/utils.js
View file

@ -28,6 +28,7 @@ exports.Databases = {
LICENSES: "license",
GENERIC_CACHE: "data_cache",
WRITE_THROUGH: "writeThrough",
LOCKS: "locks",
}
/**

23
packages/backend-core/tests/utilities/structures/accounts.ts Normal file
View file

@ -0,0 +1,23 @@
import { generator, uuid } from "."
import { AuthType, CloudAccount, Hosting } from "@budibase/types"
import * as db from "../../../src/db/utils"
export const cloudAccount = (): CloudAccount => {
return {
accountId: uuid(),
createdAt: Date.now(),
verified: true,
verificationSent: true,
tier: "",
email: generator.email(),
tenantId: generator.word(),
hosting: Hosting.CLOUD,
authType: AuthType.PASSWORD,
password: generator.word(),
tenantName: generator.word(),
name: generator.name(),
size: "10+",
profession: "Software Engineer",
budibaseUserId: db.generateGlobalUserID(),
}
}

1
packages/backend-core/tests/utilities/structures/common.ts Normal file
View file

@ -0,0 +1 @@
export { v4 as uuid } from "uuid"

7
packages/backend-core/tests/utilities/structures/index.ts
View file

@ -1 +1,8 @@
export * from "./common"
import Chance from "chance"
export const generator = new Chance()
export * as koa from "./koa"
export * as accounts from "./accounts"
export * as licenses from "./licenses"

18
packages/backend-core/tests/utilities/structures/licenses.ts Normal file
View file

@ -0,0 +1,18 @@
import { AccountPlan, License, PlanType, Quotas } from "@budibase/types"
const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => {
return {
type,
}
}
export const newLicense = (opts: {
quotas: Quotas
planType?: PlanType
}): License => {
return {
features: [],
quotas: opts.quotas,
plan: newPlan(opts.planType),
}
}

10
packages/backend-core/yarn.lock
View file

@ -663,6 +663,11 @@
"@types/connect" "*"
"@types/node" "*"
"@types/chance@1.1.3":
version "1.1.3"
resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea"
integrity sha512-X6c6ghhe4/sQh4XzcZWSFaTAUOda38GQHmq9BUanYkOE/EO7ZrkazwKmtsj3xzTjkLWmwULE++23g3d3CCWaWw==
"@types/connect@*":
version "3.4.35"
resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1"
@ -1555,6 +1560,11 @@ chalk@^4.0.0, chalk@^4.1.0:
ansi-styles "^4.1.0"
supports-color "^7.1.0"
chance@1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3"
integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg==
char-regex@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"

4
packages/bbui/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
],
"dependencies": {
"@adobe/spectrum-css-workflow-icons": "^1.2.1",
"@budibase/string-templates": "2.0.30-alpha.0",
"@budibase/string-templates": "2.0.30-alpha.3",
"@spectrum-css/actionbutton": "^1.0.1",
"@spectrum-css/actiongroup": "^1.0.1",
"@spectrum-css/avatar": "^3.0.2",

10
packages/builder/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -71,10 +71,10 @@
}
},
"dependencies": {
"@budibase/bbui": "2.0.30-alpha.0",
"@budibase/client": "2.0.30-alpha.0",
"@budibase/frontend-core": "2.0.30-alpha.0",
"@budibase/string-templates": "2.0.30-alpha.0",
"@budibase/bbui": "2.0.30-alpha.3",
"@budibase/client": "2.0.30-alpha.3",
"@budibase/frontend-core": "2.0.30-alpha.3",
"@budibase/string-templates": "2.0.30-alpha.3",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

1
packages/builder/src/pages/builder/portal/settings/upgrade.svelte
View file

@ -91,7 +91,6 @@
})
</script>
{"is adming" + $auth.isAdmin}
{#if $auth.isAdmin}
<DeleteLicenseKeyModal
bind:this={deleteLicenseKeyModal}

8
packages/cli/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {
@ -26,9 +26,9 @@
"outputPath": "build"
},
"dependencies": {
"@budibase/backend-core": "2.0.30-alpha.0",
"@budibase/string-templates": "2.0.30-alpha.0",
"@budibase/types": "2.0.30-alpha.0",
"@budibase/backend-core": "2.0.30-alpha.3",
"@budibase/string-templates": "2.0.30-alpha.3",
"@budibase/types": "2.0.30-alpha.3",
"axios": "0.21.2",
"chalk": "4.1.0",
"cli-progress": "3.11.2",

8
packages/client/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "2.0.30-alpha.0",
"@budibase/frontend-core": "2.0.30-alpha.0",
"@budibase/string-templates": "2.0.30-alpha.0",
"@budibase/bbui": "2.0.30-alpha.3",
"@budibase/frontend-core": "2.0.30-alpha.3",
"@budibase/string-templates": "2.0.30-alpha.3",
"@spectrum-css/button": "^3.0.3",
"@spectrum-css/card": "^3.0.3",
"@spectrum-css/divider": "^1.0.3",

4
packages/frontend-core/package.json
View file

@ -1,12 +1,12 @@
{
"name": "@budibase/frontend-core",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase",
"license": "MPL-2.0",
"svelte": "src/index.js",
"dependencies": {
"@budibase/bbui": "2.0.30-alpha.0",
"@budibase/bbui": "2.0.30-alpha.3",
"lodash": "^4.17.21",
"svelte": "^3.46.2"
}

2
packages/sdk/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/sdk",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"description": "Budibase Public API SDK",
"author": "Budibase",
"license": "MPL-2.0",

12
packages/server/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -77,11 +77,11 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "10.0.3",
"@budibase/backend-core": "2.0.30-alpha.0",
"@budibase/client": "2.0.30-alpha.0",
"@budibase/pro": "2.0.30-alpha.0",
"@budibase/string-templates": "2.0.30-alpha.0",
"@budibase/types": "2.0.30-alpha.0",
"@budibase/backend-core": "2.0.30-alpha.3",
"@budibase/client": "2.0.30-alpha.3",
"@budibase/pro": "2.0.30-alpha.3",
"@budibase/string-templates": "2.0.30-alpha.3",
"@budibase/types": "2.0.30-alpha.3",
"@bull-board/api": "3.7.0",
"@bull-board/koa": "3.9.4",
"@elastic/elasticsearch": "7.10.0",

1
packages/server/src/api/controllers/deploy/index.ts
View file

@ -17,7 +17,6 @@ import {
getProdAppDB,
getDevAppDB,
} from "@budibase/backend-core/context"
import { quotas } from "@budibase/pro"
import { events } from "@budibase/backend-core"
// the max time we can wait for an invalidation to complete before considering it failed

1
packages/server/src/api/controllers/query/index.ts
View file

@ -56,6 +56,7 @@ const _import = async (ctx: any) => {
config: {
url: info.url,
defaultHeaders: [],
rejectUnauthorized: true,
},
name: info.name,
}

8
packages/server/src/integrations/rest.ts
View file

@ -78,16 +78,11 @@ const SCHEMA: Integration = {
default: {},
},
rejectUnauthorized: {
display: "Reject Unauthorized",
type: DatasourceFieldType.BOOLEAN,
default: true,
required: false,
},
legacyHttpParser: {
display: "Legacy HTTP Support",
type: DatasourceFieldType.BOOLEAN,
required: false,
default: false,
},
},
query: {
create: {
@ -397,6 +392,7 @@ class RestIntegration implements IntegrationBase {
})
}
// Deprecated by rejectUnauthorized
if (this.config.legacyHttpParser) {
// https://github.com/nodejs/node/issues/43798
input.extraHttpOptions = { insecureHTTPParser: true }

42
packages/server/src/integrations/tests/rest.spec.ts
View file

@ -1,17 +1,16 @@
jest.mock("node-fetch", () =>
jest.fn(() => ({
headers: {
raw: () => {
return { "content-type": ["application/json"] }
},
get: () => ["application/json"],
const mockFetch = jest.fn(() => ({
headers: {
raw: () => {
return { "content-type": ["application/json"] }
},
json: jest.fn(() => ({
my_next_cursor: 123,
})),
text: jest.fn(),
}))
)
get: () => ["application/json"],
},
json: jest.fn(() => ({
my_next_cursor: 123,
})),
text: jest.fn(),
}))
jest.mock("node-fetch", () => mockFetch)
import fetch from "node-fetch"
import { default as RestIntegration } from "../rest"
const FormData = require("form-data")
@ -572,4 +571,21 @@ describe("REST Integration", () => {
})
})
})
it("Attaches custom agent when Reject Unauthorized option is false", async () => {
config = new TestConfiguration({
url: BASE_URL,
rejectUnauthorized: false,
})
await config.integration.read({})
const calls: any = mockFetch.mock.calls[0]
const url = calls[0]
expect(url).toBe(`${BASE_URL}/`)
const calledConfig = calls[1]
expect(calledConfig.method).toBe("GET")
expect(calledConfig.headers).toEqual({})
expect(calledConfig.agent.options.rejectUnauthorized).toBe(false)
})
})

47
packages/server/src/migrations/index.ts
View file

@ -1,5 +1,11 @@
import { migrations, redis } from "@budibase/backend-core"
import { Migration, MigrationOptions, MigrationName } from "@budibase/types"
import { locks, migrations } from "@budibase/backend-core"
import {
Migration,
MigrationOptions,
MigrationName,
LockType,
LockName,
} from "@budibase/types"
import env from "../environment"
// migration functions
@ -86,33 +92,14 @@ export const migrate = async (options?: MigrationOptions) => {
}
const migrateWithLock = async (options?: MigrationOptions) => {
// get a new lock client
const redlock = await redis.clients.getMigrationsRedlock()
// lock for 15 minutes
const ttl = 1000 * 60 * 15
let migrationLock
// acquire lock
try {
migrationLock = await redlock.lock("migrations", ttl)
} catch (e: any) {
if (e.name === "LockError") {
return
} else {
throw e
await locks.doWithLock(
{
type: LockType.TRY_ONCE,
name: LockName.MIGRATIONS,
ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
},
async () => {
await migrations.runMigrations(MIGRATIONS, options)
}
}
// run migrations
try {
await migrations.runMigrations(MIGRATIONS, options)
} finally {
// release lock
try {
await migrationLock.unlock()
} catch (e) {
console.error("unable to release migration lock")
}
}
)
}

30
packages/server/yarn.lock
View file

@ -1094,12 +1094,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.0.30-alpha.0":
version "2.0.30-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.0.tgz#aa9277ec26da51f8396fe047ebab39a0dff69c93"
integrity sha512-6+KHAceEZv1SC0qSA4GTNQBAlF1wnr9jfl85D5Tq1O2caD4J7drkIz8D6e0/DVv8zccHT+aUA7hy7YhKxbHAUA==
"@budibase/backend-core@2.0.30-alpha.3":
version "2.0.30-alpha.3"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.3.tgz#42235b33676c3357d8cf8e181e619c4c92065fd0"
integrity sha512-ixE28ddY8FqvCcaCkqsxUI5UjkFKxFu23MgNARJ8VSpj5NLq6d8qAR+xE0E2pJ58hm5r1KLn2yAZiNXZzu0dzg==
dependencies:
"@budibase/types" "2.0.30-alpha.0"
"@budibase/types" "2.0.30-alpha.3"
"@shopify/jest-koa-mocks" "5.0.1"
"@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0"
@ -1180,13 +1180,13 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/pro@2.0.30-alpha.0":
version "2.0.30-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.0.tgz#6f0d9e45bc458e0e423280355c1a339f3143da47"
integrity sha512-8PFdiIHT7wUusa1Gi+KvcpLaJdBAtnAK6bfYGayasnrS9OIasfIBAMgzXRONMAGMvY7PZz84psZVimXSkW7EEA==
"@budibase/pro@2.0.30-alpha.3":
version "2.0.30-alpha.3"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.3.tgz#ad8f8aa5ced2b5200e05b98e392267b9f84c8177"
integrity sha512-Cw7XUGVVbHCxKo3tJFvG5EKUvO0zdOoU/baFArjMiNHAivlZvFVWCsxptIua+qtFkaUpjQneTHihFiQnY9g83A==
dependencies:
"@budibase/backend-core" "2.0.30-alpha.0"
"@budibase/types" "2.0.30-alpha.0"
"@budibase/backend-core" "2.0.30-alpha.3"
"@budibase/types" "2.0.30-alpha.3"
"@koa/router" "8.0.8"
joi "17.6.0"
node-fetch "^2.6.1"
@ -1209,10 +1209,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/types@2.0.30-alpha.0":
version "2.0.30-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.0.tgz#fb5e80870b00850601785f1102f64bd22339f4e9"
integrity sha512-i9c02crPzZPRzn+RXwn2f4H4WCLWhC7HlsiXH2w4Ngk7PJvtcR9Jn7mKBKVcXM/9PAK6d0vWIQidA8XuU0CpZA==
"@budibase/types@2.0.30-alpha.3":
version "2.0.30-alpha.3"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.3.tgz#cb55bcced75b711cc8a675284fbacaa8ebf1c0f2"
integrity sha512-rHeFVuNbSSE4fMnX6uyrM2r47m+neqFXlVNOkhHU9i7KoIcIZbEYInU8CjUFR2da3ruST9ajXjJ5UenX2+MnTg==
"@bull-board/api@3.7.0":
version "3.7.0"

2
packages/string-templates/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

2
packages/types/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/types",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"description": "Budibase types",
"main": "dist/index.js",
"types": "dist/index.d.ts",

1
packages/types/src/api/account/index.ts
View file

@ -1 +1,2 @@
export * from "./user"
export * from "./license"

11
packages/types/src/api/account/license.ts Normal file
View file

@ -0,0 +1,11 @@
import { QuotaUsage } from "../../documents"
export interface GetLicenseRequest {
quotaUsage: QuotaUsage
}
export interface QuotaTriggeredRequest {
percentage: number
name: string
resetDate?: string
}

11
packages/types/src/documents/account/account.ts
View file

@ -1,4 +1,12 @@
import { Feature, Hosting, PlanType, Quotas } from "../../sdk"
import {
Feature,
Hosting,
MonthlyQuotaName,
PlanType,
Quotas,
StaticQuotaName,
} from "../../sdk"
import { MonthlyUsage, QuotaUsage, StaticUsage } from "../global"
export interface CreateAccount {
email: string
@ -42,6 +50,7 @@ export interface Account extends CreateAccount {
licenseKey?: string
licenseKeyActivatedAt?: number
licenseOverrides?: LicenseOverrides
quotaUsage?: QuotaUsage
}
export interface PasswordAccount extends Account {

30
packages/types/src/documents/global/quotas.ts
View file

@ -24,19 +24,34 @@ export interface UsageBreakdown {
}
}
export type MonthlyUsage = {
export type QuotaTriggers = {
[key: string]: string | undefined
}
export interface StaticUsage {
[StaticQuotaName.APPS]: number
[StaticQuotaName.PLUGINS]: number
[StaticQuotaName.USER_GROUPS]: number
[StaticQuotaName.ROWS]: number
triggers: {
[key in StaticQuotaName]?: QuotaTriggers
}
}
export interface MonthlyUsage {
[MonthlyQuotaName.QUERIES]: number
[MonthlyQuotaName.AUTOMATIONS]: number
[MonthlyQuotaName.DAY_PASSES]: number
triggers: {
[key in MonthlyQuotaName]?: QuotaTriggers
}
breakdown?: {
[key in BreakdownQuotaName]?: UsageBreakdown
}
}
export interface BaseQuotaUsage {
usageQuota: {
[key in StaticQuotaName]: number
}
usageQuota: StaticUsage
monthly: {
[key: string]: MonthlyUsage
}
@ -51,6 +66,13 @@ export interface QuotaUsage extends BaseQuotaUsage {
}
}
export type SetUsageValues = {
total: number
app?: number
breakdown?: number
triggers?: QuotaTriggers
}
export type UsageValues = {
total: number
app?: number

1
packages/types/src/sdk/index.ts
View file

@ -7,3 +7,4 @@ export * from "./datasources"
export * from "./search"
export * from "./koa"
export * from "./auth"
export * from "./locks"

42
packages/types/src/sdk/licensing/quota.ts
View file

@ -61,26 +61,40 @@ export type PlanQuotas = {
[PlanType.ENTERPRISE]: Quotas
}
export type MonthlyQuotas = {
[MonthlyQuotaName.QUERIES]: Quota
[MonthlyQuotaName.AUTOMATIONS]: Quota
[MonthlyQuotaName.DAY_PASSES]: Quota
}
export type StaticQuotas = {
[StaticQuotaName.ROWS]: Quota
[StaticQuotaName.APPS]: Quota
[StaticQuotaName.USER_GROUPS]: Quota
[StaticQuotaName.PLUGINS]: Quota
}
export type ConstantQuotas = {
[ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota
}
export type Quotas = {
[QuotaType.USAGE]: {
[QuotaUsageType.MONTHLY]: {
[MonthlyQuotaName.QUERIES]: Quota
[MonthlyQuotaName.AUTOMATIONS]: Quota
[MonthlyQuotaName.DAY_PASSES]: Quota
}
[QuotaUsageType.STATIC]: {
[StaticQuotaName.ROWS]: Quota
[StaticQuotaName.APPS]: Quota
[StaticQuotaName.USER_GROUPS]: Quota
[StaticQuotaName.PLUGINS]: Quota
}
}
[QuotaType.CONSTANT]: {
[ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota
[QuotaUsageType.MONTHLY]: MonthlyQuotas
[QuotaUsageType.STATIC]: StaticQuotas
}
[QuotaType.CONSTANT]: ConstantQuotas
}
export interface Quota {
name: string
value: number
/**
* Array of whole numbers (1-100) that dictate the percentage that this quota should trigger
* at in relation to the corresponding usage inside budibase.
*
* Triggering results in a budibase installation sending a request to account-portal,
* which can have subsequent effects such as sending emails to users.
*/
triggers: number[]
}

31
packages/types/src/sdk/locks.ts Normal file
View file

@ -0,0 +1,31 @@
export enum LockType {
/**
* If this lock is already held the attempted operation will not be performed.
* No retries will take place and no error will be thrown.
*/
TRY_ONCE = "try_once",
}
export enum LockName {
MIGRATIONS = "migrations",
TRIGGER_QUOTA = "trigger_quota",
}
export interface LockOptions {
/**
* The lock type determines which client to use
*/
type: LockType
/**
* The name for the lock
*/
name: LockName
/**
* The ttl to auto-expire the lock if not unlocked manually
*/
ttl: number
/**
* The suffix to add to the lock name for additional uniqueness
*/
nameSuffix?: string
}

10
packages/worker/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "2.0.30-alpha.0",
"version": "2.0.30-alpha.3",
"description": "Budibase background service",
"main": "src/index.ts",
"repository": {
@ -36,10 +36,10 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "2.0.30-alpha.0",
"@budibase/pro": "2.0.30-alpha.0",
"@budibase/string-templates": "2.0.30-alpha.0",
"@budibase/types": "2.0.30-alpha.0",
"@budibase/backend-core": "2.0.30-alpha.3",
"@budibase/pro": "2.0.30-alpha.3",
"@budibase/string-templates": "2.0.30-alpha.3",
"@budibase/types": "2.0.30-alpha.3",
"@koa/router": "8.0.8",
"@sentry/node": "6.17.7",
"@techpass/passport-openidconnect": "0.3.2",

58
packages/worker/src/api/controllers/system/tenants.js
View file

@ -1,58 +0,0 @@
const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db")
const { getTenantId } = require("@budibase/backend-core/tenancy")
const { deleteTenant } = require("@budibase/backend-core/deprovision")
const { quotas } = require("@budibase/pro")
exports.exists = async ctx => {
const tenantId = ctx.request.params
ctx.body = {
exists: await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
let exists = false
try {
const tenantsDoc = await db.get(
StaticDatabases.PLATFORM_INFO.docs.tenants
)
if (tenantsDoc) {
exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1
}
} catch (err) {
// if error it doesn't exist
}
return exists
}),
}
}
exports.fetch = async ctx => {
ctx.body = await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
let tenants = []
try {
const tenantsDoc = await db.get(
StaticDatabases.PLATFORM_INFO.docs.tenants
)
if (tenantsDoc) {
tenants = tenantsDoc.tenantIds
}
} catch (err) {
// if error it doesn't exist
}
return tenants
})
}
exports.delete = async ctx => {
const tenantId = getTenantId()
if (ctx.params.tenantId !== tenantId) {
ctx.throw(403, "Unauthorized")
}
try {
await deleteTenant(tenantId)
await quotas.bustCache()
ctx.status = 204
} catch (err) {
ctx.log.error(err)
throw err
}
}

66
packages/worker/src/api/controllers/system/tenants.ts Normal file
View file

@ -0,0 +1,66 @@
const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db")
const { getTenantId } = require("@budibase/backend-core/tenancy")
const { deleteTenant } = require("@budibase/backend-core/deprovision")
import { quotas } from "@budibase/pro"
export const exists = async (ctx: any) => {
const tenantId = ctx.request.params
ctx.body = {
exists: await doWithDB(
StaticDatabases.PLATFORM_INFO.name,
async (db: any) => {
let exists = false
try {
const tenantsDoc = await db.get(
StaticDatabases.PLATFORM_INFO.docs.tenants
)
if (tenantsDoc) {
exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1
}
} catch (err) {
// if error it doesn't exist
}
return exists
}
),
}
}
export const fetch = async (ctx: any) => {
ctx.body = await doWithDB(
StaticDatabases.PLATFORM_INFO.name,
async (db: any) => {
let tenants = []
try {
const tenantsDoc = await db.get(
StaticDatabases.PLATFORM_INFO.docs.tenants
)
if (tenantsDoc) {
tenants = tenantsDoc.tenantIds
}
} catch (err) {
// if error it doesn't exist
}
return tenants
}
)
}
const _delete = async (ctx: any) => {
const tenantId = getTenantId()
if (ctx.params.tenantId !== tenantId) {
ctx.throw(403, "Unauthorized")
}
try {
await deleteTenant(tenantId)
await quotas.bustCache()
ctx.status = 204
} catch (err) {
ctx.log.error(err)
throw err
}
}
export { _delete as delete }

47
packages/worker/src/migrations/index.ts
View file

@ -1,5 +1,11 @@
import { migrations, redis } from "@budibase/backend-core"
import { Migration, MigrationOptions, MigrationName } from "@budibase/types"
import { migrations, locks } from "@budibase/backend-core"
import {
Migration,
MigrationOptions,
MigrationName,
LockType,
LockName,
} from "@budibase/types"
import env from "../environment"
// migration functions
@ -42,33 +48,14 @@ export const migrate = async (options?: MigrationOptions) => {
}
const migrateWithLock = async (options?: MigrationOptions) => {
// get a new lock client
const redlock = await redis.clients.getMigrationsRedlock()
// lock for 15 minutes
const ttl = 1000 * 60 * 15
let migrationLock
// acquire lock
try {
migrationLock = await redlock.lock("migrations", ttl)
} catch (e: any) {
if (e.name === "LockError") {
return
} else {
throw e
await locks.doWithLock(
{
type: LockType.TRY_ONCE,
name: LockName.MIGRATIONS,
ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
},
async () => {
await migrations.runMigrations(MIGRATIONS, options)
}
}
// run migrations
try {
await migrations.runMigrations(MIGRATIONS, options)
} finally {
// release lock
try {
await migrationLock.unlock()
} catch (e) {
console.error("unable to release migration lock")
}
}
)
}

1
packages/worker/src/sdk/users/events.ts
View file

@ -1,7 +1,6 @@
import env from "../../environment"
import { events, accounts, tenancy } from "@budibase/backend-core"
import { User, UserRoles, CloudAccount } from "@budibase/types"
import { users as pro } from "@budibase/pro"
export const handleDeleteEvents = async (user: any) => {
await events.user.deleted(user)

30
packages/worker/yarn.lock
View file

@ -291,12 +291,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.0.30-alpha.0":
version "2.0.30-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.0.tgz#aa9277ec26da51f8396fe047ebab39a0dff69c93"
integrity sha512-6+KHAceEZv1SC0qSA4GTNQBAlF1wnr9jfl85D5Tq1O2caD4J7drkIz8D6e0/DVv8zccHT+aUA7hy7YhKxbHAUA==
"@budibase/backend-core@2.0.30-alpha.3":
version "2.0.30-alpha.3"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.3.tgz#42235b33676c3357d8cf8e181e619c4c92065fd0"
integrity sha512-ixE28ddY8FqvCcaCkqsxUI5UjkFKxFu23MgNARJ8VSpj5NLq6d8qAR+xE0E2pJ58hm5r1KLn2yAZiNXZzu0dzg==
dependencies:
"@budibase/types" "2.0.30-alpha.0"
"@budibase/types" "2.0.30-alpha.3"
"@shopify/jest-koa-mocks" "5.0.1"
"@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0"
@ -327,21 +327,21 @@
uuid "8.3.2"
zlib "1.0.5"
"@budibase/pro@2.0.30-alpha.0":
version "2.0.30-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.0.tgz#6f0d9e45bc458e0e423280355c1a339f3143da47"
integrity sha512-8PFdiIHT7wUusa1Gi+KvcpLaJdBAtnAK6bfYGayasnrS9OIasfIBAMgzXRONMAGMvY7PZz84psZVimXSkW7EEA==
"@budibase/pro@2.0.30-alpha.3":
version "2.0.30-alpha.3"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.3.tgz#ad8f8aa5ced2b5200e05b98e392267b9f84c8177"
integrity sha512-Cw7XUGVVbHCxKo3tJFvG5EKUvO0zdOoU/baFArjMiNHAivlZvFVWCsxptIua+qtFkaUpjQneTHihFiQnY9g83A==
dependencies:
"@budibase/backend-core" "2.0.30-alpha.0"
"@budibase/types" "2.0.30-alpha.0"
"@budibase/backend-core" "2.0.30-alpha.3"
"@budibase/types" "2.0.30-alpha.3"
"@koa/router" "8.0.8"
joi "17.6.0"
node-fetch "^2.6.1"
"@budibase/types@2.0.30-alpha.0":
version "2.0.30-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.0.tgz#fb5e80870b00850601785f1102f64bd22339f4e9"
integrity sha512-i9c02crPzZPRzn+RXwn2f4H4WCLWhC7HlsiXH2w4Ngk7PJvtcR9Jn7mKBKVcXM/9PAK6d0vWIQidA8XuU0CpZA==
"@budibase/types@2.0.30-alpha.3":
version "2.0.30-alpha.3"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.3.tgz#cb55bcced75b711cc8a675284fbacaa8ebf1c0f2"
integrity sha512-rHeFVuNbSSE4fMnX6uyrM2r47m+neqFXlVNOkhHU9i7KoIcIZbEYInU8CjUFR2da3ruST9ajXjJ5UenX2+MnTg==
"@cspotcode/source-map-consumer@0.8.0":
version "0.8.0"