diff --git a/hosting/proxy/10-listen-on-ipv6-by-default.sh b/hosting/proxy/10-listen-on-ipv6-by-default.sh new file mode 100644 index 0000000000..e2e89388a9 --- /dev/null +++ b/hosting/proxy/10-listen-on-ipv6-by-default.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# vim:sw=4:ts=4:et + +set -e + +ME=$(basename $0) +NGINX_CONF_FILE="/etc/nginx/nginx.conf" +DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf" + +# check if we have ipv6 available +if [ ! -f "/proc/net/if_inet6" ]; then + # ipv6 not available so delete lines from nginx conf + if [ -f "$NGINX_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $NGINX_CONF_FILE + fi + if [ -f "$DEFAULT_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE + fi + echo "$ME: info: ipv6 not available so delete lines from nginx conf" +else + echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf" +fi + +exit 0 diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index 298762aaf1..5fd0dc7d11 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -5,7 +5,7 @@ FROM nginx:latest # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template - +COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh # Error handling COPY error.html /usr/share/nginx/html/error.html diff --git a/lerna.json b/lerna.json index 86c6ac647a..1d092b043c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.0.24-alpha.4", + "version": "2.0.30-alpha.3", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 4b8fcb951e..abcf72491a 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.0.24-alpha.4", + "version": "2.0.30-alpha.3", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "2.0.24-alpha.4", + "@budibase/types": "2.0.30-alpha.3", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", @@ -62,6 +62,7 @@ ] }, "devDependencies": { + "@types/chance": "1.1.3", "@types/jest": "27.5.1", "@types/koa": "2.0.52", "@types/lodash": "4.14.180", @@ -72,6 +73,7 @@ "@types/semver": "7.3.7", "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", + "chance": "1.1.3", "ioredis-mock": "5.8.0", "jest": "27.5.1", "koa": "2.7.0", diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 83b23b479d..42cad17620 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -37,6 +37,7 @@ const core = { db, ...dbConstants, redis, + locks: redis.redlock, objectStore, utils, users, diff --git a/packages/backend-core/src/pkg/redis.ts b/packages/backend-core/src/pkg/redis.ts index 65ab186d9a..297c2b54f4 100644 --- a/packages/backend-core/src/pkg/redis.ts +++ b/packages/backend-core/src/pkg/redis.ts @@ -3,9 +3,11 @@ import Client from "../redis" import utils from "../redis/utils" import clients from "../redis/init" +import * as redlock from "../redis/redlock" export = { Client, utils, clients, + redlock, } diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts index 62e718d8ad..8a15320ff3 100644 --- a/packages/backend-core/src/redis/index.ts +++ b/packages/backend-core/src/redis/index.ts @@ -216,6 +216,9 @@ export = class RedisWrapper { async bulkGet(keys: string[]) { const db = this._db + if (keys.length === 0) { + return {} + } const prefixedKeys = keys.map(key => addDbPrefix(db, key)) let response = await this.getClient().mget(prefixedKeys) if (Array.isArray(response)) { diff --git a/packages/backend-core/src/redis/init.js b/packages/backend-core/src/redis/init.js index 8e5d10f838..3150ef2c1c 100644 --- a/packages/backend-core/src/redis/init.js +++ b/packages/backend-core/src/redis/init.js @@ -1,27 +1,23 @@ const Client = require("./index") const utils = require("./utils") -const { getRedlock } = require("./redlock") -let userClient, sessionClient, appClient, cacheClient, writethroughClient -let migrationsRedlock - -// turn retry off so that only one instance can ever hold the lock -const migrationsRedlockConfig = { retryCount: 0 } +let userClient, + sessionClient, + appClient, + cacheClient, + writethroughClient, + lockClient async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() + lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client( utils.Databases.WRITE_THROUGH, utils.SelectableDatabases.WRITE_THROUGH ).init() - // pass the underlying ioredis client to redlock - migrationsRedlock = getRedlock( - cacheClient.getClient(), - migrationsRedlockConfig - ) } process.on("exit", async () => { @@ -30,6 +26,7 @@ process.on("exit", async () => { if (appClient) await appClient.finish() if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() + if (lockClient) await lockClient.finish() }) module.exports = { @@ -63,10 +60,10 @@ module.exports = { } return writethroughClient }, - getMigrationsRedlock: async () => { - if (!migrationsRedlock) { + getLockClient: async () => { + if (!lockClient) { await init() } - return migrationsRedlock + return lockClient }, } diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlock.ts index beef375b55..abb13b2534 100644 --- a/packages/backend-core/src/redis/redlock.ts +++ b/packages/backend-core/src/redis/redlock.ts @@ -1,14 +1,37 @@ -import Redlock from "redlock" +import Redlock, { Options } from "redlock" +import { getLockClient } from "./init" +import { LockOptions, LockType } from "@budibase/types" +import * as tenancy from "../tenancy" -export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { - return new Redlock([redisClient], { +let noRetryRedlock: Redlock | undefined + +const getClient = async (type: LockType): Promise => { + switch (type) { + case LockType.TRY_ONCE: { + if (!noRetryRedlock) { + noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE) + } + return noRetryRedlock + } + default: { + throw new Error(`Could not get redlock client: ${type}`) + } + } +} + +export const OPTIONS = { + TRY_ONCE: { + // immediately throws an error if the lock is already held + retryCount: 0, + }, + DEFAULT: { // the expected clock drift; for more details // see http://redis.io/topics/distlock driftFactor: 0.01, // multiplied by lock ttl to determine drift time // the max number of times Redlock will attempt // to lock a resource before erroring - retryCount: opts.retryCount, + retryCount: 10, // the time in ms between attempts retryDelay: 200, // time in ms @@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { // the max time in ms randomly added to retries // to improve performance under high contention // see https://www.awsarchitectureblog.com/2015/03/backoff.html - retryJitter: 200, // time in ms - }) + retryJitter: 100, // time in ms + }, +} + +export const newRedlock = async (opts: Options = {}) => { + let options = { ...OPTIONS.DEFAULT, ...opts } + const redisWrapper = await getLockClient() + const client = redisWrapper.getClient() + return new Redlock([client], options) +} + +export const doWithLock = async (opts: LockOptions, task: any) => { + const redlock = await getClient(opts.type) + let lock + try { + // aquire lock + let name: string = `${tenancy.getTenantId()}_${opts.name}` + if (opts.nameSuffix) { + name = name + `_${opts.nameSuffix}` + } + lock = await redlock.lock(name, opts.ttl) + // perform locked task + return task() + } catch (e: any) { + // lock limit exceeded + if (e.name === "LockError") { + if (opts.type === LockType.TRY_ONCE) { + // don't throw for try-once locks, they will always error + // due to retry count (0) exceeded + return + } else { + throw e + } + } else { + throw e + } + } finally { + if (lock) { + await lock.unlock() + } + } } diff --git a/packages/backend-core/src/redis/utils.js b/packages/backend-core/src/redis/utils.js index 90b3561f31..af719197b5 100644 --- a/packages/backend-core/src/redis/utils.js +++ b/packages/backend-core/src/redis/utils.js @@ -28,6 +28,7 @@ exports.Databases = { LICENSES: "license", GENERIC_CACHE: "data_cache", WRITE_THROUGH: "writeThrough", + LOCKS: "locks", } /** diff --git a/packages/backend-core/tests/utilities/structures/accounts.ts b/packages/backend-core/tests/utilities/structures/accounts.ts new file mode 100644 index 0000000000..5d23962575 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/accounts.ts @@ -0,0 +1,23 @@ +import { generator, uuid } from "." +import { AuthType, CloudAccount, Hosting } from "@budibase/types" +import * as db from "../../../src/db/utils" + +export const cloudAccount = (): CloudAccount => { + return { + accountId: uuid(), + createdAt: Date.now(), + verified: true, + verificationSent: true, + tier: "", + email: generator.email(), + tenantId: generator.word(), + hosting: Hosting.CLOUD, + authType: AuthType.PASSWORD, + password: generator.word(), + tenantName: generator.word(), + name: generator.name(), + size: "10+", + profession: "Software Engineer", + budibaseUserId: db.generateGlobalUserID(), + } +} diff --git a/packages/backend-core/tests/utilities/structures/common.ts b/packages/backend-core/tests/utilities/structures/common.ts new file mode 100644 index 0000000000..51ae220254 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/common.ts @@ -0,0 +1 @@ +export { v4 as uuid } from "uuid" diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index 12b6ab7ad6..68064b9715 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -1 +1,8 @@ +export * from "./common" + +import Chance from "chance" +export const generator = new Chance() + export * as koa from "./koa" +export * as accounts from "./accounts" +export * as licenses from "./licenses" diff --git a/packages/backend-core/tests/utilities/structures/licenses.ts b/packages/backend-core/tests/utilities/structures/licenses.ts new file mode 100644 index 0000000000..a541e91860 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/licenses.ts @@ -0,0 +1,18 @@ +import { AccountPlan, License, PlanType, Quotas } from "@budibase/types" + +const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => { + return { + type, + } +} + +export const newLicense = (opts: { + quotas: Quotas + planType?: PlanType +}): License => { + return { + features: [], + quotas: opts.quotas, + plan: newPlan(opts.planType), + } +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 2e62aea734..6bc9b63728 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -663,6 +663,11 @@ "@types/connect" "*" "@types/node" "*" +"@types/chance@1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea" + integrity sha512-X6c6ghhe4/sQh4XzcZWSFaTAUOda38GQHmq9BUanYkOE/EO7ZrkazwKmtsj3xzTjkLWmwULE++23g3d3CCWaWw== + "@types/connect@*": version "3.4.35" resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1" @@ -1555,6 +1560,11 @@ chalk@^4.0.0, chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chance@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3" + integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg== + char-regex@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 0f9abb52a9..147a1b3785 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.0.24-alpha.4", + "version": "2.0.30-alpha.3", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "2.0.24-alpha.4", + "@budibase/string-templates": "2.0.30-alpha.3", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/cypress/integration/autoScreensUI.spec.js b/packages/builder/cypress/integration/autoScreensUI.spec.js index a22dbb0a1e..581e5c431b 100644 --- a/packages/builder/cypress/integration/autoScreensUI.spec.js +++ b/packages/builder/cypress/integration/autoScreensUI.spec.js @@ -2,7 +2,7 @@ import filterTests from "../support/filterTests" const interact = require('../support/interact') filterTests(['smoke', 'all'], () => { - context("Auto Screens UI", () => { + xcontext("Auto Screens UI", () => { before(() => { cy.login() cy.deleteAllApps() diff --git a/packages/builder/cypress/integration/datasources/postgreSql.spec.js b/packages/builder/cypress/integration/datasources/postgreSql.spec.js index 8ef574566e..e1aa0ff128 100644 --- a/packages/builder/cypress/integration/datasources/postgreSql.spec.js +++ b/packages/builder/cypress/integration/datasources/postgreSql.spec.js @@ -1,7 +1,7 @@ import filterTests from "../../support/filterTests" filterTests(["all"], () => { - context("PostgreSQL Datasource Testing", () => { + xcontext("PostgreSQL Datasource Testing", () => { if (Cypress.env("TEST_ENV")) { before(() => { cy.login() diff --git a/packages/builder/package.json b/packages/builder/package.json index 9d68b77028..8ced899d20 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.0.24-alpha.4", + "version": "2.0.30-alpha.3", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.0.24-alpha.4", - "@budibase/client": "2.0.24-alpha.4", - "@budibase/frontend-core": "2.0.24-alpha.4", - "@budibase/string-templates": "2.0.24-alpha.4", + "@budibase/bbui": "2.0.30-alpha.3", + "@budibase/client": "2.0.30-alpha.3", + "@budibase/frontend-core": "2.0.30-alpha.3", + "@budibase/string-templates": "2.0.30-alpha.3", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte index e2ccab11af..600e331d3e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte @@ -17,12 +17,21 @@ $: selectedRoleId = selectedRole._id $: otherRoles = editableRoles.filter(role => role._id !== selectedRoleId) $: isCreating = selectedRoleId == null || selectedRoleId === "" + + $: hasUniqueRoleName = !otherRoles + ?.map(role => role.name) + ?.includes(selectedRole.name) + $: valid = selectedRole.name && selectedRole.inherits && selectedRole.permissionId && !builtInRoles.includes(selectedRole.name) + $: shouldDisableRoleInput = + builtInRoles.includes(selectedRole.name) && + selectedRole.name?.toLowerCase() === selectedRoleId?.toLowerCase() + const fetchBasePermissions = async () => { try { basePermissions = await API.getBasePermissions() @@ -99,7 +108,7 @@ title="Edit Roles" confirmText={isCreating ? "Create" : "Save"} onConfirm={saveRole} - disabled={!valid} + disabled={!valid || !hasUniqueRoleName} > {#if errors.length} @@ -119,15 +128,16 @@ x._id} getOptionLabel={x => x.name} - disabled={builtInRoles.includes(selectedRole.name)} + disabled={shouldDisableRoleInput} /> {/if}
- {#if !isCreating} + {#if !isCreating && !builtInRoles.includes(selectedRole.name)} {/if}
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte index cef49d81a1..ff413094a0 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte @@ -209,27 +209,29 @@ {:else} No tables found. {/if} - -
- Relationships - -
- - Tell budibase how your tables are related to get even more smart features. - -{#if relationshipInfo && relationshipInfo.length > 0} - openRelationshipModal(detail.from, detail.to)} - schema={relationshipSchema} - data={relationshipInfo} - allowEditColumns={false} - allowEditRows={false} - allowSelectRows={false} - /> -{:else} - No relationships configured. +{#if integration.relationships !== false} + +
+ Relationships + +
+ + Tell budibase how your tables are related to get even more smart features. + + {#if relationshipInfo && relationshipInfo.length > 0} +
openRelationshipModal(detail.from, detail.to)} + schema={relationshipSchema} + data={relationshipInfo} + allowEditColumns={false} + allowEditRows={false} + allowSelectRows={false} + /> + {:else} + No relationships configured. + {/if} {/if}