diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6ace2303d9..4b37418621 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,6 +6,7 @@ labels: bug assignees: '' --- + **Checklist** - [ ] I have searched budibase discussions and github issues to check if my issue already exists diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index c64adb010f..d095e3c2a4 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -57,10 +57,9 @@ jobs: - run: yarn - run: yarn bootstrap - run: yarn test - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml name: codecov-umbrella verbose: true @@ -78,28 +77,28 @@ jobs: - run: yarn bootstrap - run: yarn test:pro - integration-test: - runs-on: ubuntu-latest - services: - couchdb: - image: ibmcom/couchdb3 - env: - COUCHDB_PASSWORD: budibase - COUCHDB_USER: budibase - ports: - - 4567:5984 - steps: - - uses: actions/checkout@v2 - - name: Use Node.js 14.x - uses: actions/setup-node@v1 - with: - node-version: 14.x - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH - - run: yarn - - run: yarn bootstrap - - run: yarn build - - run: | - cd qa-core - yarn - yarn api:test:ci +# integration-test: +# runs-on: ubuntu-latest +# services: +# couchdb: +# image: ibmcom/couchdb3 +# env: +# COUCHDB_PASSWORD: budibase +# COUCHDB_USER: budibase +# ports: +# - 4567:5984 +# steps: +# - uses: actions/checkout@v2 +# - name: Use Node.js 14.x +# uses: actions/setup-node@v1 +# with: +# node-version: 14.x +# - name: Install Pro +# run: yarn install:pro $BRANCH $BASE_BRANCH +# - run: yarn +# - run: yarn bootstrap +# - run: yarn build +# - run: | +# cd qa-core +# yarn +# yarn api:test:ci diff --git a/.github/workflows/deploy-preprod.yml b/.github/workflows/deploy-preprod.yml index 0df8d20405..57e2504ded 100644 --- a/.github/workflows/deploy-preprod.yml +++ b/.github/workflows/deploy-preprod.yml @@ -23,7 +23,6 @@ jobs: release_version=${{ github.event.inputs.version }} fi echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -38,7 +37,6 @@ jobs: -o values.preprod.yaml \ -L https://api.github.com/repos/budibase/budibase-infra/contents/kubernetes/budibase-preprod/values.yaml wc -l values.preprod.yaml - - name: Deploy to Preprod Environment uses: budibase/helm@v1.8.0 with: @@ -65,4 +63,4 @@ jobs: with: webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} content: "Preprod Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Pre-prod." - embed-title: ${{ env.RELEASE_VERSION }} \ No newline at end of file + embed-title: ${{ env.RELEASE_VERSION }} diff --git a/lerna.json b/lerna.json index 7db5c851c2..41735dd374 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.4.43", + "version": "2.4.44-alpha.1", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index fcc5acd8d3..8dff7c1867 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.4.43", + "version": "2.4.44-alpha.1", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "^2.4.43", + "@budibase/types": "2.4.44-alpha.1", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index 26c7cd4e26..fb2fd2cf51 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -199,7 +199,6 @@ export async function platformLogout(opts: PlatformLogoutOpts) { } else { // clear cookies clearCookie(ctx, Cookie.Auth) - clearCookie(ctx, Cookie.CurrentApp) } const sessionIds = sessions.map(({ sessionId }) => sessionId) diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index d346788121..a34f05e881 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -1,10 +1,13 @@ -import { structures, DBTestConfiguration } from "../../../tests" +import { + structures, + DBTestConfiguration, + expectFunctionWasCalledTimesWith, +} from "../../../tests" import { Writethrough } from "../writethrough" import { getDB } from "../../db" import tk from "timekeeper" -const START_DATE = Date.now() -tk.freeze(START_DATE) +tk.freeze(Date.now()) const DELAY = 5000 @@ -17,34 +20,99 @@ describe("writethrough", () => { const writethrough = new Writethrough(db, DELAY) const writethrough2 = new Writethrough(db2, DELAY) + const docId = structures.uuid() + + beforeEach(() => { + jest.clearAllMocks() + }) + describe("put", () => { - let first: any + let current: any it("should be able to store, will go to DB", async () => { await config.doInTenant(async () => { - const response = await writethrough.put({ _id: "test", value: 1 }) + const response = await writethrough.put({ + _id: docId, + value: 1, + }) const output = await db.get(response.id) - first = output + current = output expect(output.value).toBe(1) }) }) it("second put shouldn't update DB", async () => { await config.doInTenant(async () => { - const response = await writethrough.put({ ...first, value: 2 }) + const response = await writethrough.put({ ...current, value: 2 }) const output = await db.get(response.id) - expect(first._rev).toBe(output._rev) + expect(current._rev).toBe(output._rev) expect(output.value).toBe(1) }) }) it("should put it again after delay period", async () => { await config.doInTenant(async () => { - tk.freeze(START_DATE + DELAY + 1) - const response = await writethrough.put({ ...first, value: 3 }) + tk.freeze(Date.now() + DELAY + 1) + const response = await writethrough.put({ ...current, value: 3 }) const output = await db.get(response.id) - expect(response.rev).not.toBe(first._rev) + expect(response.rev).not.toBe(current._rev) expect(output.value).toBe(3) + + current = output + }) + }) + + it("should handle parallel DB updates ignoring conflicts", async () => { + await config.doInTenant(async () => { + tk.freeze(Date.now() + DELAY + 1) + const responses = await Promise.all([ + writethrough.put({ ...current, value: 4 }), + writethrough.put({ ...current, value: 4 }), + writethrough.put({ ...current, value: 4 }), + ]) + + const newRev = responses.map(x => x.rev).find(x => x !== current._rev) + expect(newRev).toBeDefined() + expect(responses.map(x => x.rev)).toEqual( + expect.arrayContaining([current._rev, current._rev, newRev]) + ) + expectFunctionWasCalledTimesWith( + console.warn, + 2, + "bb-warn: Ignoring redlock conflict in write-through cache" + ) + + const output = await db.get(current._id) + expect(output.value).toBe(4) + expect(output._rev).toBe(newRev) + + current = output + }) + }) + + it("should handle updates with documents falling behind", async () => { + await config.doInTenant(async () => { + tk.freeze(Date.now() + DELAY + 1) + + const id = structures.uuid() + await writethrough.put({ _id: id, value: 1 }) + const doc = await writethrough.get(id) + + // Updating document + tk.freeze(Date.now() + DELAY + 1) + await writethrough.put({ ...doc, value: 2 }) + + // Update with the old rev value + tk.freeze(Date.now() + DELAY + 1) + const res = await writethrough.put({ + ...doc, + value: 3, + }) + expect(res.ok).toBe(true) + + const output = await db.get(id) + expect(output.value).toBe(3) + expect(output._rev).toBe(res.rev) }) }) }) @@ -52,8 +120,8 @@ describe("writethrough", () => { describe("get", () => { it("should be able to retrieve", async () => { await config.doInTenant(async () => { - const response = await writethrough.get("test") - expect(response.value).toBe(3) + const response = await writethrough.get(docId) + expect(response.value).toBe(4) }) }) }) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index dc889d5b18..a3b1ecc08d 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,7 +1,8 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" import { logWarn } from "../logging" -import { Database } from "@budibase/types" +import { Database, Document, LockName, LockType } from "@budibase/types" +import * as locks from "../redis/redlockImpl" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null @@ -27,44 +28,62 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { return { doc, lastWrite: lastWrite || Date.now() } } -export async function put( +async function put( db: Database, - doc: any, + doc: Document, writeRateMs: number = DEFAULT_WRITE_RATE_MS ) { const cache = await getCache() const key = doc._id - let cacheItem: CacheItem | undefined = await cache.get(makeCacheKey(db, key)) + let cacheItem: CacheItem | undefined + if (key) { + cacheItem = await cache.get(makeCacheKey(db, key)) + } const updateDb = !cacheItem || cacheItem.lastWrite < Date.now() - writeRateMs let output = doc if (updateDb) { - const writeDb = async (toWrite: any) => { - // doc should contain the _id and _rev - const response = await db.put(toWrite) - output = { - ...doc, - _id: response.id, - _rev: response.rev, - } - } - try { - await writeDb(doc) - } catch (err: any) { - if (err.status !== 409) { - throw err - } else { - // Swallow 409s but log them - logWarn(`Ignoring conflict in write-through cache`) + const lockResponse = await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.PERSIST_WRITETHROUGH, + resource: key, + ttl: 1000, + }, + async () => { + const writeDb = async (toWrite: any) => { + // doc should contain the _id and _rev + const response = await db.put(toWrite, { force: true }) + output = { + ...doc, + _id: response.id, + _rev: response.rev, + } + } + try { + await writeDb(doc) + } catch (err: any) { + if (err.status !== 409) { + throw err + } else { + // Swallow 409s but log them + logWarn(`Ignoring conflict in write-through cache`) + } + } } + ) + if (!lockResponse.executed) { + logWarn(`Ignoring redlock conflict in write-through cache`) } } // if we are updating the DB then need to set the lastWrite to now cacheItem = makeCacheItem(output, updateDb ? null : cacheItem?.lastWrite) - await cache.store(makeCacheKey(db, key), cacheItem) + if (output._id) { + await cache.store(makeCacheKey(db, output._id), cacheItem) + } return { ok: true, id: output._id, rev: output._rev } } -export async function get(db: Database, id: string): Promise { +async function get(db: Database, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) let cacheItem: CacheItem = await cache.get(cacheKey) @@ -76,11 +95,7 @@ export async function get(db: Database, id: string): Promise { return cacheItem.doc } -export async function remove( - db: Database, - docOrId: any, - rev?: any -): Promise { +async function remove(db: Database, docOrId: any, rev?: any): Promise { const cache = await getCache() if (!docOrId) { throw new Error("No ID/Rev provided.") diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index b461497747..c279babb71 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -32,8 +32,7 @@ export async function getConfig( const db = context.getGlobalDB() try { // await to catch error - const config = (await db.get(generateConfigID(type))) as T - return config + return (await db.get(generateConfigID(type))) as T } catch (e: any) { if (e.status === 404) { return diff --git a/packages/backend-core/src/configs/tests/configs.spec.ts b/packages/backend-core/src/configs/tests/configs.spec.ts index 079f2ab681..45e56a2581 100644 --- a/packages/backend-core/src/configs/tests/configs.spec.ts +++ b/packages/backend-core/src/configs/tests/configs.spec.ts @@ -1,4 +1,9 @@ -import { DBTestConfiguration, generator, testEnv } from "../../../tests" +import { + DBTestConfiguration, + generator, + testEnv, + structures, +} from "../../../tests" import { ConfigType } from "@budibase/types" import env from "../../environment" import * as configs from "../configs" @@ -113,4 +118,71 @@ describe("configs", () => { }) }) }) + + describe("getGoogleDatasourceConfig", () => { + function setEnvVars() { + env.GOOGLE_CLIENT_SECRET = "test" + env.GOOGLE_CLIENT_ID = "test" + } + + function unsetEnvVars() { + env.GOOGLE_CLIENT_SECRET = undefined + env.GOOGLE_CLIENT_ID = undefined + } + + describe("cloud", () => { + beforeEach(() => { + testEnv.cloudHosted() + }) + + it("returns from env vars", async () => { + await config.doInTenant(async () => { + setEnvVars() + const config = await configs.getGoogleDatasourceConfig() + unsetEnvVars() + + expect(config).toEqual({ + activated: true, + clientID: "test", + clientSecret: "test", + }) + }) + }) + + it("returns undefined when no env vars are configured", async () => { + await config.doInTenant(async () => { + const config = await configs.getGoogleDatasourceConfig() + expect(config).toBeUndefined() + }) + }) + }) + + describe("self host", () => { + beforeEach(() => { + testEnv.selfHosted() + }) + + it("returns from config", async () => { + await config.doInTenant(async () => { + const googleDoc = structures.sso.googleConfigDoc() + await configs.save(googleDoc) + const config = await configs.getGoogleDatasourceConfig() + expect(config).toEqual(googleDoc.config) + }) + }) + + it("falls back to env vars when config is disabled", async () => { + await config.doInTenant(async () => { + setEnvVars() + const config = await configs.getGoogleDatasourceConfig() + unsetEnvVars() + expect(config).toEqual({ + activated: true, + clientID: "test", + clientSecret: "test", + }) + }) + }) + }) + }) }) diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index e25c90575f..15cec7a6b9 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -4,7 +4,6 @@ export enum UserStatus { } export enum Cookie { - CurrentApp = "budibase:currentapp", Auth = "budibase:auth", Init = "budibase:init", ACCOUNT_RETURN_URL = "budibase:account:returnurl", diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index cba2f0138a..71ce4ba9ac 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -199,6 +199,10 @@ export class QueryBuilder { return this } + setAllOr() { + this.query.allOr = true + } + handleSpaces(input: string) { if (this.noEscaping) { return input @@ -236,6 +240,36 @@ export class QueryBuilder { return value } + isMultiCondition() { + let count = 0 + for (let filters of Object.values(this.query)) { + // not contains is one massive filter in allOr mode + if (typeof filters === "object") { + count += Object.keys(filters).length + } + } + return count > 1 + } + + compressFilters(filters: Record) { + const compressed: typeof filters = {} + for (let key of Object.keys(filters)) { + const finalKey = removeKeyNumbering(key) + if (compressed[finalKey]) { + compressed[finalKey] = compressed[finalKey].concat(filters[key]) + } else { + compressed[finalKey] = filters[key] + } + } + // add prefixes back + const final: typeof filters = {} + let count = 1 + for (let [key, value] of Object.entries(compressed)) { + final[`${count++}:${key}`] = value + } + return final + } + buildSearchQuery() { const builder = this let allOr = this.query && this.query.allOr @@ -272,9 +306,9 @@ export class QueryBuilder { } const notContains = (key: string, value: any) => { - // @ts-ignore - const allPrefix = allOr === "" ? "*:* AND" : "" - return allPrefix + "NOT " + contains(key, value) + const allPrefix = allOr ? "*:* AND " : "" + const mode = allOr ? "AND" : undefined + return allPrefix + "NOT " + contains(key, value, mode) } const containsAny = (key: string, value: any) => { @@ -299,21 +333,32 @@ export class QueryBuilder { return `${key}:(${orStatement})` } - function build(structure: any, queryFn: any) { + function build( + structure: any, + queryFn: (key: string, value: any) => string | null, + opts?: { returnBuilt?: boolean; mode?: string } + ) { + let built = "" for (let [key, value] of Object.entries(structure)) { // check for new format - remove numbering if needed key = removeKeyNumbering(key) key = builder.preprocess(builder.handleSpaces(key), { escape: true, }) - const expression = queryFn(key, value) + let expression = queryFn(key, value) if (expression == null) { continue } - if (query.length > 0) { - query += ` ${allOr ? "OR" : "AND"} ` + if (built.length > 0 || query.length > 0) { + const mode = opts?.mode ? opts.mode : allOr ? "OR" : "AND" + built += ` ${mode} ` } - query += expression + built += expression + } + if (opts?.returnBuilt) { + return built + } else { + query += built } } @@ -384,14 +429,14 @@ export class QueryBuilder { build(this.query.contains, contains) } if (this.query.notContains) { - build(this.query.notContains, notContains) + build(this.compressFilters(this.query.notContains), notContains) } if (this.query.containsAny) { build(this.query.containsAny, containsAny) } // make sure table ID is always added as an AND if (tableId) { - query = `(${query})` + query = this.isMultiCondition() ? `(${query})` : query allOr = false build({ tableId }, equal) } diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index 23b01e18df..52017cc94c 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -6,9 +6,13 @@ import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene" const INDEX_NAME = "main" const index = `function(doc) { - let props = ["property", "number"] + let props = ["property", "number", "array"] for (let key of props) { - if (doc[key]) { + if (Array.isArray(doc[key])) { + for (let val of doc[key]) { + index(key, val) + } + } else if (doc[key]) { index(key, doc[key]) } } @@ -21,9 +25,14 @@ describe("lucene", () => { dbName = `db-${newid()}` // create the DB for testing db = getDB(dbName) - await db.put({ _id: newid(), property: "word" }) - await db.put({ _id: newid(), property: "word2" }) - await db.put({ _id: newid(), property: "word3", number: 1 }) + await db.put({ _id: newid(), property: "word", array: ["1", "4"] }) + await db.put({ _id: newid(), property: "word2", array: ["3", "1"] }) + await db.put({ + _id: newid(), + property: "word3", + number: 1, + array: ["1", "2"], + }) }) it("should be able to create a lucene index", async () => { @@ -118,6 +127,15 @@ describe("lucene", () => { const resp = await builder.run() expect(resp.rows.length).toBe(2) }) + + it("should be able to perform an or not contains search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addNotContains("array", ["1"]) + builder.addNotContains("array", ["2"]) + builder.setAllOr() + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) }) describe("paginated search", () => { diff --git a/packages/backend-core/src/errors/base.ts b/packages/backend-core/src/errors/base.ts deleted file mode 100644 index 801dcf168d..0000000000 --- a/packages/backend-core/src/errors/base.ts +++ /dev/null @@ -1,10 +0,0 @@ -export class BudibaseError extends Error { - code: string - type: string - - constructor(message: string, code: string, type: string) { - super(message) - this.code = code - this.type = type - } -} diff --git a/packages/backend-core/src/errors/errors.ts b/packages/backend-core/src/errors/errors.ts index 83e2ab5072..54ca8456ab 100644 --- a/packages/backend-core/src/errors/errors.ts +++ b/packages/backend-core/src/errors/errors.ts @@ -1,37 +1,99 @@ -import * as licensing from "./licensing" +// BASE -// combine all error codes into single object +export abstract class BudibaseError extends Error { + code: string -export const codes = { - ...licensing.codes, + constructor(message: string, code: ErrorCode) { + super(message) + this.code = code + } + + protected getPublicError?(): any } -// combine all error types -export const types = [licensing.type] +// ERROR HANDLING -// combine all error contexts -const context = { - ...licensing.context, +export enum ErrorCode { + USAGE_LIMIT_EXCEEDED = "usage_limit_exceeded", + FEATURE_DISABLED = "feature_disabled", + INVALID_API_KEY = "invalid_api_key", + HTTP = "http", } -// derive a public error message using codes, types and any custom contexts +/** + * For the given error, build the public representation that is safe + * to be exposed over an api. + */ export const getPublicError = (err: any) => { let error - if (err.code || err.type) { + if (err.code) { // add generic error information error = { code: err.code, - type: err.type, } - if (err.code && context[err.code]) { + if (err.getPublicError) { error = { ...error, // get any additional context from this error - ...context[err.code](err), + ...err.getPublicError(), } } } return error } + +// HTTP + +export class HTTPError extends BudibaseError { + status: number + + constructor(message: string, httpStatus: number, code = ErrorCode.HTTP) { + super(message, code) + this.status = httpStatus + } +} + +// LICENSING + +export class UsageLimitError extends HTTPError { + limitName: string + + constructor(message: string, limitName: string) { + super(message, 400, ErrorCode.USAGE_LIMIT_EXCEEDED) + this.limitName = limitName + } + + getPublicError() { + return { + limitName: this.limitName, + } + } +} + +export class FeatureDisabledError extends HTTPError { + featureName: string + + constructor(message: string, featureName: string) { + super(message, 400, ErrorCode.FEATURE_DISABLED) + this.featureName = featureName + } + + getPublicError() { + return { + featureName: this.featureName, + } + } +} + +// AUTH + +export class InvalidAPIKeyError extends BudibaseError { + constructor() { + super( + "Invalid API key - may need re-generated, or user doesn't exist", + ErrorCode.INVALID_API_KEY + ) + } +} diff --git a/packages/backend-core/src/errors/generic.ts b/packages/backend-core/src/errors/generic.ts deleted file mode 100644 index 71b3352438..0000000000 --- a/packages/backend-core/src/errors/generic.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { BudibaseError } from "./base" - -export class GenericError extends BudibaseError { - constructor(message: string, code: string, type: string) { - super(message, code, type ? type : "generic") - } -} diff --git a/packages/backend-core/src/errors/http.ts b/packages/backend-core/src/errors/http.ts deleted file mode 100644 index 182e009f58..0000000000 --- a/packages/backend-core/src/errors/http.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { GenericError } from "./generic" - -export class HTTPError extends GenericError { - status: number - - constructor( - message: string, - httpStatus: number, - code = "http", - type = "generic" - ) { - super(message, code, type) - this.status = httpStatus - } -} diff --git a/packages/backend-core/src/errors/index.ts b/packages/backend-core/src/errors/index.ts index 814d836590..a079f46484 100644 --- a/packages/backend-core/src/errors/index.ts +++ b/packages/backend-core/src/errors/index.ts @@ -1,3 +1 @@ export * from "./errors" -export { UsageLimitError, FeatureDisabledError } from "./licensing" -export { HTTPError } from "./http" diff --git a/packages/backend-core/src/errors/licensing.ts b/packages/backend-core/src/errors/licensing.ts deleted file mode 100644 index 7ffcefa167..0000000000 --- a/packages/backend-core/src/errors/licensing.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { HTTPError } from "./http" - -export const type = "license_error" - -export const codes = { - USAGE_LIMIT_EXCEEDED: "usage_limit_exceeded", - FEATURE_DISABLED: "feature_disabled", -} - -export const context = { - [codes.USAGE_LIMIT_EXCEEDED]: (err: any) => { - return { - limitName: err.limitName, - } - }, - [codes.FEATURE_DISABLED]: (err: any) => { - return { - featureName: err.featureName, - } - }, -} - -export class UsageLimitError extends HTTPError { - limitName: string - - constructor(message: string, limitName: string) { - super(message, 400, codes.USAGE_LIMIT_EXCEEDED, type) - this.limitName = limitName - } -} - -export class FeatureDisabledError extends HTTPError { - featureName: string - - constructor(message: string, featureName: string) { - super(message, 400, codes.FEATURE_DISABLED, type) - this.featureName = featureName - } -} diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 724ecd21ba..30072196ba 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -24,6 +24,7 @@ export * as redis from "./redis" export * as locks from "./redis/redlockImpl" export * as utils from "./utils" export * as errors from "./errors" +export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" export { SearchParams } from "./db" diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 5e546b4c1c..8a97319586 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -14,6 +14,7 @@ import { decrypt } from "../security/encryption" import * as identity from "../context/identity" import env from "../environment" import { Ctx, EndpointMatcher } from "@budibase/types" +import { InvalidAPIKeyError, ErrorCode } from "../errors" const ONE_MINUTE = env.SESSION_UPDATE_PERIOD ? parseInt(env.SESSION_UPDATE_PERIOD) @@ -48,22 +49,27 @@ async function checkApiKey(apiKey: string, populateUser?: Function) { const decrypted = decrypt(apiKey) const tenantId = decrypted.split(SEPARATOR)[0] return doInTenant(tenantId, async () => { - const db = getGlobalDB() - // api key is encrypted in the database - const userId = (await queryGlobalView( - ViewName.BY_API_KEY, - { - key: apiKey, - }, - db - )) as string + let userId + try { + const db = getGlobalDB() + // api key is encrypted in the database + userId = (await queryGlobalView( + ViewName.BY_API_KEY, + { + key: apiKey, + }, + db + )) as string + } catch (err) { + userId = undefined + } if (userId) { return { valid: true, user: await getUser(userId, tenantId, populateUser), } } else { - throw "Invalid API key" + throw new InvalidAPIKeyError() } }) } @@ -164,8 +170,10 @@ export default function ( console.error(`Auth Error: ${err.message}`) console.error(err) // invalid token, clear the cookie - if (err && err.name === "JsonWebTokenError") { + if (err?.name === "JsonWebTokenError") { clearCookie(ctx, Cookie.Auth) + } else if (err?.code === ErrorCode.INVALID_API_KEY) { + ctx.throw(403, err.message) } // allow configuring for public access if ((opts && opts.publicAllowed) || publicEndpoint) { diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index 32451cb8d2..6fd4e9ff32 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -78,17 +78,23 @@ export async function postAuth( ), { successRedirect: "/", failureRedirect: "/error" }, async (err: any, tokens: string[]) => { + const baseUrl = `/builder/app/${authStateCookie.appId}/data` // update the DB for the datasource with all the user info await doWithDB(authStateCookie.appId, async (db: Database) => { - const datasource = await db.get(authStateCookie.datasourceId) + let datasource + try { + datasource = await db.get(authStateCookie.datasourceId) + } catch (err: any) { + if (err.status === 404) { + ctx.redirect(baseUrl) + } + } if (!datasource.config) { datasource.config = {} } datasource.config.auth = { type: "google", ...tokens } await db.put(datasource) - ctx.redirect( - `/builder/app/${authStateCookie.appId}/data/datasource/${authStateCookie.datasourceId}` - ) + ctx.redirect(`${baseUrl}/datasource/${authStateCookie.datasourceId}`) }) } )(ctx, next) diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index c57ebafb1f..0658147709 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -4,6 +4,7 @@ import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" import BullQueue from "bull" import { addListeners, StalledFn } from "./listeners" +import * as timers from "../timers" const CLEANUP_PERIOD_MS = 60 * 1000 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] @@ -29,8 +30,8 @@ export function createQueue( } addListeners(queue, jobQueue, opts?.removeStalledCb) QUEUES.push(queue) - if (!cleanupInterval) { - cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS) + if (!cleanupInterval && !env.isTest()) { + cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS) // fire off an initial cleanup cleanup().catch(err => { console.error(`Unable to cleanup automation queue initially - ${err}`) @@ -41,7 +42,7 @@ export function createQueue( export async function shutdown() { if (cleanupInterval) { - clearInterval(cleanupInterval) + timers.clear(cleanupInterval) } if (QUEUES.length) { for (let queue of QUEUES) { diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 951369496a..186865ccda 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -8,6 +8,7 @@ import { SEPARATOR, SelectableDatabase, } from "./utils" +import * as timers from "../timers" const RETRY_PERIOD_MS = 2000 const STARTUP_TIMEOUT_MS = 5000 @@ -117,9 +118,9 @@ function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) { return } // check if the connection is ready - const interval = setInterval(() => { + const interval = timers.set(() => { if (CONNECTED) { - clearInterval(interval) + timers.clear(interval) resolve("") } }, 500) diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 136d7f5d33..5e71488689 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -24,7 +24,7 @@ const getClient = async (type: LockType): Promise => { } } -export const OPTIONS = { +const OPTIONS = { TRY_ONCE: { // immediately throws an error if the lock is already held retryCount: 0, @@ -56,14 +56,29 @@ export const OPTIONS = { }, } -export const newRedlock = async (opts: Options = {}) => { +const newRedlock = async (opts: Options = {}) => { let options = { ...OPTIONS.DEFAULT, ...opts } const redisWrapper = await getLockClient() const client = redisWrapper.getClient() return new Redlock([client], options) } -export const doWithLock = async (opts: LockOptions, task: any) => { +type SuccessfulRedlockExecution = { + executed: true + result: T +} +type UnsuccessfulRedlockExecution = { + executed: false +} + +type RedlockExecution = + | SuccessfulRedlockExecution + | UnsuccessfulRedlockExecution + +export const doWithLock = async ( + opts: LockOptions, + task: () => Promise +): Promise> => { const redlock = await getClient(opts.type) let lock try { @@ -73,8 +88,8 @@ export const doWithLock = async (opts: LockOptions, task: any) => { let name: string = `lock:${prefix}_${opts.name}` // add additional unique name if required - if (opts.nameSuffix) { - name = name + `_${opts.nameSuffix}` + if (opts.resource) { + name = name + `_${opts.resource}` } // create the lock @@ -83,7 +98,7 @@ export const doWithLock = async (opts: LockOptions, task: any) => { // perform locked task // need to await to ensure completion before unlocking const result = await task() - return result + return { executed: true, result } } catch (e: any) { console.warn("lock error") // lock limit exceeded @@ -92,7 +107,7 @@ export const doWithLock = async (opts: LockOptions, task: any) => { // don't throw for try-once locks, they will always error // due to retry count (0) exceeded console.warn(e) - return + return { executed: false } } else { console.error(e) throw e diff --git a/packages/backend-core/src/timers/index.ts b/packages/backend-core/src/timers/index.ts new file mode 100644 index 0000000000..c9d642709f --- /dev/null +++ b/packages/backend-core/src/timers/index.ts @@ -0,0 +1 @@ +export * from "./timers" diff --git a/packages/backend-core/src/timers/timers.ts b/packages/backend-core/src/timers/timers.ts new file mode 100644 index 0000000000..000be74821 --- /dev/null +++ b/packages/backend-core/src/timers/timers.ts @@ -0,0 +1,22 @@ +let intervals: NodeJS.Timeout[] = [] + +export function set(callback: () => any, period: number) { + const interval = setInterval(callback, period) + intervals.push(interval) + return interval +} + +export function clear(interval: NodeJS.Timeout) { + const idx = intervals.indexOf(interval) + if (idx !== -1) { + intervals.splice(idx, 1) + } + clearInterval(interval) +} + +export function cleanup() { + for (let interval of intervals) { + clearInterval(interval) + } + intervals = [] +} diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 8963f7c141..dfc544c3ed 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -5,6 +5,8 @@ import { generateAppUserID, queryGlobalView, UNICODE_MAX, + DocumentType, + SEPARATOR, directCouchFind, } from "./db" import { BulkDocsResponse, User } from "@budibase/types" @@ -45,6 +47,16 @@ export const bulkGetGlobalUsersById = async ( return users } +export const getAllUserIds = async () => { + const db = getGlobalDB() + const startKey = `${DocumentType.USER}${SEPARATOR}` + const response = await db.allDocs({ + startkey: startKey, + endkey: `${startKey}${UNICODE_MAX}`, + }) + return response.rows.map(row => row.id) +} + export const bulkUpdateGlobalUsers = async (users: User[]) => { const db = getGlobalDB() return (await db.bulkDocs(users)) as BulkDocsResponse diff --git a/packages/backend-core/tests/jestSetup.ts b/packages/backend-core/tests/jestSetup.ts index e786086de6..be81fbff75 100644 --- a/packages/backend-core/tests/jestSetup.ts +++ b/packages/backend-core/tests/jestSetup.ts @@ -1,5 +1,6 @@ import "./logging" import env from "../src/environment" +import { cleanup } from "../src/timers" import { mocks, testContainerUtils } from "./utilities" // must explicitly enable fetch mock @@ -21,3 +22,7 @@ if (!process.env.CI) { } testContainerUtils.setupEnv(env) + +afterAll(() => { + cleanup() +}) diff --git a/packages/backend-core/tests/utilities/index.ts b/packages/backend-core/tests/utilities/index.ts index efe014908b..1c73216d76 100644 --- a/packages/backend-core/tests/utilities/index.ts +++ b/packages/backend-core/tests/utilities/index.ts @@ -4,4 +4,6 @@ export { generator } from "./structures" export * as testEnv from "./testEnv" export * as testContainerUtils from "./testContainerUtils" +export * from "./jestUtils" + export { default as DBTestConfiguration } from "./DBTestConfiguration" diff --git a/packages/backend-core/tests/utilities/jestUtils.ts b/packages/backend-core/tests/utilities/jestUtils.ts new file mode 100644 index 0000000000..d84eac548c --- /dev/null +++ b/packages/backend-core/tests/utilities/jestUtils.ts @@ -0,0 +1,9 @@ +export function expectFunctionWasCalledTimesWith( + jestFunction: any, + times: number, + argument: any +) { + expect( + jestFunction.mock.calls.filter((call: any) => call[0] === argument).length + ).toBe(times) +} diff --git a/packages/backend-core/tests/utilities/structures/db.ts b/packages/backend-core/tests/utilities/structures/db.ts index e25b707cb9..f4a677e777 100644 --- a/packages/backend-core/tests/utilities/structures/db.ts +++ b/packages/backend-core/tests/utilities/structures/db.ts @@ -1,5 +1,12 @@ +import { structures } from ".." import { newid } from "../../../src/newid" export function id() { return `db_${newid()}` } + +export function rev() { + return `${structures.generator.character({ + numeric: true, + })}-${structures.uuid().replace(/-/, "")}` +} diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index ca77f476d0..ff2e5b147f 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -8,4 +8,5 @@ export * as plugins from "./plugins" export * as sso from "./sso" export * as tenant from "./tenants" export * as users from "./users" +export * as userGroups from "./userGroups" export { generator } from "./generator" diff --git a/packages/backend-core/tests/utilities/structures/sso.ts b/packages/backend-core/tests/utilities/structures/sso.ts index 7413fa3c09..9da9c82223 100644 --- a/packages/backend-core/tests/utilities/structures/sso.ts +++ b/packages/backend-core/tests/utilities/structures/sso.ts @@ -1,4 +1,6 @@ import { + ConfigType, + GoogleConfig, GoogleInnerConfig, JwtClaims, OAuth2, @@ -10,10 +12,10 @@ import { User, } from "@budibase/types" import { generator } from "./generator" -import { uuid, email } from "./common" +import { email, uuid } from "./common" import * as shared from "./shared" -import _ from "lodash" import { user } from "./shared" +import _ from "lodash" export function OAuth(): OAuth2 { return { @@ -107,3 +109,11 @@ export function googleConfig(): GoogleInnerConfig { clientSecret: generator.string(), } } + +export function googleConfigDoc(): GoogleConfig { + return { + _id: "config_google", + type: ConfigType.GOOGLE, + config: googleConfig(), + } +} diff --git a/packages/backend-core/tests/utilities/structures/userGroups.ts b/packages/backend-core/tests/utilities/structures/userGroups.ts new file mode 100644 index 0000000000..4dc870a00a --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/userGroups.ts @@ -0,0 +1,10 @@ +import { UserGroup } from "@budibase/types" +import { generator } from "./generator" + +export function userGroup(): UserGroup { + return { + name: generator.word(), + icon: generator.word(), + color: generator.word(), + } +} diff --git a/packages/bbui/package.json b/packages/bbui/package.json index c4cf512fce..7d0af9a709 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.4.43", + "version": "2.4.44-alpha.1", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "^2.4.43", - "@budibase/string-templates": "^2.4.43", + "@budibase/shared-core": "2.4.44-alpha.1", + "@budibase/string-templates": "2.4.44-alpha.1", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index 01f5033e6c..60c8bec80b 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -113,6 +113,9 @@ .spectrum-ActionButton--quiet { padding: 0 8px; } + .spectrum-ActionButton--quiet.is-selected { + color: var(--spectrum-global-color-gray-900); + } .is-selected:not(.emphasized) .spectrum-Icon { color: var(--spectrum-global-color-gray-900); } diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js index abc7188985..ecbb5747c4 100644 --- a/packages/bbui/src/Actions/position_dropdown.js +++ b/packages/bbui/src/Actions/position_dropdown.js @@ -31,6 +31,7 @@ export default function positionDropdown(element, opts) { styles.top = anchorBounds.top } else if (window.innerHeight - anchorBounds.bottom < 100) { styles.top = anchorBounds.top - elementBounds.height - offset + styles.maxHeight = 240 } else { styles.top = anchorBounds.bottom + offset styles.maxHeight = window.innerHeight - anchorBounds.bottom - 20 diff --git a/packages/bbui/src/Drawer/Drawer.svelte b/packages/bbui/src/Drawer/Drawer.svelte index 43729cd794..932236bc0c 100644 --- a/packages/bbui/src/Drawer/Drawer.svelte +++ b/packages/bbui/src/Drawer/Drawer.svelte @@ -7,7 +7,7 @@ export let title export let fillWidth export let left = "314px" - export let width = "calc(100% - 576px)" + export let width = "calc(100% - 626px)" let visible = false diff --git a/packages/bbui/src/Form/Core/File.svelte b/packages/bbui/src/Form/Core/File.svelte new file mode 100644 index 0000000000..618cccd941 --- /dev/null +++ b/packages/bbui/src/Form/Core/File.svelte @@ -0,0 +1,115 @@ + + + + +
+ {#if value} +
+ {#if previewUrl} + + {/if} +
{value.name}
+ {#if value.size} +
+ {#if value.size <= BYTES_IN_MB} + {`${value.size / BYTES_IN_KB} KB`} + {:else} + {`${value.size / BYTES_IN_MB} MB`} + {/if} +
+ {/if} + {#if !disabled || (allowClear === true && disabled)} +
+ +
+ {/if} +
+ {/if} + {title} +
+ + diff --git a/packages/bbui/src/Form/Core/index.js b/packages/bbui/src/Form/Core/index.js index 7c81cfd70b..b0edf52748 100644 --- a/packages/bbui/src/Form/Core/index.js +++ b/packages/bbui/src/Form/Core/index.js @@ -13,3 +13,4 @@ export { default as CoreDropzone } from "./Dropzone.svelte" export { default as CoreStepper } from "./Stepper.svelte" export { default as CoreRichTextField } from "./RichTextField.svelte" export { default as CoreSlider } from "./Slider.svelte" +export { default as CoreFile } from "./File.svelte" diff --git a/packages/bbui/src/Form/File.svelte b/packages/bbui/src/Form/File.svelte new file mode 100644 index 0000000000..03cacea814 --- /dev/null +++ b/packages/bbui/src/Form/File.svelte @@ -0,0 +1,37 @@ + + + + + diff --git a/packages/builder/src/components/common/inputs/CopyInput.svelte b/packages/bbui/src/Input/CopyInput.svelte similarity index 91% rename from packages/builder/src/components/common/inputs/CopyInput.svelte rename to packages/bbui/src/Input/CopyInput.svelte index fe7746d1f9..b4d6e5107f 100644 --- a/packages/builder/src/components/common/inputs/CopyInput.svelte +++ b/packages/bbui/src/Input/CopyInput.svelte @@ -1,5 +1,7 @@ + \ No newline at end of file diff --git a/packages/builder/package.json b/packages/builder/package.json index 51bce9ab4a..8d9c52e7fd 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.4.43", + "version": "2.4.44-alpha.1", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,11 +58,11 @@ } }, "dependencies": { - "@budibase/bbui": "^2.4.43", - "@budibase/client": "^2.4.43", - "@budibase/frontend-core": "^2.4.43", - "@budibase/shared-core": "^2.4.43", - "@budibase/string-templates": "^2.4.43", + "@budibase/bbui": "2.4.44-alpha.1", + "@budibase/client": "2.4.44-alpha.1", + "@budibase/frontend-core": "2.4.44-alpha.1", + "@budibase/shared-core": "2.4.44-alpha.1", + "@budibase/string-templates": "2.4.44-alpha.1", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 51f88add27..3fc0eb769e 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -22,6 +22,7 @@ import { findComponent, getComponentSettings, makeComponentUnique, + findComponentPath, } from "../componentUtils" import { Helpers } from "@budibase/bbui" import { Utils } from "@budibase/frontend-core" @@ -30,7 +31,12 @@ import { DB_TYPE_INTERNAL, DB_TYPE_EXTERNAL, } from "constants/backend" -import { getSchemaForDatasource } from "builderStore/dataBinding" +import { + buildFormSchema, + getSchemaForDatasource, +} from "builderStore/dataBinding" +import { makePropSafe as safe } from "@budibase/string-templates" +import { getComponentFieldOptions } from "helpers/formFields" const INITIAL_FRONTEND_STATE = { apps: [], @@ -63,17 +69,19 @@ const INITIAL_FRONTEND_STATE = { customTheme: {}, previewDevice: "desktop", highlightedSettingKey: null, + builderSidePanel: false, // URL params selectedScreenId: null, selectedComponentId: null, selectedLayoutId: null, - // onboarding + // Client state + selectedComponentInstance: null, + + // Onboarding onboarding: false, tourNodes: null, - - builderSidePanel: false, } export const getFrontendStore = () => { @@ -262,22 +270,27 @@ export const getFrontendStore = () => { } }, save: async screen => { - /* - Temporarily disabled to accomodate migration issues. - store.actions.screens.validate(screen) - */ - const state = get(store) + // Validate screen structure + // Temporarily disabled to accommodate migration issues + // store.actions.screens.validate(screen) + + // Check screen definition for any component settings which need updated + store.actions.screens.enrichEmptySettings(screen) + + // Save screen const creatingNewScreen = screen._id === undefined const savedScreen = await API.saveScreen(screen) const routesResponse = await API.fetchAppRoutes() - let usedPlugins = state.usedPlugins // If plugins changed we need to fetch the latest app metadata + const state = get(store) + let usedPlugins = state.usedPlugins if (savedScreen.pluginAdded) { const { application } = await API.fetchAppPackage(state.appId) usedPlugins = application.usedPlugins || [] } + // Update state store.update(state => { // Update screen object const idx = state.screens.findIndex(x => x._id === savedScreen._id) @@ -298,7 +311,6 @@ export const getFrontendStore = () => { // Update used plugins state.usedPlugins = usedPlugins - return state }) return savedScreen @@ -406,6 +418,17 @@ export const getFrontendStore = () => { } await store.actions.screens.patch(patch, screen._id) }, + enrichEmptySettings: screen => { + // Flatten the recursive component tree + const components = findAllMatchingComponents(screen.props, x => x) + + // Iterate over all components and run checks + components.forEach(component => { + store.actions.components.enrichEmptySettings(component, { + screen, + }) + }) + }, }, preview: { setDevice: device => { @@ -493,65 +516,155 @@ export const getFrontendStore = () => { } return get(store).components[componentName] }, - createInstance: (componentName, presetProps) => { + getDefaultDatasource: () => { + // Ignore users table + const validTables = get(tables).list.filter(x => x._id !== "ta_users") + + // Try to use their own internal table first + let table = validTables.find(table => { + return ( + table.sourceId !== BUDIBASE_INTERNAL_DB_ID && + table.type === DB_TYPE_INTERNAL + ) + }) + if (table) { + return table + } + + // Then try sample data + table = validTables.find(table => { + return ( + table.sourceId === BUDIBASE_INTERNAL_DB_ID && + table.type === DB_TYPE_INTERNAL + ) + }) + if (table) { + return table + } + + // Finally try an external table + return validTables.find(table => table.type === DB_TYPE_EXTERNAL) + }, + enrichEmptySettings: (component, opts) => { + if (!component?._component) { + return + } + const defaultDS = store.actions.components.getDefaultDatasource() + const settings = getComponentSettings(component._component) + const { parent, screen, useDefaultValues } = opts || {} + const treeId = parent?._id || component._id + if (!screen) { + return + } + settings.forEach(setting => { + const value = component[setting.key] + + // Fill empty settings + if (value == null || value === "") { + if (setting.type === "multifield" && setting.selectAllFields) { + // Select all schema fields where required + component[setting.key] = Object.keys(defaultDS?.schema || {}) + } else if ( + (setting.type === "dataSource" || setting.type === "table") && + defaultDS + ) { + // Select default datasource where required + component[setting.key] = { + label: defaultDS.name, + tableId: defaultDS._id, + type: "table", + } + } else if (setting.type === "dataProvider") { + // Pick closest data provider where required + const path = findComponentPath(screen.props, treeId) + const providers = path.filter(component => + component._component?.endsWith("/dataprovider") + ) + if (providers.length) { + const id = providers[providers.length - 1]?._id + component[setting.key] = `{{ literal ${safe(id)} }}` + } + } else if (setting.type.startsWith("field/")) { + // Autofill form field names + // Get all available field names in this form schema + let fieldOptions = getComponentFieldOptions( + screen.props, + treeId, + setting.type, + false + ) + + // Get all currently used fields + const form = findClosestMatchingComponent( + screen.props, + treeId, + x => x._component === "@budibase/standard-components/form" + ) + const usedFields = Object.keys(buildFormSchema(form) || {}) + + // Filter out already used fields + fieldOptions = fieldOptions.filter(x => !usedFields.includes(x)) + + // Set field name and also assume we have a label setting + if (fieldOptions[0]) { + component[setting.key] = fieldOptions[0] + component.label = fieldOptions[0] + } + } else if (useDefaultValues && setting.defaultValue !== undefined) { + // Use default value where required + component[setting.key] = setting.defaultValue + } + } + + // Validate non-empty settings + else { + if (setting.type === "dataProvider") { + // Validate data provider exists, or else clear it + const treeId = parent?._id || component._id + const path = findComponentPath(screen?.props, treeId) + const providers = path.filter(component => + component._component?.endsWith("/dataprovider") + ) + // Validate non-empty values + const valid = providers?.some(dp => value.includes?.(dp._id)) + if (!valid) { + if (providers.length) { + const id = providers[providers.length - 1]?._id + component[setting.key] = `{{ literal ${safe(id)} }}` + } else { + delete component[setting.key] + } + } + } + } + }) + }, + createInstance: (componentName, presetProps, parent) => { const definition = store.actions.components.getDefinition(componentName) if (!definition) { return null } - // Flattened settings - const settings = getComponentSettings(componentName) - - let dataSourceField = settings.find( - setting => setting.type == "dataSource" || setting.type == "table" - ) - - let defaultDatasource - if (dataSourceField) { - const _tables = get(tables) - const filteredTables = _tables.list.filter( - table => table._id != "ta_users" - ) - - const internalTable = filteredTables.find( - table => - table.sourceId === BUDIBASE_INTERNAL_DB_ID && - table.type == DB_TYPE_INTERNAL - ) - - const defaultSourceTable = filteredTables.find( - table => - table.sourceId !== BUDIBASE_INTERNAL_DB_ID && - table.type == DB_TYPE_INTERNAL - ) - - const defaultExternalTable = filteredTables.find( - table => table.type == DB_TYPE_EXTERNAL - ) - - defaultDatasource = - defaultSourceTable || internalTable || defaultExternalTable + // Generate basic component structure + let instance = { + _id: Helpers.uuid(), + _component: definition.component, + _styles: { + normal: {}, + hover: {}, + active: {}, + }, + _instanceName: `New ${definition.friendlyName || definition.name}`, + ...presetProps, } - // Generate default props - let props = { ...presetProps } - settings.forEach(setting => { - if (setting.type === "multifield" && setting.selectAllFields) { - props[setting.key] = Object.keys(defaultDatasource.schema || {}) - } else if (setting.defaultValue !== undefined) { - props[setting.key] = setting.defaultValue - } + // Enrich empty settings + store.actions.components.enrichEmptySettings(instance, { + parent, + screen: get(selectedScreen), + useDefaultValues: true, }) - // Set a default datasource - if (dataSourceField && defaultDatasource) { - props[dataSourceField.key] = { - label: defaultDatasource.name, - tableId: defaultDatasource._id, - type: "table", - } - } - // Add any extra properties the component needs let extras = {} if (definition.hasChildren) { @@ -569,17 +682,8 @@ export const getFrontendStore = () => { extras.step = formSteps.length + 1 extras._instanceName = `Step ${formSteps.length + 1}` } - return { - _id: Helpers.uuid(), - _component: definition.component, - _styles: { - normal: {}, - hover: {}, - active: {}, - }, - _instanceName: `New ${definition.friendlyName || definition.name}`, - ...cloneDeep(props), + ...cloneDeep(instance), ...extras, } }, @@ -587,7 +691,8 @@ export const getFrontendStore = () => { const state = get(store) const componentInstance = store.actions.components.createInstance( componentName, - presetProps + presetProps, + parent ) if (!componentInstance) { return @@ -1123,6 +1228,52 @@ export const getFrontendStore = () => { }) } }, + addParent: async (componentId, parentType) => { + if (!componentId || !parentType) { + return + } + + // Create new parent instance + const newParentDefinition = store.actions.components.createInstance( + parentType, + null, + parent + ) + if (!newParentDefinition) { + return + } + + // Replace component with a version wrapped in a new parent + await store.actions.screens.patch(screen => { + // Get this component definition and parent definition + let definition = findComponent(screen.props, componentId) + let oldParentDefinition = findComponentParent( + screen.props, + componentId + ) + if (!definition || !oldParentDefinition) { + return false + } + + // Replace component with parent + const index = oldParentDefinition._children.findIndex( + component => component._id === componentId + ) + if (index === -1) { + return false + } + oldParentDefinition._children[index] = { + ...newParentDefinition, + _children: [definition], + } + }) + + // Select the new parent + store.update(state => { + state.selectedComponentId = newParentDefinition._id + return state + }) + }, }, links: { save: async (url, title) => { diff --git a/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte b/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte index 9ba4140b51..e42334e042 100644 --- a/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte +++ b/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte @@ -1,5 +1,5 @@ diff --git a/packages/builder/src/components/deploy/DeployModal.svelte b/packages/builder/src/components/deploy/DeployModal.svelte index 2b881e524e..05b620da71 100644 --- a/packages/builder/src/components/deploy/DeployModal.svelte +++ b/packages/builder/src/components/deploy/DeployModal.svelte @@ -5,12 +5,12 @@ notifications, ModalContent, Layout, + ProgressCircle, + CopyInput, } from "@budibase/bbui" import { API } from "api" import analytics, { Events, EventSource } from "analytics" import { store } from "builderStore" - import { ProgressCircle } from "@budibase/bbui" - import CopyInput from "components/common/inputs/CopyInput.svelte" import TourWrap from "../portal/onboarding/TourWrap.svelte" import { TOUR_STEP_KEYS } from "../portal/onboarding/tours.js" diff --git a/packages/builder/src/components/deploy/VersionModal.svelte b/packages/builder/src/components/deploy/VersionModal.svelte index 23d9fd83a0..f357cc7820 100644 --- a/packages/builder/src/components/deploy/VersionModal.svelte +++ b/packages/builder/src/components/deploy/VersionModal.svelte @@ -24,7 +24,10 @@ let updateModal $: appId = $store.appId - $: updateAvailable = clientPackage.version !== $store.version + $: updateAvailable = + clientPackage.version && + $store.version && + clientPackage.version !== $store.version $: revertAvailable = $store.revertableVersion != null const refreshAppPackage = async () => { diff --git a/packages/builder/src/components/design/Panel.svelte b/packages/builder/src/components/design/Panel.svelte index a1e3bd7eb7..dbf42c51a5 100644 --- a/packages/builder/src/components/design/Panel.svelte +++ b/packages/builder/src/components/design/Panel.svelte @@ -3,7 +3,6 @@ export let title export let icon - export let expandable = false export let showAddButton = false export let showBackButton = false export let showCloseButton = false @@ -12,12 +11,13 @@ export let onClickCloseButton export let borderLeft = false export let borderRight = false + export let wide = false - let wide = false + $: customHeaderContent = $$slots["panel-header-content"]
-
+
{#if showBackButton} {/if} @@ -27,13 +27,6 @@
{title || ""}
- {#if expandable} - (wide = !wide)} - /> - {/if} {#if showAddButton}
@@ -43,6 +36,13 @@ {/if}
+ + {#if customHeaderContent} + + + + {/if} +
@@ -66,8 +66,8 @@ border-right: var(--border-light); } .panel.wide { - width: 420px; - flex: 0 0 420px; + width: 310px; + flex: 0 0 310px; } .header { flex: 0 0 48px; @@ -116,4 +116,10 @@ justify-content: flex-start; align-items: stretch; } + .header.custom { + border: none; + } + .custom-content-wrap { + border-bottom: var(--border-light); + } diff --git a/packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte b/packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte index 098a8f7ed7..59340d4898 100644 --- a/packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte @@ -27,7 +27,7 @@ : enrichedSchemaFields?.map(field => field.name) $: sanitisedValue = getValidColumns(value, options) $: updateBoundValue(sanitisedValue) - $: enrichedSchemaFields = getFields(Object.values(schema) || [], { + $: enrichedSchemaFields = getFields(Object.values(schema || {}), { allowLinks: true, }) diff --git a/packages/builder/src/components/design/settings/controls/DataProviderSelect.svelte b/packages/builder/src/components/design/settings/controls/DataProviderSelect.svelte index a5b7a08255..83255ec325 100644 --- a/packages/builder/src/components/design/settings/controls/DataProviderSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataProviderSelect.svelte @@ -3,23 +3,13 @@ import { makePropSafe } from "@budibase/string-templates" import { currentAsset, store } from "builderStore" import { findComponentPath } from "builderStore/componentUtils" - import { createEventDispatcher, onMount } from "svelte" export let value - const dispatch = createEventDispatcher() const getValue = component => `{{ literal ${makePropSafe(component._id)} }}` $: path = findComponentPath($currentAsset?.props, $store.selectedComponentId) $: providers = path.filter(c => c._component?.endsWith("/dataprovider")) - - // Set initial value to closest data provider - onMount(() => { - const valid = value && providers.find(x => getValue(x) === value) != null - if (!valid && providers.length) { - dispatch("change", getValue(providers[providers.length - 1])) - } - }) { + let clone = { ...config } + clone.platformTitle = e.detail ? e.detail : "" + config = clone + }} + value={config.platformTitle || ""} + disabled={!brandingEnabled || saving} + /> +
+ {/if} +
+ { + let clone = { ...config } + clone.emailBrandingEnabled = !e.detail + config = clone + }} + value={!config.emailBrandingEnabled} + disabled={!brandingEnabled || saving} + /> +
+
+ + {#if !isCloud} + + + Login page + + + + {/if} + + + Application previews + Customise the meta tags on your app preview + +
+
+
+ + { + let clone = { ...config } + clone.metaImageUrl = e.detail ? e.detail : "" + config = clone + }} + value={config.metaImageUrl} + disabled={!brandingEnabled || saving} + /> +
+
+ + { + let clone = { ...config } + clone.metaTitle = e.detail ? e.detail : "" + config = clone + }} + value={config.metaTitle} + disabled={!brandingEnabled || saving} + /> +
+
+ +