diff --git a/.eslintrc.json b/.eslintrc.json index 3de9d13046..ae9512152f 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -44,7 +44,8 @@ "no-undef": "off", "no-prototype-builtins": "off", "local-rules/no-budibase-imports": "error", - "local-rules/no-test-com": "error" + "local-rules/no-test-com": "error", + "local-rules/email-domain-example-com": "error" } }, { diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js index 71bb5068da..177b0a129c 100644 --- a/eslint-local-rules/index.js +++ b/eslint-local-rules/index.js @@ -51,4 +51,41 @@ module.exports = { } }, }, + "email-domain-example-com": { + meta: { + type: "problem", + docs: { + description: + "enforce using the example.com domain for generator.email calls", + category: "Possible Errors", + recommended: false, + }, + fixable: "code", + schema: [], + }, + create: function (context) { + return { + CallExpression(node) { + if ( + node.callee.type === "MemberExpression" && + node.callee.object.name === "generator" && + node.callee.property.name === "email" && + node.arguments.length === 0 + ) { + context.report({ + node, + message: + "Prefer using generator.email with the domain \"{ domain: 'example.com' }\".", + fix: function (fixer) { + return fixer.replaceText( + node, + 'generator.email({ domain: "example.com" })' + ) + }, + }) + } + }, + } + }, + }, } diff --git a/lerna.json b/lerna.json index 6fb032ac77..a77a16a24e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.2", + "version": "2.21.3", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 90b3316c3f..fe56780982 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -78,6 +78,7 @@ "jest-serial-runner": "1.2.1", "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", + "testcontainers": "^10.7.2", "timekeeper": "2.2.0", "typescript": "5.2.2" }, diff --git a/packages/backend-core/src/cache/base/index.ts b/packages/backend-core/src/cache/base/index.ts index 942d70ae72..74da4fe0d2 100644 --- a/packages/backend-core/src/cache/base/index.ts +++ b/packages/backend-core/src/cache/base/index.ts @@ -23,6 +23,18 @@ export default class BaseCache { return client.keys(pattern) } + async exists(key: string, opts = { useTenancy: true }) { + key = opts.useTenancy ? generateTenantKey(key) : key + const client = await this.getClient() + return client.exists(key) + } + + async scan(key: string, opts = { useTenancy: true }) { + key = opts.useTenancy ? generateTenantKey(key) : key + const client = await this.getClient() + return client.scan(key) + } + /** * Read only from the cache. */ @@ -32,6 +44,15 @@ export default class BaseCache { return client.get(key) } + /** + * Read only from the cache. + */ + async bulkGet(keys: string[], opts = { useTenancy: true }) { + keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys + const client = await this.getClient() + return client.bulkGet(keys) + } + /** * Write to the cache. */ @@ -74,6 +95,15 @@ export default class BaseCache { return client.delete(key) } + /** + * Remove from cache. + */ + async bulkDelete(keys: string[], opts = { useTenancy: true }) { + keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys + const client = await this.getClient() + return client.bulkDelete(keys) + } + /** * Read from the cache. Write to the cache if not exists. */ @@ -108,4 +138,13 @@ export default class BaseCache { throw err } } + + /** + * Delete the entry if the provided value matches the stored one. + */ + async deleteIfValue(key: string, value: any, opts = { useTenancy: true }) { + key = opts.useTenancy ? generateTenantKey(key) : key + const client = await this.getClient() + await client.deleteIfValue(key, value) + } } diff --git a/packages/backend-core/src/cache/docWritethrough.ts b/packages/backend-core/src/cache/docWritethrough.ts index a0bc14ec5c..c8b28b3877 100644 --- a/packages/backend-core/src/cache/docWritethrough.ts +++ b/packages/backend-core/src/cache/docWritethrough.ts @@ -1,78 +1,77 @@ -import BaseCache from "./base" -import { getDocWritethroughClient } from "../redis/init" -import { AnyDocument, Database } from "@budibase/types" +import { AnyDocument, Database, LockName, LockType } from "@budibase/types" import { JobQueue, createQueue } from "../queue" import * as dbUtils from "../db" - -let CACHE: BaseCache | null = null -async function getCache() { - if (!CACHE) { - const client = await getDocWritethroughClient() - CACHE = new BaseCache(client) - } - return CACHE -} +import { string } from "yargs" +import { db } from ".." +import { locks } from "../redis" +import { Duration } from "../utils" interface ProcessDocMessage { dbName: string docId: string - cacheKeyPrefix: string + + data: Record } export const docWritethroughProcessorQueue = createQueue( JobQueue.DOC_WRITETHROUGH_QUEUE ) -docWritethroughProcessorQueue.process(async message => { - await persistToDb(message.data) - console.log("DocWritethrough persisted", { data: message.data }) -}) - -export async function persistToDb({ - dbName, - docId, - cacheKeyPrefix, -}: { - dbName: string - docId: string - cacheKeyPrefix: string -}) { - const cache = await getCache() - - const db = dbUtils.getDB(dbName) - let doc: AnyDocument | undefined - try { - doc = await db.get(docId) - } catch { - doc = { _id: docId } +class DocWritethroughProcessor { + init() { + docWritethroughProcessorQueue.process(async message => { + const result = await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.PERSIST_DOC_WRITETHROUGH, + resource: `${message.data.dbName}:${message.data.docId}`, + ttl: Duration.fromSeconds(60).toMs(), + }, + async () => { + await this.persistToDb(message.data) + } + ) + if (!result.executed) { + throw new Error( + `Error persisting docWritethrough message: ${message.id}` + ) + } + }) + return this } - const keysToPersist = await cache.keys(`${cacheKeyPrefix}:data:*`) - for (const key of keysToPersist) { - const data = await cache.get(key, { useTenancy: false }) - doc[data.key] = data.value - } + private async persistToDb({ + dbName, + docId, + data, + }: { + dbName: string + docId: string + data: Record + }) { + const db = dbUtils.getDB(dbName) + let doc: AnyDocument | undefined + try { + doc = await db.get(docId) + } catch { + doc = { _id: docId } + } - await db.put(doc) - - for (const key of keysToPersist) { - await cache.delete(key, { useTenancy: false }) + doc = { ...doc, ...data } + await db.put(doc) } } +export const processor = new DocWritethroughProcessor().init() + export class DocWritethrough { private db: Database private _docId: string - private writeRateMs: number - private cacheKeyPrefix: string - - constructor(db: Database, docId: string, writeRateMs: number) { + constructor(db: Database, docId: string) { this.db = db this._docId = docId - this.writeRateMs = writeRateMs - this.cacheKeyPrefix = `${this.db.name}:${this.docId}` } get docId() { @@ -80,30 +79,10 @@ export class DocWritethrough { } async patch(data: Record) { - const cache = await getCache() - - await this.storeToCache(cache, data) - - docWritethroughProcessorQueue.add( - { - dbName: this.db.name, - docId: this.docId, - cacheKeyPrefix: this.cacheKeyPrefix, - }, - { - delay: this.writeRateMs, - jobId: this.cacheKeyPrefix, - removeOnFail: true, - removeOnComplete: true, - } - ) - } - - private async storeToCache(cache: BaseCache, data: Record) { - data = Object.entries(data).reduce((acc, [key, value]) => { - acc[this.cacheKeyPrefix + ":data:" + key] = { key, value } - return acc - }, {} as Record) - await cache.bulkStore(data, null) + await docWritethroughProcessorQueue.add({ + dbName: this.db.name, + docId: this.docId, + data, + }) } } diff --git a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts index 9bbcd6af44..d90c83afd3 100644 --- a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts @@ -1,31 +1,21 @@ +import tk from "timekeeper" + +import _ from "lodash" import { DBTestConfiguration, generator, structures } from "../../../tests" import { getDB } from "../../db" -import _ from "lodash" import { DocWritethrough, docWritethroughProcessorQueue, } from "../docWritethrough" -import InMemoryQueue from "../../queue/inMemoryQueue" -const WRITE_RATE_MS = 1000 +import InMemoryQueue from "../../queue/inMemoryQueue" const initialTime = Date.now() -jest.useFakeTimers({ - now: initialTime, -}) - -function resetTime() { - jest.setSystemTime(initialTime) -} -async function travelForward(ms: number) { - await jest.advanceTimersByTimeAsync(ms) - +async function waitForQueueCompletion() { const queue: InMemoryQueue = docWritethroughProcessorQueue as never - while (queue.hasRunningJobs()) { - await jest.runOnlyPendingTimersAsync() - } + await queue.waitForCompletion() } describe("docWritethrough", () => { @@ -45,30 +35,28 @@ describe("docWritethrough", () => { } beforeEach(async () => { - resetTime() + jest.clearAllMocks() documentId = structures.uuid() - docWritethrough = new DocWritethrough(db, documentId, WRITE_RATE_MS) + docWritethrough = new DocWritethrough(db, documentId) }) - it("patching will not persist if timeout does not hit", async () => { + it("patching will not persist until the messages are persisted", async () => { await config.doInTenant(async () => { - await travelForward(WRITE_RATE_MS) await docWritethrough.patch(generatePatchObject(2)) await docWritethrough.patch(generatePatchObject(2)) - await travelForward(WRITE_RATE_MS - 1) expect(await db.exists(documentId)).toBe(false) }) }) - it("patching will persist if timeout hits", async () => { + it("patching will persist when the messages are persisted", async () => { await config.doInTenant(async () => { const patch1 = generatePatchObject(2) const patch2 = generatePatchObject(2) await docWritethrough.patch(patch1) await docWritethrough.patch(patch2) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() // This will not be persisted const patch3 = generatePatchObject(3) @@ -78,9 +66,9 @@ describe("docWritethrough", () => { _id: documentId, ...patch1, ...patch2, - _rev: expect.stringMatching(/1-.+/), - createdAt: new Date(initialTime + WRITE_RATE_MS).toISOString(), - updatedAt: new Date(initialTime + WRITE_RATE_MS).toISOString(), + _rev: expect.stringMatching(/2-.+/), + createdAt: new Date(initialTime).toISOString(), + updatedAt: new Date(initialTime).toISOString(), }) }) }) @@ -92,12 +80,12 @@ describe("docWritethrough", () => { await docWritethrough.patch(patch1) await docWritethrough.patch(patch2) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() const patch3 = generatePatchObject(3) await docWritethrough.patch(patch3) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() expect(await db.get(documentId)).toEqual( expect.objectContaining({ @@ -115,12 +103,13 @@ describe("docWritethrough", () => { const patch1 = generatePatchObject(2) const patch2 = generatePatchObject(2) await docWritethrough.patch(patch1) - await travelForward(WRITE_RATE_MS) const date1 = new Date() + await waitForQueueCompletion() await docWritethrough.patch(patch2) - await travelForward(WRITE_RATE_MS) + tk.travel(Date.now() + 100) const date2 = new Date() + await waitForQueueCompletion() expect(date1).not.toEqual(date2) expect(await db.get(documentId)).toEqual( @@ -136,7 +125,7 @@ describe("docWritethrough", () => { await config.doInTenant(async () => { const patch1 = generatePatchObject(2) await docWritethrough.patch(patch1) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() const patch2 = generatePatchObject(1) await docWritethrough.patch(patch2) @@ -147,14 +136,14 @@ describe("docWritethrough", () => { }) ) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() const patch3 = { ...generatePatchObject(3), [keyToOverride]: generator.word(), } await docWritethrough.patch(patch3) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() expect(await db.get(documentId)).toEqual( expect.objectContaining({ @@ -170,8 +159,7 @@ describe("docWritethrough", () => { await config.doInTenant(async () => { const secondDocWritethrough = new DocWritethrough( db, - structures.db.id(), - WRITE_RATE_MS + structures.db.id() ) const doc1Patch = generatePatchObject(2) @@ -179,13 +167,13 @@ describe("docWritethrough", () => { const doc2Patch = generatePatchObject(1) await secondDocWritethrough.patch(doc2Patch) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() const doc1Patch2 = generatePatchObject(3) await docWritethrough.patch(doc1Patch2) const doc2Patch2 = generatePatchObject(3) await secondDocWritethrough.patch(doc2Patch2) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() expect(await db.get(docWritethrough.docId)).toEqual( expect.objectContaining({ @@ -208,9 +196,7 @@ describe("docWritethrough", () => { const initialPatch = generatePatchObject(5) await docWritethrough.patch(initialPatch) - await travelForward(WRITE_RATE_MS) - - await docWritethrough.patch({}) + await waitForQueueCompletion() expect(await db.get(documentId)).toEqual( expect.objectContaining(initialPatch) @@ -218,10 +204,10 @@ describe("docWritethrough", () => { await db.remove(await db.get(documentId)) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() const extraPatch = generatePatchObject(5) await docWritethrough.patch(extraPatch) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() expect(await db.get(documentId)).toEqual( expect.objectContaining(extraPatch) @@ -232,58 +218,70 @@ describe("docWritethrough", () => { }) }) - it("concurrent calls will not cause multiple saves", async () => { + it("concurrent calls will not cause conflicts", async () => { async function parallelPatch(count: number) { - await Promise.all( - Array.from({ length: count }).map(() => - docWritethrough.patch(generatePatchObject(1)) - ) + const patches = Array.from({ length: count }).map(() => + generatePatchObject(1) ) - } + await Promise.all(patches.map(p => docWritethrough.patch(p))) - const storeToCacheSpy = jest.spyOn(docWritethrough as any, "storeToCache") + return patches.reduce((acc, c) => { + acc = { ...acc, ...c } + return acc + }, {}) + } + const queueMessageSpy = jest.spyOn(docWritethroughProcessorQueue, "add") await config.doInTenant(async () => { - await parallelPatch(5) - expect(storeToCacheSpy).toBeCalledTimes(5) - expect(await db.exists(documentId)).toBe(false) + let patches = await parallelPatch(5) + expect(queueMessageSpy).toBeCalledTimes(5) - await travelForward(WRITE_RATE_MS) - - await parallelPatch(40) - - expect(storeToCacheSpy).toBeCalledTimes(45) - - // Ideally we want to spy on persistToDb from ./docWritethrough, but due our barrel files configuration required quite of a complex setup. - // We are relying on the document being stored only once (otherwise we would have _rev updated) + await waitForQueueCompletion() expect(await db.get(documentId)).toEqual( - expect.objectContaining({ - _id: documentId, - _rev: expect.stringMatching(/1-.+/), - }) + expect.objectContaining(patches) ) - await parallelPatch(10) + patches = { ...patches, ...(await parallelPatch(40)) } + expect(queueMessageSpy).toBeCalledTimes(45) - expect(storeToCacheSpy).toBeCalledTimes(55) + await waitForQueueCompletion() expect(await db.get(documentId)).toEqual( - expect.objectContaining({ - _id: documentId, - _rev: expect.stringMatching(/1-.+/), - }) + expect.objectContaining(patches) ) - await travelForward(WRITE_RATE_MS) + patches = { ...patches, ...(await parallelPatch(10)) } + expect(queueMessageSpy).toBeCalledTimes(55) - await parallelPatch(5) - await travelForward(WRITE_RATE_MS) + await waitForQueueCompletion() expect(await db.get(documentId)).toEqual( - expect.objectContaining({ - _id: documentId, - _rev: expect.stringMatching(/3-.+/), - }) + expect.objectContaining(patches) + ) + }) + }) + + // This is not yet supported + it.skip("patches will execute in order", async () => { + let incrementalValue = 0 + const keyToOverride = generator.word() + async function incrementalPatches(count: number) { + for (let i = 0; i < count; i++) { + await docWritethrough.patch({ [keyToOverride]: incrementalValue++ }) + } + } + + await config.doInTenant(async () => { + await incrementalPatches(5) + + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ [keyToOverride]: 5 }) + ) + + await incrementalPatches(40) + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ [keyToOverride]: 45 }) ) - expect(storeToCacheSpy).toBeCalledTimes(60) }) }) }) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index 313b9a4d4a..ecfa20f99e 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -6,7 +6,7 @@ import env from "../environment" import * as accounts from "../accounts" import { UserDB } from "../users" import { sdk } from "@budibase/shared-core" -import { User } from "@budibase/types" +import { User, UserMetadata } from "@budibase/types" const EXPIRY_SECONDS = 3600 @@ -15,7 +15,7 @@ const EXPIRY_SECONDS = 3600 */ async function populateFromDB(userId: string, tenantId: string) { const db = tenancy.getTenantDB(tenantId) - const user = await db.get(userId) + const user = await db.get(userId) user.budibaseAccess = true if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { const account = await accounts.getAccount(user.email) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index f91a37ce8f..9c960d76dd 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,66 +1,57 @@ +import PouchDB from "pouchdb" import { getPouchDB, closePouchDB } from "./couch" import { DocumentType } from "../constants" class Replication { - source: any - target: any - replication: any + source: PouchDB.Database + target: PouchDB.Database - /** - * - * @param source - the DB you want to replicate or rollback to - * @param target - the DB you want to replicate to, or rollback from - */ - constructor({ source, target }: any) { + constructor({ source, target }: { source: string; target: string }) { this.source = getPouchDB(source) this.target = getPouchDB(target) } - close() { - return Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) + async close() { + await Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) } - promisify(operation: any, opts = {}) { - return new Promise(resolve => { - operation(this.target, opts) - .on("denied", function (err: any) { + replicate(opts: PouchDB.Replication.ReplicateOptions = {}) { + return new Promise>(resolve => { + this.source.replicate + .to(this.target, opts) + .on("denied", function (err) { // a document failed to replicate (e.g. due to permissions) throw new Error(`Denied: Document failed to replicate ${err}`) }) - .on("complete", function (info: any) { + .on("complete", function (info) { return resolve(info) }) - .on("error", function (err: any) { + .on("error", function (err) { throw new Error(`Replication Error: ${err}`) }) }) } - /** - * Two way replication operation, intended to be promise based. - * @param opts - PouchDB replication options - */ - sync(opts = {}) { - this.replication = this.promisify(this.source.sync, opts) - return this.replication - } + appReplicateOpts( + opts: PouchDB.Replication.ReplicateOptions = {} + ): PouchDB.Replication.ReplicateOptions { + if (typeof opts.filter === "string") { + return opts + } - /** - * One way replication operation, intended to be promise based. - * @param opts - PouchDB replication options - */ - replicate(opts = {}) { - this.replication = this.promisify(this.source.replicate.to, opts) - return this.replication - } + const filter = opts.filter + delete opts.filter - appReplicateOpts() { return { - filter: (doc: any) => { + ...opts, + filter: (doc: any, params: any) => { if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) { return false } - return doc._id !== DocumentType.APP_METADATA + if (doc._id === DocumentType.APP_METADATA) { + return false + } + return filter ? filter(doc, params) : true }, } } @@ -75,10 +66,6 @@ class Replication { // take the opportunity to remove deleted tombstones await this.replicate() } - - cancel() { - this.replication.cancel() - } } export default Replication diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 6c8107c7a4..afb5592562 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -1,8 +1,9 @@ import events from "events" -import { timeout } from "../utils" +import { newid, timeout } from "../utils" import { Queue, QueueOptions, JobOptions } from "./queue" interface JobMessage { + id: string timestamp: number queue: string data: any @@ -20,6 +21,7 @@ interface JobMessage { */ function newJob(queue: string, message: any, opts?: JobOptions): JobMessage { return { + id: newid(), timestamp: Date.now(), queue: queue, data: message, @@ -74,8 +76,23 @@ class InMemoryQueue implements Partial { let msg = this._messages.shift() let resp = func(msg) + + async function retryFunc(fnc: any) { + try { + await fnc + } catch (e: any) { + await new Promise(r => setTimeout(() => r(), 50)) + + await retryFunc(func(msg)) + } + } + if (resp.then != null) { - await resp + try { + await retryFunc(resp) + } catch (e: any) { + console.error(e) + } } this._runCount++ const jobId = msg?.opts?.jobId?.toString() @@ -173,7 +190,7 @@ class InMemoryQueue implements Partial { async waitForCompletion() { do { await timeout(50) - } while (this.hasRunningJobs) + } while (this.hasRunningJobs()) } hasRunningJobs() { diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 8cfa3db5c1..79f75421d3 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -291,23 +291,16 @@ class RedisWrapper { return acc }, {} as Record) - const luaScript = ` - for i, key in ipairs(KEYS) do - redis.call('MSET', key, ARGV[i]) - ${ - expirySeconds !== null - ? `redis.call('EXPIRE', key, ARGV[#ARGV])` - : "" - } - end - ` - const keys = Object.keys(dataToStore) - const values = Object.values(dataToStore) + const pipeline = client.pipeline() + pipeline.mset(dataToStore) + if (expirySeconds !== null) { - values.push(expirySeconds) + for (const key of Object.keys(dataToStore)) { + pipeline.expire(key, expirySeconds) + } } - await client.eval(luaScript, keys.length, ...keys, ...values) + await pipeline.exec() } async getTTL(key: string) { @@ -327,10 +320,35 @@ class RedisWrapper { await this.getClient().del(addDbPrefix(db, key)) } + async bulkDelete(keys: string[]) { + const db = this._db + await this.getClient().del(keys.map(key => addDbPrefix(db, key))) + } + async clear() { let items = await this.scan() await Promise.all(items.map((obj: any) => this.delete(obj.key))) } + + async increment(key: string) { + const result = await this.getClient().incr(addDbPrefix(this._db, key)) + if (isNaN(result)) { + throw new Error(`Redis ${key} does not contain a number`) + } + return result + } + + async deleteIfValue(key: string, value: any) { + const client = this.getClient() + + const luaScript = ` + if redis.call('GET', KEYS[1]) == ARGV[1] then + redis.call('DEL', KEYS[1]) + end + ` + + await client.eval(luaScript, 1, addDbPrefix(this._db, key), value) + } } export default RedisWrapper diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index 1fd40acc37..c2c9e4a14e 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -1,8 +1,28 @@ +import { GenericContainer, StartedTestContainer } from "testcontainers" import { generator, structures } from "../../../tests" import RedisWrapper from "../redis" +import { env } from "../.." + +jest.setTimeout(30000) describe("redis", () => { let redis: RedisWrapper + let container: StartedTestContainer + + beforeAll(async () => { + const container = await new GenericContainer("redis") + .withExposedPorts(6379) + .start() + + env._set( + "REDIS_URL", + `${container.getHost()}:${container.getMappedPort(6379)}` + ) + env._set("MOCK_REDIS", 0) + env._set("REDIS_PASSWORD", 0) + }) + + afterAll(() => container?.stop()) beforeEach(async () => { redis = new RedisWrapper(structures.db.id()) @@ -107,4 +127,88 @@ describe("redis", () => { expect(await redis.keys("*")).toHaveLength(0) }) }) + + describe("increment", () => { + it("can increment on a new key", async () => { + const key = structures.uuid() + const result = await redis.increment(key) + expect(result).toBe(1) + }) + + it("can increment multiple times", async () => { + const key = structures.uuid() + const results = [ + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + ] + expect(results).toEqual([1, 2, 3, 4, 5]) + }) + + it("can increment on a new key", async () => { + const key1 = structures.uuid() + const key2 = structures.uuid() + + const result1 = await redis.increment(key1) + expect(result1).toBe(1) + + const result2 = await redis.increment(key2) + expect(result2).toBe(1) + }) + + it("can increment multiple times in parallel", async () => { + const key = structures.uuid() + const results = await Promise.all( + Array.from({ length: 100 }).map(() => redis.increment(key)) + ) + expect(results).toHaveLength(100) + expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1)) + }) + + it("can increment existing set keys", async () => { + const key = structures.uuid() + await redis.store(key, 70) + await redis.increment(key) + + const result = await redis.increment(key) + expect(result).toBe(72) + }) + + it.each([ + generator.word(), + generator.bool(), + { [generator.word()]: generator.word() }, + ])("cannot increment if the store value is not a number", async value => { + const key = structures.uuid() + await redis.store(key, value) + + await expect(redis.increment(key)).rejects.toThrowError( + "ERR value is not an integer or out of range" + ) + }) + }) + + describe("deleteIfValue", () => { + it("can delete if the value matches", async () => { + const key = structures.uuid() + const value = generator.word() + await redis.store(key, value) + + await redis.deleteIfValue(key, value) + + expect(await redis.get(key)).toBeNull() + }) + + it("will not delete if the value does not matches", async () => { + const key = structures.uuid() + const value = generator.word() + await redis.store(key, value) + + await redis.deleteIfValue(key, generator.word()) + + expect(await redis.get(key)).toEqual(value) + }) + }) }) diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 01473ad991..a64be6b319 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -101,10 +101,7 @@ export function getBuiltinRole(roleId: string): Role | undefined { /** * Works through the inheritance ranks to see how far up the builtin stack this ID is. */ -export function builtinRoleToNumber(id?: string) { - if (!id) { - return 0 - } +export function builtinRoleToNumber(id: string) { const builtins = getBuiltinRoles() const MAX = Object.values(builtins).length + 1 if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) { diff --git a/packages/backend-core/tests/core/utilities/structures/accounts.ts b/packages/backend-core/tests/core/utilities/structures/accounts.ts index 515f94db1e..7dcc2de116 100644 --- a/packages/backend-core/tests/core/utilities/structures/accounts.ts +++ b/packages/backend-core/tests/core/utilities/structures/accounts.ts @@ -18,7 +18,7 @@ export const account = (partial: Partial = {}): Account => { return { accountId: uuid(), tenantId: generator.word(), - email: generator.email(), + email: generator.email({ domain: "example.com" }), tenantName: generator.word(), hosting: Hosting.SELF, createdAt: Date.now(), diff --git a/packages/backend-core/tests/core/utilities/structures/scim.ts b/packages/backend-core/tests/core/utilities/structures/scim.ts index 80f41c605d..f424b2881a 100644 --- a/packages/backend-core/tests/core/utilities/structures/scim.ts +++ b/packages/backend-core/tests/core/utilities/structures/scim.ts @@ -13,7 +13,7 @@ interface CreateUserRequestFields { export function createUserRequest(userData?: Partial) { const defaultValues = { externalId: uuid(), - email: generator.email(), + email: `${uuid()}@example.com`, firstName: generator.first(), lastName: generator.last(), username: generator.name(), diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js index cc169eac09..d259b9197a 100644 --- a/packages/bbui/src/Actions/position_dropdown.js +++ b/packages/bbui/src/Actions/position_dropdown.js @@ -35,7 +35,10 @@ export default function positionDropdown(element, opts) { } if (typeof customUpdate === "function") { - styles = customUpdate(anchorBounds, elementBounds, styles) + styles = customUpdate(anchorBounds, elementBounds, { + ...styles, + offset: opts.offset, + }) } else { // Determine vertical styles if (align === "right-outside") { diff --git a/packages/builder/src/analytics/index.js b/packages/builder/src/analytics/index.js index 6bb10acdb5..3a80a05d7f 100644 --- a/packages/builder/src/analytics/index.js +++ b/packages/builder/src/analytics/index.js @@ -9,13 +9,17 @@ const intercom = new IntercomClient(process.env.INTERCOM_TOKEN) class AnalyticsHub { constructor() { this.clients = [posthog, intercom] + this.initialised = false } async activate() { // Check analytics are enabled const analyticsStatus = await API.getAnalyticsStatus() - if (analyticsStatus.enabled) { - this.clients.forEach(client => client.init()) + if (analyticsStatus.enabled && !this.initialised) { + this.clients.forEach(client => { + client.init() + }) + this.initialised = true } } diff --git a/packages/builder/src/components/deploy/AppActions.svelte b/packages/builder/src/components/deploy/AppActions.svelte index 86427ff761..9e5de1d3bf 100644 --- a/packages/builder/src/components/deploy/AppActions.svelte +++ b/packages/builder/src/components/deploy/AppActions.svelte @@ -164,9 +164,10 @@ {/if}
@@ -209,7 +210,7 @@
- + Publish - import EditComponentPopover from "../EditComponentPopover.svelte" + import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte" import { Icon } from "@budibase/bbui" import { runtimeToReadableBinding } from "dataBinding" import { isJSBinding } from "@budibase/string-templates" diff --git a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte b/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte similarity index 83% rename from packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte rename to packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte index 4e645fe343..39e4bc2ada 100644 --- a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte +++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte @@ -3,6 +3,7 @@ import { componentStore } from "stores/builder" import { cloneDeep } from "lodash/fp" import { createEventDispatcher, getContext } from "svelte" + import { customPositionHandler } from "." import ComponentSettingsSection from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte" export let anchor @@ -54,25 +55,6 @@ dispatch("change", nestedComponentInstance) } - - const customPositionHandler = (anchorBounds, eleBounds, cfg) => { - let { left, top } = cfg - let percentageOffset = 30 - // left-outside - left = anchorBounds.left - eleBounds.width - 18 - - // shift up from the anchor, if space allows - let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset - let defaultTop = anchorBounds.top - offsetPos - - if (window.innerHeight - defaultTop < eleBounds.height) { - top = window.innerHeight - eleBounds.height - 5 - } else { - top = anchorBounds.top - offsetPos - } - - return { ...cfg, left, top } - } 0} maxHeight={600} + offset={18} handlePostionUpdate={customPositionHandler} > diff --git a/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js b/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js new file mode 100644 index 0000000000..2dc3f60185 --- /dev/null +++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js @@ -0,0 +1,18 @@ +export const customPositionHandler = (anchorBounds, eleBounds, cfg) => { + let { left, top, offset } = cfg + let percentageOffset = 30 + // left-outside + left = anchorBounds.left - eleBounds.width - (offset || 5) + + // shift up from the anchor, if space allows + let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset + let defaultTop = anchorBounds.top - offsetPos + + if (window.innerHeight - defaultTop < eleBounds.height) { + top = window.innerHeight - eleBounds.height - 5 + } else { + top = anchorBounds.top - offsetPos + } + + return { ...cfg, left, top } +} diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte index 27590a9858..771bcf20e0 100644 --- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte +++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte @@ -1,5 +1,5 @@
{#if label && !labelHidden} @@ -115,6 +122,16 @@
diff --git a/packages/builder/src/components/portal/onboarding/steps/index.js b/packages/builder/src/components/portal/onboarding/steps/index.js index 8e27748f36..e15d191652 100644 --- a/packages/builder/src/components/portal/onboarding/steps/index.js +++ b/packages/builder/src/components/portal/onboarding/steps/index.js @@ -1,3 +1,5 @@ export { default as OnboardingData } from "./OnboardingData.svelte" export { default as OnboardingDesign } from "./OnboardingDesign.svelte" export { default as OnboardingPublish } from "./OnboardingPublish.svelte" +export { default as NewViewUpdateFormRowId } from "./NewViewUpdateFormRowId.svelte" +export { default as NewFormSteps } from "./NewFormSteps.svelte" diff --git a/packages/builder/src/components/portal/onboarding/tours.js b/packages/builder/src/components/portal/onboarding/tours.js index 623a00fcc2..fab97cdd03 100644 --- a/packages/builder/src/components/portal/onboarding/tours.js +++ b/packages/builder/src/components/portal/onboarding/tours.js @@ -2,8 +2,15 @@ import { get } from "svelte/store" import { builderStore } from "stores/builder" import { auth } from "stores/portal" import analytics from "analytics" -import { OnboardingData, OnboardingDesign, OnboardingPublish } from "./steps" +import { + OnboardingData, + OnboardingDesign, + OnboardingPublish, + NewViewUpdateFormRowId, + NewFormSteps, +} from "./steps" import { API } from "api" +import { customPositionHandler } from "components/design/settings/controls/EditComponentPopover" const ONBOARDING_EVENT_PREFIX = "onboarding" @@ -14,11 +21,26 @@ export const TOUR_STEP_KEYS = { BUILDER_USER_MANAGEMENT: "builder-user-management", BUILDER_AUTOMATION_SECTION: "builder-automation-section", FEATURE_USER_MANAGEMENT: "feature-user-management", + BUILDER_FORM_CREATE_STEPS: "builder-form-create-steps", + BUILDER_FORM_VIEW_UPDATE_STEPS: "builder-form-view-update-steps", + BUILDER_FORM_ROW_ID: "builder-form-row-id", } export const TOUR_KEYS = { TOUR_BUILDER_ONBOARDING: "builder-onboarding", FEATURE_ONBOARDING: "feature-onboarding", + BUILDER_FORM_CREATE: "builder-form-create", + BUILDER_FORM_VIEW_UPDATE: "builder-form-view-update", +} + +export const getCurrentStepIdx = (steps, tourStepKey) => { + if (!steps?.length) { + return + } + if (steps?.length && !tourStepKey) { + return 0 + } + return steps.findIndex(step => step.id === tourStepKey) } const endUserOnboarding = async ({ skipped = false } = {}) => { @@ -37,13 +59,8 @@ const endUserOnboarding = async ({ skipped = false } = {}) => { // Update the cached user await auth.getSelf() - builderStore.update(state => ({ - ...state, - tourNodes: null, - tourKey: null, - tourStepKey: null, - onboarding: false, - })) + builderStore.endBuilderOnboarding() + builderStore.setTour() } catch (e) { console.error("Onboarding failed", e) return false @@ -52,9 +69,29 @@ const endUserOnboarding = async ({ skipped = false } = {}) => { } } -const tourEvent = eventKey => { +const endTour = async ({ key, skipped = false } = {}) => { + const { tours = {} } = get(auth).user + tours[key] = new Date().toISOString() + + await API.updateSelf({ + tours, + }) + + if (skipped) { + tourEvent(key, skipped) + } + + // Update the cached user + await auth.getSelf() + + // Reset tour state + builderStore.setTour() +} + +const tourEvent = (eventKey, skipped) => { analytics.captureEvent(`${ONBOARDING_EVENT_PREFIX}:${eventKey}`, { eventSource: EventSource.PORTAL, + skipped, }) } @@ -135,7 +172,71 @@ const getTours = () => { }, ], }, + [TOUR_KEYS.BUILDER_FORM_CREATE]: { + steps: [ + { + id: TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS, + title: "Add multiple steps", + layout: NewFormSteps, + query: "#steps-prop-control-wrap", + onComplete: () => { + builderStore.highlightSetting() + endTour({ key: TOUR_KEYS.BUILDER_FORM_CREATE }) + }, + onLoad: () => { + tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS) + builderStore.highlightSetting("steps", "info") + }, + positionHandler: customPositionHandler, + align: "left-outside", + }, + ], + }, + [TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE]: { + steps: [ + { + id: TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID, + title: "Add row ID to update a row", + layout: NewViewUpdateFormRowId, + query: "#rowId-prop-control-wrap", + onLoad: () => { + tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID) + builderStore.highlightSetting("rowId", "info") + }, + positionHandler: customPositionHandler, + align: "left-outside", + }, + { + id: TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS, + title: "Add multiple steps", + layout: NewFormSteps, + query: "#steps-prop-control-wrap", + onComplete: () => { + builderStore.highlightSetting() + endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE }) + }, + onLoad: () => { + tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS) + builderStore.highlightSetting("steps", "info") + }, + positionHandler: customPositionHandler, + align: "left-outside", + scrollIntoView: true, + }, + ], + onSkip: async () => { + builderStore.highlightSetting() + endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE, skipped: true }) + }, + }, } } export const TOURS = getTours() +export const TOURSBYSTEP = Object.keys(TOURS).reduce((acc, tour) => { + TOURS[tour].steps.forEach(element => { + acc[element.id] = element + acc[element.id]["tour"] = tour + }) + return acc +}, {}) diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js index 0442a67da9..8fdf71c156 100644 --- a/packages/builder/src/dataBinding.js +++ b/packages/builder/src/dataBinding.js @@ -1146,7 +1146,7 @@ export const getAllStateVariables = () => { "@budibase/standard-components/multistepformblockstep" ) - steps.forEach(step => { + steps?.forEach(step => { parseComponentSettings(stepDefinition, step) }) }) diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index dd66f5bc34..16a1adadee 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -96,7 +96,7 @@ const release_date = new Date("2023-03-01T00:00:00.000Z") const onboarded = new Date($auth.user?.onboardedAt) if (onboarded < release_date) { - builderStore.startTour(TOUR_KEYS.FEATURE_ONBOARDING) + builderStore.setTour(TOUR_KEYS.FEATURE_ONBOARDING) } } } @@ -144,7 +144,7 @@ {#each $layout.children as { path, title }} - + { try { @@ -46,7 +54,6 @@ $: id = $selectedComponent?._id $: id, (section = tabs[0]) - $: componentName = getComponentName(componentInstance) @@ -92,13 +99,21 @@
{#if section == "settings"} - + + + {/if} {#if section == "styles"} import { helpers } from "@budibase/shared-core" import { DetailSummary, notifications } from "@budibase/bbui" - import { componentStore } from "stores/builder" + import { componentStore, builderStore } from "stores/builder" import PropertyControl from "components/design/settings/controls/PropertyControl.svelte" import ResetFieldsButton from "components/design/settings/controls/ResetFieldsButton.svelte" import EjectBlockButton from "components/design/settings/controls/EjectBlockButton.svelte" @@ -177,9 +177,7 @@ defaultValue={setting.defaultValue} nested={setting.nested} onChange={val => updateSetting(setting, val)} - highlighted={$componentStore.highlightedSettingKey === - setting.key} - propertyFocus={$componentStore.propertyFocus === setting.key} + propertyFocus={$builderStore.propertyFocus === setting.key} info={setting.info} disableBindings={setting.disableBindings} props={{ diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index fa126bbc99..bf66c28a09 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -174,7 +174,7 @@ } else if (type === "request-add-component") { toggleAddComponent() } else if (type === "highlight-setting") { - builderStore.highlightSetting(data.setting) + builderStore.highlightSetting(data.setting, "error") // Also scroll setting into view const selector = `#${data.setting}-prop-control` diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte index eadd9d4257..8c1a11289d 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte @@ -3,13 +3,23 @@ import DatasourceModal from "./DatasourceModal.svelte" import ScreenRoleModal from "./ScreenRoleModal.svelte" import sanitizeUrl from "helpers/sanitizeUrl" + import FormTypeModal from "./FormTypeModal.svelte" import { Modal, notifications } from "@budibase/bbui" - import { screenStore, navigationStore, tables } from "stores/builder" + import { + screenStore, + navigationStore, + tables, + builderStore, + } from "stores/builder" + import { auth } from "stores/portal" import { get } from "svelte/store" import getTemplates from "templates" import { Roles } from "constants/backend" import { capitalise } from "helpers" import { goto } from "@roxi/routify" + import { TOUR_KEYS } from "components/portal/onboarding/tours.js" + import formScreen from "templates/formScreen" + import rowListScreen from "templates/rowListScreen" let mode let pendingScreen @@ -18,12 +28,18 @@ let screenDetailsModal let datasourceModal let screenAccessRoleModal + let formTypeModal // Cache variables for workflow let screenAccessRole = Roles.BASIC - let selectedTemplates = null + + let templates = null + let screens = null + + let selectedDatasources = null let blankScreenUrl = null let screenMode = null + let formType = null // Creates an array of screens, checking and sanitising their URLs const createScreens = async ({ screens, screenAccessRole }) => { @@ -32,7 +48,7 @@ } try { - let screenId + let createdScreens = [] for (let screen of screens) { // Check we aren't clashing with an existing URL @@ -56,7 +72,7 @@ // Create the screen const response = await screenStore.save(screen) - screenId = response._id + createdScreens.push(response) // Add link in layout. We only ever actually create 1 screen now, even // for autoscreens, so it's always safe to do this. @@ -66,9 +82,7 @@ ) } - // Go to new screen - $goto(`./${screenId}`) - screenStore.select(screenId) + return createdScreens } catch (error) { console.error(error) notifications.error("Error creating screens") @@ -104,13 +118,16 @@ // Handler for NewScreenModal export const show = newMode => { mode = newMode - selectedTemplates = null + templates = null + screens = null + selectedDatasources = null blankScreenUrl = null screenMode = mode pendingScreen = null screenAccessRole = Roles.BASIC + formType = null - if (mode === "table" || mode === "grid") { + if (mode === "table" || mode === "grid" || mode === "form") { datasourceModal.show() } else if (mode === "blank") { let templates = getTemplates($tables.list) @@ -125,19 +142,26 @@ } // Handler for DatasourceModal confirmation, move to screen access select - const confirmScreenDatasources = async ({ templates }) => { - selectedTemplates = templates - screenAccessRoleModal.show() + const confirmScreenDatasources = async ({ datasources }) => { + selectedDatasources = datasources + if (screenMode === "form") { + formTypeModal.show() + } else { + screenAccessRoleModal.show() + } } // Handler for Datasource Screen Creation const completeDatasourceScreenCreation = async () => { - const screens = selectedTemplates.map(template => { + templates = rowListScreen(selectedDatasources, mode) + + const screens = templates.map(template => { let screenTemplate = template.create() screenTemplate.autoTableId = template.resourceId return screenTemplate }) - await createScreens({ screens, screenAccessRole }) + const createdScreens = await createScreens({ screens, screenAccessRole }) + loadNewScreen(createdScreens) } const confirmScreenBlank = async ({ screenUrl }) => { @@ -154,7 +178,54 @@ return } pendingScreen.routing.route = screenUrl - await createScreens({ screens: [pendingScreen], screenAccessRole }) + const createdScreens = await createScreens({ + screens: [pendingScreen], + screenAccessRole, + }) + loadNewScreen(createdScreens) + } + + const onConfirmFormType = () => { + screenAccessRoleModal.show() + } + + const loadNewScreen = createdScreens => { + const lastScreen = createdScreens.slice(-1)[0] + + // Go to new screen + if (lastScreen?.props?._children.length) { + // Focus on the main component for the streen type + const mainComponent = lastScreen?.props?._children?.[0]._id + $goto(`./${lastScreen._id}/${mainComponent}`) + } else { + $goto(`./${lastScreen._id}`) + } + + screenStore.select(lastScreen._id) + } + + const confirmFormScreenCreation = async () => { + templates = formScreen(selectedDatasources, { actionType: formType }) + screens = templates.map(template => { + let screenTemplate = template.create() + return screenTemplate + }) + const createdScreens = await createScreens({ screens, screenAccessRole }) + + if (formType === "Update" || formType === "Create") { + const associatedTour = + formType === "Update" + ? TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE + : TOUR_KEYS.BUILDER_FORM_CREATE + + const tourRequired = !$auth?.user?.tours?.[associatedTour] + if (tourRequired) { + builderStore.setTour(associatedTour) + } + } + + // Go to new screen + loadNewScreen(createdScreens) } // Submit screen config for creation. @@ -164,6 +235,8 @@ screenUrl: blankScreenUrl, screenAccessRole, }) + } else if (screenMode === "form") { + confirmFormScreenCreation() } else { completeDatasourceScreenCreation() } @@ -179,19 +252,18 @@ - + { + confirmScreenCreation() + }} bind:screenAccessRole + onCancel={roleSelectBack} screenUrl={blankScreenUrl} + confirmText={screenMode === "form" ? "Confirm" : "Done"} /> @@ -201,3 +273,17 @@ initialUrl={blankScreenUrl} /> + + + { + formTypeModal.hide() + datasourceModal.show() + }} + on:select={e => { + formType = e.detail + }} + type={formType} + /> + diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte index bb6c47c4b1..8e2428d832 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte @@ -4,37 +4,33 @@ import ICONS from "components/backend/DatasourceNavigator/icons" import { IntegrationNames } from "constants" import { onMount } from "svelte" - import rowListScreen from "templates/rowListScreen" import DatasourceTemplateRow from "./DatasourceTemplateRow.svelte" - export let mode export let onCancel export let onConfirm - export let initialScreens = [] - let selectedScreens = [...initialScreens] + let selectedSources = [] $: filteredSources = $datasources.list?.filter(datasource => { return datasource.source !== IntegrationNames.REST && datasource["entities"] }) const toggleSelection = datasource => { - const { resourceId } = datasource - if (selectedScreens.find(s => s.resourceId === resourceId)) { - selectedScreens = selectedScreens.filter( - screen => screen.resourceId !== resourceId + const exists = selectedSources.find( + d => d.resourceId === datasource.resourceId + ) + if (exists) { + selectedSources = selectedSources.filter( + d => d.resourceId === datasource.resourceId ) } else { - selectedScreens = [ - ...selectedScreens, - rowListScreen([datasource], mode)[0], - ] + selectedSources = [...selectedSources, datasource] } } const confirmDatasourceSelection = async () => { await onConfirm({ - templates: selectedScreens, + datasources: selectedSources, }) } @@ -54,7 +50,7 @@ cancelText="Back" onConfirm={confirmDatasourceSelection} {onCancel} - disabled={!selectedScreens.length} + disabled={!selectedSources.length} size="L" > @@ -85,8 +81,8 @@ resourceId: table._id, type: "table", }} - {@const selected = selectedScreens.find( - screen => screen.resourceId === tableDS.resourceId + {@const selected = selectedSources.find( + datasource => datasource.resourceId === tableDS.resourceId )} toggleSelection(tableDS)} @@ -103,7 +99,7 @@ tableId: view.tableId, type: "viewV2", }} - {@const selected = selectedScreens.find( + {@const selected = selectedSources.find( x => x.resourceId === viewDS.resourceId )} + import { ModalContent, Layout, Body, Icon } from "@budibase/bbui" + import { createEventDispatcher } from "svelte" + + export let onCancel = () => {} + export let onConfirm = () => {} + export let type + + const dispatch = createEventDispatcher() + + + + + + + +
{ + dispatch("select", "Create") + }} + > +
+
+ Create a new row + + For capturing and storing new data from your users + +
+ {#if type === "Create"} + + + + {/if} +
+
+
{ + dispatch("select", "Update") + }} + > +
+
+ Update an existing row + For viewing and updating existing data +
+ {#if type === "Update"} + + + + {/if} +
+
+
{ + dispatch("select", "View") + }} + > +
+
+ View an existing row + For a read only view of your data +
+ {#if type === "View"} + + + + {/if} +
+
+ + + + + diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte index 5e34d32219..8605412c1c 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte @@ -9,6 +9,7 @@ export let onCancel export let screenUrl export let screenAccessRole + export let confirmText = "Done" let error @@ -40,7 +41,7 @@ import { Body } from "@budibase/bbui" import CreationPage from "components/common/CreationPage.svelte" - import blankImage from "./blank.png" - import tableImage from "./table.png" - import gridImage from "./grid.png" + import blankImage from "./images/blank.png" + import tableImage from "./images/table.png" + import gridImage from "./images/grid.png" + import formImage from "./images/form.png" import CreateScreenModal from "./CreateScreenModal.svelte" import { screenStore } from "stores/builder" @@ -56,6 +57,16 @@ View and manipulate rows on a grid
+ +
createScreenModal.show("form")}> +
+ +
+
+ Form + Capture data from your users +
+
diff --git a/packages/builder/src/stores/builder/app.js b/packages/builder/src/stores/builder/app.js index cb58f69fa8..275d143f80 100644 --- a/packages/builder/src/stores/builder/app.js +++ b/packages/builder/src/stores/builder/app.js @@ -29,6 +29,7 @@ export const INITIAL_APP_META_STATE = { initialised: false, hasAppPackage: false, usedPlugins: null, + automations: {}, routes: {}, } @@ -63,6 +64,7 @@ export class AppMetaStore extends BudiStore { ...app.features, }, initialised: true, + automations: app.automations || {}, hasAppPackage: true, })) } diff --git a/packages/builder/src/stores/builder/builder.js b/packages/builder/src/stores/builder/builder.js index 94fbfafb19..398f3c1606 100644 --- a/packages/builder/src/stores/builder/builder.js +++ b/packages/builder/src/stores/builder/builder.js @@ -2,12 +2,11 @@ import { get } from "svelte/store" import { createBuilderWebsocket } from "./websocket.js" import { BuilderSocketEvent } from "@budibase/shared-core" import BudiStore from "./BudiStore" -import { previewStore } from "./preview.js" import { TOUR_KEYS } from "components/portal/onboarding/tours.js" export const INITIAL_BUILDER_STATE = { previousTopNavPath: {}, - highlightedSettingKey: null, + highlightedSetting: null, propertyFocus: null, builderSidePanel: false, onboarding: false, @@ -26,7 +25,6 @@ export class BuilderStore extends BudiStore { this.reset = this.reset.bind(this) this.highlightSetting = this.highlightSetting.bind(this) this.propertyFocus = this.propertyFocus.bind(this) - this.hover = this.hover.bind(this) this.hideBuilderSidePanel = this.hideBuilderSidePanel.bind(this) this.showBuilderSidePanel = this.showBuilderSidePanel.bind(this) this.setPreviousTopNavPath = this.setPreviousTopNavPath.bind(this) @@ -58,10 +56,10 @@ export class BuilderStore extends BudiStore { this.websocket = null } - highlightSetting(key) { + highlightSetting(key, type) { this.update(state => ({ ...state, - highlightedSettingKey: key, + highlightedSetting: key ? { key, type: type || "info" } : null, })) } @@ -135,25 +133,20 @@ export class BuilderStore extends BudiStore { })) } - startTour(tourKey) { + endBuilderOnboarding() { this.update(state => ({ ...state, - tourKey: tourKey, + onboarding: false, })) } - hover(componentId, notifyClient = true) { - const store = get(this.store) - if (componentId === store.hoveredComponentId) { - return - } - this.update(state => { - state.hoveredComponentId = componentId - return state - }) - if (notifyClient) { - previewStore.sendEvent("hover-component", componentId) - } + setTour(tourKey) { + this.update(state => ({ + ...state, + tourStepKey: null, + tourNodes: null, + tourKey: tourKey, + })) } } diff --git a/packages/builder/src/stores/builder/tests/builder.test.js b/packages/builder/src/stores/builder/tests/builder.test.js index 7aac2489db..e6f52689aa 100644 --- a/packages/builder/src/stores/builder/tests/builder.test.js +++ b/packages/builder/src/stores/builder/tests/builder.test.js @@ -88,14 +88,42 @@ describe("Builder store", () => { ) }) - it("Sync a highlighted setting key to state", ctx => { - expect(ctx.test.store.highlightedSettingKey).toBeNull() + it("Sync a highlighted setting key to state. Default to info type", ctx => { + expect(ctx.test.store.highlightedSetting).toBeNull() ctx.test.builderStore.highlightSetting("testing") expect(ctx.test.store).toStrictEqual({ ...INITIAL_BUILDER_STATE, - highlightedSettingKey: "testing", + highlightedSetting: { + key: "testing", + type: "info", + }, + }) + }) + + it("Sync a highlighted setting key to state. Use provided type", ctx => { + expect(ctx.test.store.highlightedSetting).toBeNull() + + ctx.test.builderStore.highlightSetting("testing", "error") + + expect(ctx.test.store).toStrictEqual({ + ...INITIAL_BUILDER_STATE, + highlightedSetting: { + key: "testing", + type: "error", + }, + }) + }) + + it("Sync a highlighted setting key to state. Unset when no value is passed", ctx => { + expect(ctx.test.store.highlightedSetting).toBeNull() + + ctx.test.builderStore.highlightSetting("testing", "error") + ctx.test.builderStore.highlightSetting() + + expect(ctx.test.store).toStrictEqual({ + ...INITIAL_BUILDER_STATE, }) }) diff --git a/packages/builder/src/templates/formScreen.js b/packages/builder/src/templates/formScreen.js new file mode 100644 index 0000000000..b99744e8e8 --- /dev/null +++ b/packages/builder/src/templates/formScreen.js @@ -0,0 +1,43 @@ +import { Screen } from "./Screen" +import { Component } from "./Component" +import sanitizeUrl from "helpers/sanitizeUrl" + +export const FORM_TEMPLATE = "FORM_TEMPLATE" +export const formUrl = datasource => sanitizeUrl(`/${datasource.label}-form`) + +// Mode not really necessary +export default function (datasources, config) { + if (!Array.isArray(datasources)) { + return [] + } + return datasources.map(datasource => { + return { + name: `${datasource.label} - Form`, + create: () => createScreen(datasource, config), + id: FORM_TEMPLATE, + resourceId: datasource.resourceId, + } + }) +} + +const generateMultistepFormBlock = (dataSource, { actionType } = {}) => { + const multistepFormBlock = new Component( + "@budibase/standard-components/multistepformblock" + ) + multistepFormBlock + .customProps({ + actionType, + dataSource, + steps: [{}], + }) + .instanceName(`${dataSource.label} - Multistep Form block`) + return multistepFormBlock +} + +const createScreen = (datasource, config) => { + return new Screen() + .route(formUrl(datasource)) + .instanceName(`${datasource.label} - Form`) + .addChild(generateMultistepFormBlock(datasource, config)) + .json() +} diff --git a/packages/builder/src/templates/index.js b/packages/builder/src/templates/index.js index 3ff42fdec6..fff31cc070 100644 --- a/packages/builder/src/templates/index.js +++ b/packages/builder/src/templates/index.js @@ -1,7 +1,11 @@ import rowListScreen from "./rowListScreen" import createFromScratchScreen from "./createFromScratchScreen" +import formScreen from "./formScreen" -const allTemplates = datasources => [...rowListScreen(datasources)] +const allTemplates = datasources => [ + ...rowListScreen(datasources), + ...formScreen(datasources), +] // Allows us to apply common behaviour to all create() functions const createTemplateOverride = template => () => { @@ -19,6 +23,7 @@ export default datasources => { }) const fromScratch = enrichTemplate(createFromScratchScreen) const tableTemplates = allTemplates(datasources).map(enrichTemplate) + return [ fromScratch, ...tableTemplates.sort((templateA, templateB) => { diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 43b75ebe26..10f9c5f412 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -525,6 +525,38 @@ "barTitle": "Disable button", "key": "disabled" }, + { + "type": "icon", + "label": "Icon", + "key": "icon" + }, + { + "type": "select", + "label": "Gap", + "key": "gap", + "showInBar": true, + "barStyle": "picker", + "dependsOn": "icon", + "options": [ + { + "label": "None", + "value": "N" + }, + { + "label": "Small", + "value": "S" + }, + { + "label": "Medium", + "value": "M" + }, + { + "label": "Large", + "value": "L" + } + ], + "defaultValue": "M" + }, { "type": "event", "label": "On click", diff --git a/packages/client/src/components/app/Button.svelte b/packages/client/src/components/app/Button.svelte index 361e64a983..c43face1bb 100644 --- a/packages/client/src/components/app/Button.svelte +++ b/packages/client/src/components/app/Button.svelte @@ -13,9 +13,10 @@ export let size = "M" export let type = "cta" export let quiet = false + export let icon = null + export let gap = "M" // For internal use only for now - not defined in the manifest - export let icon = null export let active = false const handleOnClick = async () => { @@ -47,7 +48,7 @@ {#key $component.editing} @@ -92,4 +85,13 @@ .active { color: var(--spectrum-global-color-blue-600); } + .gap-S { + gap: 8px; + } + .gap-M { + gap: 16px; + } + .gap-L { + gap: 32px; + } diff --git a/packages/client/src/components/app/ButtonGroup.svelte b/packages/client/src/components/app/ButtonGroup.svelte index 3ee703e253..2cf6b3db7d 100644 --- a/packages/client/src/components/app/ButtonGroup.svelte +++ b/packages/client/src/components/app/ButtonGroup.svelte @@ -20,7 +20,7 @@ wrap: true, }} > - {#each buttons as { text, type, quiet, disabled, onClick, size }} + {#each buttons as { text, type, quiet, disabled, onClick, size, icon, gap }} diff --git a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte index 199a6122ab..549574e89b 100644 --- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte +++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte @@ -92,9 +92,9 @@ {#if schemaLoaded}