CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-04 05:50:57 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'BUDI-8064/doc-writethrough' into BUDI-8046/scim-logger

Browse source
This commit is contained in:
Adria Navarro 2024-03-07 11:05:16 +01:00
parent db75c05942 977daff05c
commit e60033bd89
82 changed files with 1686 additions and 527 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.eslintrc.json
View file

@ -44,7 +44,8 @@
"no-undef": "off",
"no-prototype-builtins": "off",
"local-rules/no-budibase-imports": "error",
"local-rules/no-test-com": "error"
"local-rules/no-test-com": "error",
"local-rules/email-domain-example-com": "error"
}
},
{

37
eslint-local-rules/index.js
View file

@ -51,4 +51,41 @@ module.exports = {
}
},
},
"email-domain-example-com": {
meta: {
type: "problem",
docs: {
description:
"enforce using the example.com domain for generator.email calls",
category: "Possible Errors",
recommended: false,
},
fixable: "code",
schema: [],
},
create: function (context) {
return {
CallExpression(node) {
if (
node.callee.type === "MemberExpression" &&
node.callee.object.name === "generator" &&
node.callee.property.name === "email" &&
node.arguments.length === 0
) {
context.report({
node,
message:
"Prefer using generator.email with the domain \"{ domain: 'example.com' }\".",
fix: function (fixer) {
return fixer.replaceText(
node,
'generator.email({ domain: "example.com" })'
)
},
})
}
},
}
},
},
}

2
lerna.json
View file

@ -1,5 +1,5 @@
{
"version": "2.21.2",
"version": "2.21.3",
"npmClient": "yarn",
"packages": [
"packages/*",

1
packages/backend-core/package.json
View file

@ -78,6 +78,7 @@
"jest-serial-runner": "1.2.1",
"pino-pretty": "10.0.0",
"pouchdb-adapter-memory": "7.2.2",
"testcontainers": "^10.7.2",
"timekeeper": "2.2.0",
"typescript": "5.2.2"
},

39
packages/backend-core/src/cache/base/index.ts vendored
View file

@ -23,6 +23,18 @@ export default class BaseCache {
return client.keys(pattern)
}
async exists(key: string, opts = { useTenancy: true }) {
key = opts.useTenancy ? generateTenantKey(key) : key
const client = await this.getClient()
return client.exists(key)
}
async scan(key: string, opts = { useTenancy: true }) {
key = opts.useTenancy ? generateTenantKey(key) : key
const client = await this.getClient()
return client.scan(key)
}
/**
* Read only from the cache.
*/
@ -32,6 +44,15 @@ export default class BaseCache {
return client.get(key)
}
/**
* Read only from the cache.
*/
async bulkGet<T>(keys: string[], opts = { useTenancy: true }) {
keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys
const client = await this.getClient()
return client.bulkGet<T>(keys)
}
/**
* Write to the cache.
*/
@ -74,6 +95,15 @@ export default class BaseCache {
return client.delete(key)
}
/**
* Remove from cache.
*/
async bulkDelete(keys: string[], opts = { useTenancy: true }) {
keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys
const client = await this.getClient()
return client.bulkDelete(keys)
}
/**
* Read from the cache. Write to the cache if not exists.
*/
@ -108,4 +138,13 @@ export default class BaseCache {
throw err
}
}
/**
* Delete the entry if the provided value matches the stored one.
*/
async deleteIfValue(key: string, value: any, opts = { useTenancy: true }) {
key = opts.useTenancy ? generateTenantKey(key) : key
const client = await this.getClient()
await client.deleteIfValue(key, value)
}
}

129
packages/backend-core/src/cache/docWritethrough.ts vendored
View file

@ -1,78 +1,77 @@
import BaseCache from "./base"
import { getDocWritethroughClient } from "../redis/init"
import { AnyDocument, Database } from "@budibase/types"
import { AnyDocument, Database, LockName, LockType } from "@budibase/types"
import { JobQueue, createQueue } from "../queue"
import * as dbUtils from "../db"
let CACHE: BaseCache | null = null
async function getCache() {
if (!CACHE) {
const client = await getDocWritethroughClient()
CACHE = new BaseCache(client)
}
return CACHE
}
import { string } from "yargs"
import { db } from ".."
import { locks } from "../redis"
import { Duration } from "../utils"
interface ProcessDocMessage {
dbName: string
docId: string
cacheKeyPrefix: string
data: Record<string, any>
}
export const docWritethroughProcessorQueue = createQueue<ProcessDocMessage>(
JobQueue.DOC_WRITETHROUGH_QUEUE
)
docWritethroughProcessorQueue.process(async message => {
await persistToDb(message.data)
console.log("DocWritethrough persisted", { data: message.data })
})
export async function persistToDb({
dbName,
docId,
cacheKeyPrefix,
}: {
dbName: string
docId: string
cacheKeyPrefix: string
}) {
const cache = await getCache()
const db = dbUtils.getDB(dbName)
let doc: AnyDocument | undefined
try {
doc = await db.get(docId)
} catch {
doc = { _id: docId }
class DocWritethroughProcessor {
init() {
docWritethroughProcessorQueue.process(async message => {
const result = await locks.doWithLock(
{
type: LockType.TRY_ONCE,
name: LockName.PERSIST_DOC_WRITETHROUGH,
resource: `${message.data.dbName}:${message.data.docId}`,
ttl: Duration.fromSeconds(60).toMs(),
},
async () => {
await this.persistToDb(message.data)
}
)
if (!result.executed) {
throw new Error(
`Error persisting docWritethrough message: ${message.id}`
)
}
})
return this
}
const keysToPersist = await cache.keys(`${cacheKeyPrefix}:data:*`)
for (const key of keysToPersist) {
const data = await cache.get(key, { useTenancy: false })
doc[data.key] = data.value
}
private async persistToDb({
dbName,
docId,
data,
}: {
dbName: string
docId: string
data: Record<string, any>
}) {
const db = dbUtils.getDB(dbName)
let doc: AnyDocument | undefined
try {
doc = await db.get(docId)
} catch {
doc = { _id: docId }
}
await db.put(doc)
for (const key of keysToPersist) {
await cache.delete(key, { useTenancy: false })
doc = { ...doc, ...data }
await db.put(doc)
}
}
export const processor = new DocWritethroughProcessor().init()
export class DocWritethrough {
private db: Database
private _docId: string
private writeRateMs: number
private cacheKeyPrefix: string
constructor(db: Database, docId: string, writeRateMs: number) {
constructor(db: Database, docId: string) {
this.db = db
this._docId = docId
this.writeRateMs = writeRateMs
this.cacheKeyPrefix = `${this.db.name}:${this.docId}`
}
get docId() {
@ -80,30 +79,10 @@ export class DocWritethrough {
}
async patch(data: Record<string, any>) {
const cache = await getCache()
await this.storeToCache(cache, data)
docWritethroughProcessorQueue.add(
{
dbName: this.db.name,
docId: this.docId,
cacheKeyPrefix: this.cacheKeyPrefix,
},
{
delay: this.writeRateMs,
jobId: this.cacheKeyPrefix,
removeOnFail: true,
removeOnComplete: true,
}
)
}
private async storeToCache(cache: BaseCache, data: Record<string, any>) {
data = Object.entries(data).reduce((acc, [key, value]) => {
acc[this.cacheKeyPrefix + ":data:" + key] = { key, value }
return acc
}, {} as Record<string, any>)
await cache.bulkStore(data, null)
await docWritethroughProcessorQueue.add({
dbName: this.db.name,
docId: this.docId,
data,
})
}
}

154
packages/backend-core/src/cache/tests/docWritethrough.spec.ts vendored
View file

@ -1,31 +1,21 @@
import tk from "timekeeper"
import _ from "lodash"
import { DBTestConfiguration, generator, structures } from "../../../tests"
import { getDB } from "../../db"
import _ from "lodash"
import {
DocWritethrough,
docWritethroughProcessorQueue,
} from "../docWritethrough"
import InMemoryQueue from "../../queue/inMemoryQueue"
const WRITE_RATE_MS = 1000
import InMemoryQueue from "../../queue/inMemoryQueue"
const initialTime = Date.now()
jest.useFakeTimers({
now: initialTime,
})
function resetTime() {
jest.setSystemTime(initialTime)
}
async function travelForward(ms: number) {
await jest.advanceTimersByTimeAsync(ms)
async function waitForQueueCompletion() {
const queue: InMemoryQueue = docWritethroughProcessorQueue as never
while (queue.hasRunningJobs()) {
await jest.runOnlyPendingTimersAsync()
}
await queue.waitForCompletion()
}
describe("docWritethrough", () => {
@ -45,30 +35,28 @@ describe("docWritethrough", () => {
}
beforeEach(async () => {
resetTime()
jest.clearAllMocks()
documentId = structures.uuid()
docWritethrough = new DocWritethrough(db, documentId, WRITE_RATE_MS)
docWritethrough = new DocWritethrough(db, documentId)
})
it("patching will not persist if timeout does not hit", async () => {
it("patching will not persist until the messages are persisted", async () => {
await config.doInTenant(async () => {
await travelForward(WRITE_RATE_MS)
await docWritethrough.patch(generatePatchObject(2))
await docWritethrough.patch(generatePatchObject(2))
await travelForward(WRITE_RATE_MS - 1)
expect(await db.exists(documentId)).toBe(false)
})
})
it("patching will persist if timeout hits", async () => {
it("patching will persist when the messages are persisted", async () => {
await config.doInTenant(async () => {
const patch1 = generatePatchObject(2)
const patch2 = generatePatchObject(2)
await docWritethrough.patch(patch1)
await docWritethrough.patch(patch2)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
// This will not be persisted
const patch3 = generatePatchObject(3)
@ -78,9 +66,9 @@ describe("docWritethrough", () => {
_id: documentId,
...patch1,
...patch2,
_rev: expect.stringMatching(/1-.+/),
createdAt: new Date(initialTime + WRITE_RATE_MS).toISOString(),
updatedAt: new Date(initialTime + WRITE_RATE_MS).toISOString(),
_rev: expect.stringMatching(/2-.+/),
createdAt: new Date(initialTime).toISOString(),
updatedAt: new Date(initialTime).toISOString(),
})
})
})
@ -92,12 +80,12 @@ describe("docWritethrough", () => {
await docWritethrough.patch(patch1)
await docWritethrough.patch(patch2)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
const patch3 = generatePatchObject(3)
await docWritethrough.patch(patch3)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect.objectContaining({
@ -115,12 +103,13 @@ describe("docWritethrough", () => {
const patch1 = generatePatchObject(2)
const patch2 = generatePatchObject(2)
await docWritethrough.patch(patch1)
await travelForward(WRITE_RATE_MS)
const date1 = new Date()
await waitForQueueCompletion()
await docWritethrough.patch(patch2)
await travelForward(WRITE_RATE_MS)
tk.travel(Date.now() + 100)
const date2 = new Date()
await waitForQueueCompletion()
expect(date1).not.toEqual(date2)
expect(await db.get(documentId)).toEqual(
@ -136,7 +125,7 @@ describe("docWritethrough", () => {
await config.doInTenant(async () => {
const patch1 = generatePatchObject(2)
await docWritethrough.patch(patch1)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
const patch2 = generatePatchObject(1)
await docWritethrough.patch(patch2)
@ -147,14 +136,14 @@ describe("docWritethrough", () => {
})
)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
const patch3 = {
...generatePatchObject(3),
[keyToOverride]: generator.word(),
}
await docWritethrough.patch(patch3)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect.objectContaining({
@ -170,8 +159,7 @@ describe("docWritethrough", () => {
await config.doInTenant(async () => {
const secondDocWritethrough = new DocWritethrough(
db,
structures.db.id(),
WRITE_RATE_MS
structures.db.id()
)
const doc1Patch = generatePatchObject(2)
@ -179,13 +167,13 @@ describe("docWritethrough", () => {
const doc2Patch = generatePatchObject(1)
await secondDocWritethrough.patch(doc2Patch)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
const doc1Patch2 = generatePatchObject(3)
await docWritethrough.patch(doc1Patch2)
const doc2Patch2 = generatePatchObject(3)
await secondDocWritethrough.patch(doc2Patch2)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
expect(await db.get(docWritethrough.docId)).toEqual(
expect.objectContaining({
@ -208,9 +196,7 @@ describe("docWritethrough", () => {
const initialPatch = generatePatchObject(5)
await docWritethrough.patch(initialPatch)
await travelForward(WRITE_RATE_MS)
await docWritethrough.patch({})
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect.objectContaining(initialPatch)
@ -218,10 +204,10 @@ describe("docWritethrough", () => {
await db.remove(await db.get(documentId))
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
const extraPatch = generatePatchObject(5)
await docWritethrough.patch(extraPatch)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect.objectContaining(extraPatch)
@ -232,58 +218,70 @@ describe("docWritethrough", () => {
})
})
it("concurrent calls will not cause multiple saves", async () => {
it("concurrent calls will not cause conflicts", async () => {
async function parallelPatch(count: number) {
await Promise.all(
Array.from({ length: count }).map(() =>
docWritethrough.patch(generatePatchObject(1))
)
const patches = Array.from({ length: count }).map(() =>
generatePatchObject(1)
)
}
await Promise.all(patches.map(p => docWritethrough.patch(p)))
const storeToCacheSpy = jest.spyOn(docWritethrough as any, "storeToCache")
return patches.reduce((acc, c) => {
acc = { ...acc, ...c }
return acc
}, {})
}
const queueMessageSpy = jest.spyOn(docWritethroughProcessorQueue, "add")
await config.doInTenant(async () => {
await parallelPatch(5)
expect(storeToCacheSpy).toBeCalledTimes(5)
expect(await db.exists(documentId)).toBe(false)
let patches = await parallelPatch(5)
expect(queueMessageSpy).toBeCalledTimes(5)
await travelForward(WRITE_RATE_MS)
await parallelPatch(40)
expect(storeToCacheSpy).toBeCalledTimes(45)
// Ideally we want to spy on persistToDb from ./docWritethrough, but due our barrel files configuration required quite of a complex setup.
// We are relying on the document being stored only once (otherwise we would have _rev updated)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect.objectContaining({
_id: documentId,
_rev: expect.stringMatching(/1-.+/),
})
expect.objectContaining(patches)
)
await parallelPatch(10)
patches = { ...patches, ...(await parallelPatch(40)) }
expect(queueMessageSpy).toBeCalledTimes(45)
expect(storeToCacheSpy).toBeCalledTimes(55)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect.objectContaining({
_id: documentId,
_rev: expect.stringMatching(/1-.+/),
})
expect.objectContaining(patches)
)
await travelForward(WRITE_RATE_MS)
patches = { ...patches, ...(await parallelPatch(10)) }
expect(queueMessageSpy).toBeCalledTimes(55)
await parallelPatch(5)
await travelForward(WRITE_RATE_MS)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect.objectContaining({
_id: documentId,
_rev: expect.stringMatching(/3-.+/),
})
expect.objectContaining(patches)
)
})
})
// This is not yet supported
it.skip("patches will execute in order", async () => {
let incrementalValue = 0
const keyToOverride = generator.word()
async function incrementalPatches(count: number) {
for (let i = 0; i < count; i++) {
await docWritethrough.patch({ [keyToOverride]: incrementalValue++ })
}
}
await config.doInTenant(async () => {
await incrementalPatches(5)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect.objectContaining({ [keyToOverride]: 5 })
)
await incrementalPatches(40)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect.objectContaining({ [keyToOverride]: 45 })
)
expect(storeToCacheSpy).toBeCalledTimes(60)
})
})
})

4
packages/backend-core/src/cache/user.ts vendored
View file

@ -6,7 +6,7 @@ import env from "../environment"
import * as accounts from "../accounts"
import { UserDB } from "../users"
import { sdk } from "@budibase/shared-core"
import { User } from "@budibase/types"
import { User, UserMetadata } from "@budibase/types"
const EXPIRY_SECONDS = 3600
@ -15,7 +15,7 @@ const EXPIRY_SECONDS = 3600
*/
async function populateFromDB(userId: string, tenantId: string) {
const db = tenancy.getTenantDB(tenantId)
const user = await db.get<any>(userId)
const user = await db.get<UserMetadata>(userId)
user.budibaseAccess = true
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
const account = await accounts.getAccount(user.email)

67
packages/backend-core/src/db/Replication.ts
View file

@ -1,66 +1,57 @@
import PouchDB from "pouchdb"
import { getPouchDB, closePouchDB } from "./couch"
import { DocumentType } from "../constants"
class Replication {
source: any
target: any
replication: any
source: PouchDB.Database
target: PouchDB.Database
/**
*
* @param source - the DB you want to replicate or rollback to
* @param target - the DB you want to replicate to, or rollback from
*/
constructor({ source, target }: any) {
constructor({ source, target }: { source: string; target: string }) {
this.source = getPouchDB(source)
this.target = getPouchDB(target)
}
close() {
return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
async close() {
await Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
}
promisify(operation: any, opts = {}) {
return new Promise(resolve => {
operation(this.target, opts)
.on("denied", function (err: any) {
replicate(opts: PouchDB.Replication.ReplicateOptions = {}) {
return new Promise<PouchDB.Replication.ReplicationResult<{}>>(resolve => {
this.source.replicate
.to(this.target, opts)
.on("denied", function (err) {
// a document failed to replicate (e.g. due to permissions)
throw new Error(`Denied: Document failed to replicate ${err}`)
})
.on("complete", function (info: any) {
.on("complete", function (info) {
return resolve(info)
})
.on("error", function (err: any) {
.on("error", function (err) {
throw new Error(`Replication Error: ${err}`)
})
})
}
/**
* Two way replication operation, intended to be promise based.
* @param opts - PouchDB replication options
*/
sync(opts = {}) {
this.replication = this.promisify(this.source.sync, opts)
return this.replication
}
appReplicateOpts(
opts: PouchDB.Replication.ReplicateOptions = {}
): PouchDB.Replication.ReplicateOptions {
if (typeof opts.filter === "string") {
return opts
}
/**
* One way replication operation, intended to be promise based.
* @param opts - PouchDB replication options
*/
replicate(opts = {}) {
this.replication = this.promisify(this.source.replicate.to, opts)
return this.replication
}
const filter = opts.filter
delete opts.filter
appReplicateOpts() {
return {
filter: (doc: any) => {
...opts,
filter: (doc: any, params: any) => {
if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) {
return false
}
return doc._id !== DocumentType.APP_METADATA
if (doc._id === DocumentType.APP_METADATA) {
return false
}
return filter ? filter(doc, params) : true
},
}
}
@ -75,10 +66,6 @@ class Replication {
// take the opportunity to remove deleted tombstones
await this.replicate()
}
cancel() {
this.replication.cancel()
}
}
export default Replication

23
packages/backend-core/src/queue/inMemoryQueue.ts
View file

@ -1,8 +1,9 @@
import events from "events"
import { timeout } from "../utils"
import { newid, timeout } from "../utils"
import { Queue, QueueOptions, JobOptions } from "./queue"
interface JobMessage {
id: string
timestamp: number
queue: string
data: any
@ -20,6 +21,7 @@ interface JobMessage {
*/
function newJob(queue: string, message: any, opts?: JobOptions): JobMessage {
return {
id: newid(),
timestamp: Date.now(),
queue: queue,
data: message,
@ -74,8 +76,23 @@ class InMemoryQueue implements Partial<Queue> {
let msg = this._messages.shift()
let resp = func(msg)
async function retryFunc(fnc: any) {
try {
await fnc
} catch (e: any) {
await new Promise<void>(r => setTimeout(() => r(), 50))
await retryFunc(func(msg))
}
}
if (resp.then != null) {
await resp
try {
await retryFunc(resp)
} catch (e: any) {
console.error(e)
}
}
this._runCount++
const jobId = msg?.opts?.jobId?.toString()
@ -173,7 +190,7 @@ class InMemoryQueue implements Partial<Queue> {
async waitForCompletion() {
do {
await timeout(50)
} while (this.hasRunningJobs)
} while (this.hasRunningJobs())
}
hasRunningJobs() {

46
packages/backend-core/src/redis/redis.ts
View file

@ -291,23 +291,16 @@ class RedisWrapper {
return acc
}, {} as Record<string, any>)
const luaScript = `
for i, key in ipairs(KEYS) do
redis.call('MSET', key, ARGV[i])
${
expirySeconds !== null
? `redis.call('EXPIRE', key, ARGV[#ARGV])`
: ""
}
end
`
const keys = Object.keys(dataToStore)
const values = Object.values(dataToStore)
const pipeline = client.pipeline()
pipeline.mset(dataToStore)
if (expirySeconds !== null) {
values.push(expirySeconds)
for (const key of Object.keys(dataToStore)) {
pipeline.expire(key, expirySeconds)
}
}
await client.eval(luaScript, keys.length, ...keys, ...values)
await pipeline.exec()
}
async getTTL(key: string) {
@ -327,10 +320,35 @@ class RedisWrapper {
await this.getClient().del(addDbPrefix(db, key))
}
async bulkDelete(keys: string[]) {
const db = this._db
await this.getClient().del(keys.map(key => addDbPrefix(db, key)))
}
async clear() {
let items = await this.scan()
await Promise.all(items.map((obj: any) => this.delete(obj.key)))
}
async increment(key: string) {
const result = await this.getClient().incr(addDbPrefix(this._db, key))
if (isNaN(result)) {
throw new Error(`Redis ${key} does not contain a number`)
}
return result
}
async deleteIfValue(key: string, value: any) {
const client = this.getClient()
const luaScript = `
if redis.call('GET', KEYS[1]) == ARGV[1] then
redis.call('DEL', KEYS[1])
end
`
await client.eval(luaScript, 1, addDbPrefix(this._db, key), value)
}
}
export default RedisWrapper

104
packages/backend-core/src/redis/tests/redis.spec.ts
View file

@ -1,8 +1,28 @@
import { GenericContainer, StartedTestContainer } from "testcontainers"
import { generator, structures } from "../../../tests"
import RedisWrapper from "../redis"
import { env } from "../.."
jest.setTimeout(30000)
describe("redis", () => {
let redis: RedisWrapper
let container: StartedTestContainer
beforeAll(async () => {
const container = await new GenericContainer("redis")
.withExposedPorts(6379)
.start()
env._set(
"REDIS_URL",
`${container.getHost()}:${container.getMappedPort(6379)}`
)
env._set("MOCK_REDIS", 0)
env._set("REDIS_PASSWORD", 0)
})
afterAll(() => container?.stop())
beforeEach(async () => {
redis = new RedisWrapper(structures.db.id())
@ -107,4 +127,88 @@ describe("redis", () => {
expect(await redis.keys("*")).toHaveLength(0)
})
})
describe("increment", () => {
it("can increment on a new key", async () => {
const key = structures.uuid()
const result = await redis.increment(key)
expect(result).toBe(1)
})
it("can increment multiple times", async () => {
const key = structures.uuid()
const results = [
await redis.increment(key),
await redis.increment(key),
await redis.increment(key),
await redis.increment(key),
await redis.increment(key),
]
expect(results).toEqual([1, 2, 3, 4, 5])
})
it("can increment on a new key", async () => {
const key1 = structures.uuid()
const key2 = structures.uuid()
const result1 = await redis.increment(key1)
expect(result1).toBe(1)
const result2 = await redis.increment(key2)
expect(result2).toBe(1)
})
it("can increment multiple times in parallel", async () => {
const key = structures.uuid()
const results = await Promise.all(
Array.from({ length: 100 }).map(() => redis.increment(key))
)
expect(results).toHaveLength(100)
expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1))
})
it("can increment existing set keys", async () => {
const key = structures.uuid()
await redis.store(key, 70)
await redis.increment(key)
const result = await redis.increment(key)
expect(result).toBe(72)
})
it.each([
generator.word(),
generator.bool(),
{ [generator.word()]: generator.word() },
])("cannot increment if the store value is not a number", async value => {
const key = structures.uuid()
await redis.store(key, value)
await expect(redis.increment(key)).rejects.toThrowError(
"ERR value is not an integer or out of range"
)
})
})
describe("deleteIfValue", () => {
it("can delete if the value matches", async () => {
const key = structures.uuid()
const value = generator.word()
await redis.store(key, value)
await redis.deleteIfValue(key, value)
expect(await redis.get(key)).toBeNull()
})
it("will not delete if the value does not matches", async () => {
const key = structures.uuid()
const value = generator.word()
await redis.store(key, value)
await redis.deleteIfValue(key, generator.word())
expect(await redis.get(key)).toEqual(value)
})
})
})

5
packages/backend-core/src/security/roles.ts
View file

@ -101,10 +101,7 @@ export function getBuiltinRole(roleId: string): Role | undefined {
/**
* Works through the inheritance ranks to see how far up the builtin stack this ID is.
*/
export function builtinRoleToNumber(id?: string) {
if (!id) {
return 0
}
export function builtinRoleToNumber(id: string) {
const builtins = getBuiltinRoles()
const MAX = Object.values(builtins).length + 1
if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) {

2
packages/backend-core/tests/core/utilities/structures/accounts.ts
View file

@ -18,7 +18,7 @@ export const account = (partial: Partial<Account> = {}): Account => {
return {
accountId: uuid(),
tenantId: generator.word(),
email: generator.email(),
email: generator.email({ domain: "example.com" }),
tenantName: generator.word(),
hosting: Hosting.SELF,
createdAt: Date.now(),

2
packages/backend-core/tests/core/utilities/structures/scim.ts
View file

@ -13,7 +13,7 @@ interface CreateUserRequestFields {
export function createUserRequest(userData?: Partial<CreateUserRequestFields>) {
const defaultValues = {
externalId: uuid(),
email: generator.email(),
email: `${uuid()}@example.com`,
firstName: generator.first(),
lastName: generator.last(),
username: generator.name(),

5
packages/bbui/src/Actions/position_dropdown.js
View file

@ -35,7 +35,10 @@ export default function positionDropdown(element, opts) {
}
if (typeof customUpdate === "function") {
styles = customUpdate(anchorBounds, elementBounds, styles)
styles = customUpdate(anchorBounds, elementBounds, {
...styles,
offset: opts.offset,
})
} else {
// Determine vertical styles
if (align === "right-outside") {

8
packages/builder/src/analytics/index.js
View file

@ -9,13 +9,17 @@ const intercom = new IntercomClient(process.env.INTERCOM_TOKEN)
class AnalyticsHub {
constructor() {
this.clients = [posthog, intercom]
this.initialised = false
}
async activate() {
// Check analytics are enabled
const analyticsStatus = await API.getAnalyticsStatus()
if (analyticsStatus.enabled) {
this.clients.forEach(client => client.init())
if (analyticsStatus.enabled && !this.initialised) {
this.clients.forEach(client => {
client.init()
})
this.initialised = true
}
}

9
packages/builder/src/components/deploy/AppActions.svelte
View file

@ -164,9 +164,10 @@
</div>
{/if}
<TourWrap
tourStepKey={$builderStore.onboarding
? TOUR_STEP_KEYS.BUILDER_USER_MANAGEMENT
: TOUR_STEP_KEYS.FEATURE_USER_MANAGEMENT}
stepKeys={[
TOUR_STEP_KEYS.BUILDER_USER_MANAGEMENT,
TOUR_STEP_KEYS.FEATURE_USER_MANAGEMENT,
]}
>
<div class="app-action-button users">
<div class="app-action" id="builder-app-users-button">
@ -209,7 +210,7 @@
<div bind:this={appActionPopoverAnchor}>
<div class="app-action">
<Icon name={isPublished ? "GlobeCheck" : "GlobeStrike"} />
<TourWrap tourStepKey={TOUR_STEP_KEYS.BUILDER_APP_PUBLISH}>
<TourWrap stepKeys={[TOUR_STEP_KEYS.BUILDER_APP_PUBLISH]}>
<span class="publish-open" id="builder-app-publish-button">
Publish
<Icon

2
packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
View file

@ -1,5 +1,5 @@
<script>
import EditComponentPopover from "../EditComponentPopover.svelte"
import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
import { Icon } from "@budibase/bbui"
import { runtimeToReadableBinding } from "dataBinding"
import { isJSBinding } from "@budibase/string-templates"

21
packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte → packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte
View file

@ -3,6 +3,7 @@
import { componentStore } from "stores/builder"
import { cloneDeep } from "lodash/fp"
import { createEventDispatcher, getContext } from "svelte"
import { customPositionHandler } from "."
import ComponentSettingsSection from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
export let anchor
@ -54,25 +55,6 @@
dispatch("change", nestedComponentInstance)
}
const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
let { left, top } = cfg
let percentageOffset = 30
// left-outside
left = anchorBounds.left - eleBounds.width - 18
// shift up from the anchor, if space allows
let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
let defaultTop = anchorBounds.top - offsetPos
if (window.innerHeight - defaultTop < eleBounds.height) {
top = window.innerHeight - eleBounds.height - 5
} else {
top = anchorBounds.top - offsetPos
}
return { ...cfg, left, top }
}
</script>
<Icon
@ -104,6 +86,7 @@
showPopover={drawers.length === 0}
clickOutsideOverride={drawers.length > 0}
maxHeight={600}
offset={18}
handlePostionUpdate={customPositionHandler}
>
<span class="popover-wrap">

18
packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js Normal file
View file

@ -0,0 +1,18 @@
export const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
let { left, top, offset } = cfg
let percentageOffset = 30
// left-outside
left = anchorBounds.left - eleBounds.width - (offset || 5)
// shift up from the anchor, if space allows
let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
let defaultTop = anchorBounds.top - offsetPos
if (window.innerHeight - defaultTop < eleBounds.height) {
top = window.innerHeight - eleBounds.height - 5
} else {
top = anchorBounds.top - offsetPos
}
return { ...cfg, left, top }
}

2
packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
View file

@ -1,5 +1,5 @@
<script>
import EditComponentPopover from "../EditComponentPopover.svelte"
import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
import { Toggle, Icon } from "@budibase/bbui"
import { createEventDispatcher } from "svelte"
import { cloneDeep } from "lodash/fp"

4
packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte
View file

@ -52,8 +52,8 @@
_id: Helpers.uuid(),
_component: componentType,
_instanceName: `Step ${currentStep + 1}`,
title: stepSettings.title ?? defaults.title,
buttons: stepSettings.buttons || defaults.buttons,
title: stepSettings.title ?? defaults?.title,
buttons: stepSettings.buttons || defaults?.buttons,
fields: stepSettings.fields,
desc: stepSettings.desc,

2
packages/builder/src/components/design/settings/controls/GridColumnConfiguration/FieldSetting.svelte
View file

@ -1,5 +1,5 @@
<script>
import EditComponentPopover from "../EditComponentPopover.svelte"
import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
import { Toggle, Icon } from "@budibase/bbui"
import { createEventDispatcher } from "svelte"
import { cloneDeep } from "lodash/fp"

2
packages/builder/src/components/design/settings/controls/GridColumnConfiguration/PrimaryColumnFieldSetting.svelte
View file

@ -1,5 +1,5 @@
<script>
import EditComponentPopover from "../EditComponentPopover.svelte"
import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
import { Icon } from "@budibase/bbui"
import { setContext } from "svelte"
import { writable } from "svelte/store"

17
packages/builder/src/components/design/settings/controls/IconSelect/IconSelect.svelte
View file

@ -139,10 +139,22 @@
{/each}
</div>
<div class="search-input">
<div class="input-wrapper">
<Input bind:value={searchTerm} thin placeholder="Search Icon" />
<div class="input-wrapper" style={`width: ${value ? "425" : "510"}px`}>
<Input
bind:value={searchTerm}
on:keyup={event => {
if (event.key === "Enter") {
searchForIcon()
}
}}
thin
placeholder="Search Icon"
/>
</div>
<Button secondary on:click={searchForIcon}>Search</Button>
{#if value}
<Button primary on:click={() => (value = null)}>Clear</Button>
{/if}
</div>
<div class="page-area">
<div class="pager">
@ -239,6 +251,7 @@
flex-flow: row nowrap;
width: 100%;
padding-right: 15px;
gap: 10px;
}
.input-wrapper {
width: 510px;

31
packages/builder/src/components/design/settings/controls/PropertyControl.svelte
View file

@ -20,17 +20,23 @@
export let bindings = []
export let componentBindings = []
export let nested = false
export let highlighted = false
export let propertyFocus = false
export let info = null
export let disableBindings = false
export let wide
$: nullishValue = value == null || value === ""
let highlightType
$: highlightedProp = $builderStore.highlightedSetting
$: allBindings = getAllBindings(bindings, componentBindings, nested)
$: safeValue = getSafeValue(value, defaultValue, allBindings)
$: replaceBindings = val => readableToRuntimeBinding(allBindings, val)
$: if (!Array.isArray(value)) {
highlightType =
highlightedProp?.key === key ? `highlighted-${highlightedProp?.type}` : ""
}
const getAllBindings = (bindings, componentBindings, nested) => {
if (!nested) {
return bindings
@ -71,16 +77,17 @@
}
onDestroy(() => {
if (highlighted) {
if (highlightedProp) {
builderStore.highlightSetting(null)
}
})
</script>
<div
class="property-control"
id={`${key}-prop-control-wrap`}
class={`property-control ${highlightType}`}
class:wide={!label || labelHidden || wide === true}
class:highlighted={highlighted && nullishValue}
class:highlighted={highlightType}
class:property-focus={propertyFocus}
>
{#if label && !labelHidden}
@ -115,6 +122,16 @@
</div>
<style>
.property-control.highlighted.highlighted-info {
border-color: var(--spectrum-semantic-informative-color-background);
}
.property-control.highlighted.highlighted-error {
border-color: var(--spectrum-global-color-static-red-600);
}
.property-control.highlighted.highlighted-warning {
border-color: var(--spectrum-global-color-static-orange-700);
}
.property-control {
position: relative;
display: grid;
@ -132,6 +149,10 @@
.property-control.highlighted {
background: var(--spectrum-global-color-gray-300);
border-color: var(--spectrum-global-color-static-red-600);
margin-top: -3.5px;
margin-bottom: -3.5px;
padding-bottom: 3.5px;
padding-top: 3.5px;
}
.property-control.property-focus :global(input) {

25
packages/builder/src/components/portal/onboarding/TourPopover.svelte
View file

@ -1,6 +1,6 @@
<script>
import { Popover, Layout, Heading, Body, Button, Link } from "@budibase/bbui"
import { TOURS } from "./tours.js"
import { TOURS, getCurrentStepIdx } from "./tours.js"
import { goto, layout, isActive } from "@roxi/routify"
import { builderStore } from "stores/builder"
@ -20,6 +20,13 @@
const updateTourStep = (targetStepKey, tourKey) => {
if (!tourKey) {
tourSteps = null
tourStepIdx = null
lastStep = null
tourStep = null
popoverAnchor = null
popover = null
skipping = false
return
}
if (!tourSteps?.length) {
@ -78,16 +85,6 @@
}
}
}
const getCurrentStepIdx = (steps, tourStepKey) => {
if (!steps?.length) {
return
}
if (steps?.length && !tourStepKey) {
return 0
}
return steps.findIndex(step => step.id === tourStepKey)
}
</script>
{#if tourKey}
@ -98,7 +95,9 @@
anchor={popoverAnchor}
maxWidth={300}
dismissible={false}
offset={15}
offset={12}
handlePostionUpdate={tourStep?.positionHandler}
customZindex={3}
>
<div class="tour-content">
<Layout noPadding gap="M">
@ -119,7 +118,7 @@
</Body>
<div class="tour-footer">
<div class="tour-navigation">
{#if typeof tourOnSkip === "function"}
{#if typeof tourOnSkip === "function" && !lastStep}
<Link
secondary
quiet

54
packages/builder/src/components/portal/onboarding/TourWrap.svelte
View file

@ -1,39 +1,63 @@
<script>
import { tourHandler } from "./tourHandler"
import { TOURS } from "./tours"
import { TOURSBYSTEP, TOURS, getCurrentStepIdx } from "./tours"
import { onMount, onDestroy } from "svelte"
import { builderStore } from "stores/builder"
export let tourStepKey
export let stepKeys = []
let currentTourStep
let ready = false
let registered = false
let handler
let registered = {}
const registerTourNode = (tourKey, stepKey) => {
if (ready && !registered && tourKey) {
currentTourStep = TOURS[tourKey].steps.find(step => step.id === stepKey)
if (!currentTourStep) {
return
const step = TOURSBYSTEP[stepKey]
if (ready && step && !registered[stepKey] && step?.tour === tourKey) {
const elem = document.querySelector(step.query)
registered[stepKey] = tourHandler(elem, stepKey)
}
}
const scrollToStep = () => {
let tourStepIdx = getCurrentStepIdx(
TOURS[tourKeyWatch]?.steps,
tourStepKeyWatch
)
let currentStep = TOURS[tourKeyWatch]?.steps?.[tourStepIdx]
if (currentStep?.scrollIntoView) {
let currentNode = $builderStore.tourNodes?.[currentStep.id]
if (currentNode) {
currentNode.scrollIntoView({ behavior: "smooth", block: "center" })
}
const elem = document.querySelector(currentTourStep.query)
handler = tourHandler(elem, stepKey)
registered = true
}
}
$: tourKeyWatch = $builderStore.tourKey
$: registerTourNode(tourKeyWatch, tourStepKey, ready)
$: tourStepKeyWatch = $builderStore.tourStepKey
$: if (tourKeyWatch || stepKeys || ready) {
stepKeys.forEach(tourStepKey => {
registerTourNode(tourKeyWatch, tourStepKey)
})
}
$: scrollToStep(tourKeyWatch, tourStepKeyWatch)
onMount(() => {
ready = true
})
onDestroy(() => {
if (handler) {
Object.entries(registered).forEach(entry => {
const handler = entry[1]
const stepKey = entry[0]
// Run step destroy, de-register nodes in the builderStore and local cache
handler.destroy()
}
delete registered[stepKey]
// Check if the step is part of an active tour. End the tour if that is the case
const step = TOURSBYSTEP[stepKey]
if (step.tour === tourKeyWatch) {
builderStore.setTour()
}
})
})
</script>

9
packages/builder/src/components/portal/onboarding/steps/NewFormSteps.svelte Normal file
View file

@ -0,0 +1,9 @@
<div>
When faced with a sizable form, consider implementing a multi-step approach to
enhance user experience.
<p>
Breaking the form into multiple steps can significantly improve usability by
making the process more digestible for your users.
</p>
</div>

17
packages/builder/src/components/portal/onboarding/steps/NewViewUpdateFormRowId.svelte Normal file
View file

@ -0,0 +1,17 @@
<div>
You can use bindings to set the Row ID on your form.
<p>
This will allow you to pull the correct information into your form and allow
you to update!
</p>
<a href="https://docs.budibase.com/docs/form-block" target="_blank">
How to pass a row ID using bindings
</a>
</div>
<style>
a {
color: inherit;
text-decoration: underline;
}
</style>

2
packages/builder/src/components/portal/onboarding/steps/index.js
View file

@ -1,3 +1,5 @@
export { default as OnboardingData } from "./OnboardingData.svelte"
export { default as OnboardingDesign } from "./OnboardingDesign.svelte"
export { default as OnboardingPublish } from "./OnboardingPublish.svelte"
export { default as NewViewUpdateFormRowId } from "./NewViewUpdateFormRowId.svelte"
export { default as NewFormSteps } from "./NewFormSteps.svelte"

119
packages/builder/src/components/portal/onboarding/tours.js
View file

@ -2,8 +2,15 @@ import { get } from "svelte/store"
import { builderStore } from "stores/builder"
import { auth } from "stores/portal"
import analytics from "analytics"
import { OnboardingData, OnboardingDesign, OnboardingPublish } from "./steps"
import {
OnboardingData,
OnboardingDesign,
OnboardingPublish,
NewViewUpdateFormRowId,
NewFormSteps,
} from "./steps"
import { API } from "api"
import { customPositionHandler } from "components/design/settings/controls/EditComponentPopover"
const ONBOARDING_EVENT_PREFIX = "onboarding"
@ -14,11 +21,26 @@ export const TOUR_STEP_KEYS = {
BUILDER_USER_MANAGEMENT: "builder-user-management",
BUILDER_AUTOMATION_SECTION: "builder-automation-section",
FEATURE_USER_MANAGEMENT: "feature-user-management",
BUILDER_FORM_CREATE_STEPS: "builder-form-create-steps",
BUILDER_FORM_VIEW_UPDATE_STEPS: "builder-form-view-update-steps",
BUILDER_FORM_ROW_ID: "builder-form-row-id",
}
export const TOUR_KEYS = {
TOUR_BUILDER_ONBOARDING: "builder-onboarding",
FEATURE_ONBOARDING: "feature-onboarding",
BUILDER_FORM_CREATE: "builder-form-create",
BUILDER_FORM_VIEW_UPDATE: "builder-form-view-update",
}
export const getCurrentStepIdx = (steps, tourStepKey) => {
if (!steps?.length) {
return
}
if (steps?.length && !tourStepKey) {
return 0
}
return steps.findIndex(step => step.id === tourStepKey)
}
const endUserOnboarding = async ({ skipped = false } = {}) => {
@ -37,13 +59,8 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
// Update the cached user
await auth.getSelf()
builderStore.update(state => ({
...state,
tourNodes: null,
tourKey: null,
tourStepKey: null,
onboarding: false,
}))
builderStore.endBuilderOnboarding()
builderStore.setTour()
} catch (e) {
console.error("Onboarding failed", e)
return false
@ -52,9 +69,29 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
}
}
const tourEvent = eventKey => {
const endTour = async ({ key, skipped = false } = {}) => {
const { tours = {} } = get(auth).user
tours[key] = new Date().toISOString()
await API.updateSelf({
tours,
})
if (skipped) {
tourEvent(key, skipped)
}
// Update the cached user
await auth.getSelf()
// Reset tour state
builderStore.setTour()
}
const tourEvent = (eventKey, skipped) => {
analytics.captureEvent(`${ONBOARDING_EVENT_PREFIX}:${eventKey}`, {
eventSource: EventSource.PORTAL,
skipped,
})
}
@ -135,7 +172,71 @@ const getTours = () => {
},
],
},
[TOUR_KEYS.BUILDER_FORM_CREATE]: {
steps: [
{
id: TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS,
title: "Add multiple steps",
layout: NewFormSteps,
query: "#steps-prop-control-wrap",
onComplete: () => {
builderStore.highlightSetting()
endTour({ key: TOUR_KEYS.BUILDER_FORM_CREATE })
},
onLoad: () => {
tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS)
builderStore.highlightSetting("steps", "info")
},
positionHandler: customPositionHandler,
align: "left-outside",
},
],
},
[TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE]: {
steps: [
{
id: TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID,
title: "Add row ID to update a row",
layout: NewViewUpdateFormRowId,
query: "#rowId-prop-control-wrap",
onLoad: () => {
tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID)
builderStore.highlightSetting("rowId", "info")
},
positionHandler: customPositionHandler,
align: "left-outside",
},
{
id: TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS,
title: "Add multiple steps",
layout: NewFormSteps,
query: "#steps-prop-control-wrap",
onComplete: () => {
builderStore.highlightSetting()
endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE })
},
onLoad: () => {
tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS)
builderStore.highlightSetting("steps", "info")
},
positionHandler: customPositionHandler,
align: "left-outside",
scrollIntoView: true,
},
],
onSkip: async () => {
builderStore.highlightSetting()
endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE, skipped: true })
},
},
}
}
export const TOURS = getTours()
export const TOURSBYSTEP = Object.keys(TOURS).reduce((acc, tour) => {
TOURS[tour].steps.forEach(element => {
acc[element.id] = element
acc[element.id]["tour"] = tour
})
return acc
}, {})

2
packages/builder/src/dataBinding.js
View file

@ -1146,7 +1146,7 @@ export const getAllStateVariables = () => {
"@budibase/standard-components/multistepformblockstep"
)
steps.forEach(step => {
steps?.forEach(step => {
parseComponentSettings(stepDefinition, step)
})
})

4
packages/builder/src/pages/builder/app/[application]/_layout.svelte
View file

@ -96,7 +96,7 @@
const release_date = new Date("2023-03-01T00:00:00.000Z")
const onboarded = new Date($auth.user?.onboardedAt)
if (onboarded < release_date) {
builderStore.startTour(TOUR_KEYS.FEATURE_ONBOARDING)
builderStore.setTour(TOUR_KEYS.FEATURE_ONBOARDING)
}
}
}
@ -144,7 +144,7 @@
</span>
<Tabs {selected} size="M">
{#each $layout.children as { path, title }}
<TourWrap tourStepKey={`builder-${title}-section`}>
<TourWrap stepKeys={[`builder-${title}-section`]}>
<Tab
quiet
selected={$isActive(path)}

31
packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte
View file

@ -16,6 +16,14 @@
} from "dataBinding"
import { ActionButton, notifications } from "@budibase/bbui"
import { capitalise } from "helpers"
import TourWrap from "components/portal/onboarding/TourWrap.svelte"
import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js"
const {
BUILDER_FORM_CREATE_STEPS,
BUILDER_FORM_VIEW_UPDATE_STEPS,
BUILDER_FORM_ROW_ID,
} = TOUR_STEP_KEYS
const onUpdateName = async value => {
try {
@ -46,7 +54,6 @@
$: id = $selectedComponent?._id
$: id, (section = tabs[0])
$: componentName = getComponentName(componentInstance)
</script>
@ -92,13 +99,21 @@
</div>
</span>
{#if section == "settings"}
<ComponentSettingsSection
{componentInstance}
{componentDefinition}
{bindings}
{componentBindings}
{isScreen}
/>
<TourWrap
stepKeys={[
BUILDER_FORM_CREATE_STEPS,
BUILDER_FORM_VIEW_UPDATE_STEPS,
BUILDER_FORM_ROW_ID,
]}
>
<ComponentSettingsSection
{componentInstance}
{componentDefinition}
{bindings}
{componentBindings}
{isScreen}
/>
</TourWrap>
{/if}
{#if section == "styles"}
<DesignSection

6
packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
View file

@ -1,7 +1,7 @@
<script>
import { helpers } from "@budibase/shared-core"
import { DetailSummary, notifications } from "@budibase/bbui"
import { componentStore } from "stores/builder"
import { componentStore, builderStore } from "stores/builder"
import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
import ResetFieldsButton from "components/design/settings/controls/ResetFieldsButton.svelte"
import EjectBlockButton from "components/design/settings/controls/EjectBlockButton.svelte"
@ -177,9 +177,7 @@
defaultValue={setting.defaultValue}
nested={setting.nested}
onChange={val => updateSetting(setting, val)}
highlighted={$componentStore.highlightedSettingKey ===
setting.key}
propertyFocus={$componentStore.propertyFocus === setting.key}
propertyFocus={$builderStore.propertyFocus === setting.key}
info={setting.info}
disableBindings={setting.disableBindings}
props={{

2
packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
View file

@ -174,7 +174,7 @@
} else if (type === "request-add-component") {
toggleAddComponent()
} else if (type === "highlight-setting") {
builderStore.highlightSetting(data.setting)
builderStore.highlightSetting(data.setting, "error")
// Also scroll setting into view
const selector = `#${data.setting}-prop-control`

130
packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
View file

@ -3,13 +3,23 @@
import DatasourceModal from "./DatasourceModal.svelte"
import ScreenRoleModal from "./ScreenRoleModal.svelte"
import sanitizeUrl from "helpers/sanitizeUrl"
import FormTypeModal from "./FormTypeModal.svelte"
import { Modal, notifications } from "@budibase/bbui"
import { screenStore, navigationStore, tables } from "stores/builder"
import {
screenStore,
navigationStore,
tables,
builderStore,
} from "stores/builder"
import { auth } from "stores/portal"
import { get } from "svelte/store"
import getTemplates from "templates"
import { Roles } from "constants/backend"
import { capitalise } from "helpers"
import { goto } from "@roxi/routify"
import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
import formScreen from "templates/formScreen"
import rowListScreen from "templates/rowListScreen"
let mode
let pendingScreen
@ -18,12 +28,18 @@
let screenDetailsModal
let datasourceModal
let screenAccessRoleModal
let formTypeModal
// Cache variables for workflow
let screenAccessRole = Roles.BASIC
let selectedTemplates = null
let templates = null
let screens = null
let selectedDatasources = null
let blankScreenUrl = null
let screenMode = null
let formType = null
// Creates an array of screens, checking and sanitising their URLs
const createScreens = async ({ screens, screenAccessRole }) => {
@ -32,7 +48,7 @@
}
try {
let screenId
let createdScreens = []
for (let screen of screens) {
// Check we aren't clashing with an existing URL
@ -56,7 +72,7 @@
// Create the screen
const response = await screenStore.save(screen)
screenId = response._id
createdScreens.push(response)
// Add link in layout. We only ever actually create 1 screen now, even
// for autoscreens, so it's always safe to do this.
@ -66,9 +82,7 @@
)
}
// Go to new screen
$goto(`./${screenId}`)
screenStore.select(screenId)
return createdScreens
} catch (error) {
console.error(error)
notifications.error("Error creating screens")
@ -104,13 +118,16 @@
// Handler for NewScreenModal
export const show = newMode => {
mode = newMode
selectedTemplates = null
templates = null
screens = null
selectedDatasources = null
blankScreenUrl = null
screenMode = mode
pendingScreen = null
screenAccessRole = Roles.BASIC
formType = null
if (mode === "table" || mode === "grid") {
if (mode === "table" || mode === "grid" || mode === "form") {
datasourceModal.show()
} else if (mode === "blank") {
let templates = getTemplates($tables.list)
@ -125,19 +142,26 @@
}
// Handler for DatasourceModal confirmation, move to screen access select
const confirmScreenDatasources = async ({ templates }) => {
selectedTemplates = templates
screenAccessRoleModal.show()
const confirmScreenDatasources = async ({ datasources }) => {
selectedDatasources = datasources
if (screenMode === "form") {
formTypeModal.show()
} else {
screenAccessRoleModal.show()
}
}
// Handler for Datasource Screen Creation
const completeDatasourceScreenCreation = async () => {
const screens = selectedTemplates.map(template => {
templates = rowListScreen(selectedDatasources, mode)
const screens = templates.map(template => {
let screenTemplate = template.create()
screenTemplate.autoTableId = template.resourceId
return screenTemplate
})
await createScreens({ screens, screenAccessRole })
const createdScreens = await createScreens({ screens, screenAccessRole })
loadNewScreen(createdScreens)
}
const confirmScreenBlank = async ({ screenUrl }) => {
@ -154,7 +178,54 @@
return
}
pendingScreen.routing.route = screenUrl
await createScreens({ screens: [pendingScreen], screenAccessRole })
const createdScreens = await createScreens({
screens: [pendingScreen],
screenAccessRole,
})
loadNewScreen(createdScreens)
}
const onConfirmFormType = () => {
screenAccessRoleModal.show()
}
const loadNewScreen = createdScreens => {
const lastScreen = createdScreens.slice(-1)[0]
// Go to new screen
if (lastScreen?.props?._children.length) {
// Focus on the main component for the streen type
const mainComponent = lastScreen?.props?._children?.[0]._id
$goto(`./${lastScreen._id}/${mainComponent}`)
} else {
$goto(`./${lastScreen._id}`)
}
screenStore.select(lastScreen._id)
}
const confirmFormScreenCreation = async () => {
templates = formScreen(selectedDatasources, { actionType: formType })
screens = templates.map(template => {
let screenTemplate = template.create()
return screenTemplate
})
const createdScreens = await createScreens({ screens, screenAccessRole })
if (formType === "Update" || formType === "Create") {
const associatedTour =
formType === "Update"
? TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE
: TOUR_KEYS.BUILDER_FORM_CREATE
const tourRequired = !$auth?.user?.tours?.[associatedTour]
if (tourRequired) {
builderStore.setTour(associatedTour)
}
}
// Go to new screen
loadNewScreen(createdScreens)
}
// Submit screen config for creation.
@ -164,6 +235,8 @@
screenUrl: blankScreenUrl,
screenAccessRole,
})
} else if (screenMode === "form") {
confirmFormScreenCreation()
} else {
completeDatasourceScreenCreation()
}
@ -179,19 +252,18 @@
</script>
<Modal bind:this={datasourceModal} autoFocus={false}>
<DatasourceModal
{mode}
onConfirm={confirmScreenDatasources}
initialScreens={!selectedTemplates ? [] : [...selectedTemplates]}
/>
<DatasourceModal {mode} onConfirm={confirmScreenDatasources} />
</Modal>
<Modal bind:this={screenAccessRoleModal}>
<ScreenRoleModal
onConfirm={confirmScreenCreation}
onCancel={roleSelectBack}
onConfirm={() => {
confirmScreenCreation()
}}
bind:screenAccessRole
onCancel={roleSelectBack}
screenUrl={blankScreenUrl}
confirmText={screenMode === "form" ? "Confirm" : "Done"}
/>
</Modal>
@ -201,3 +273,17 @@
initialUrl={blankScreenUrl}
/>
</Modal>
<Modal bind:this={formTypeModal}>
<FormTypeModal
onConfirm={onConfirmFormType}
onCancel={() => {
formTypeModal.hide()
datasourceModal.show()
}}
on:select={e => {
formType = e.detail
}}
type={formType}
/>
</Modal>

30
packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte
View file

@ -4,37 +4,33 @@
import ICONS from "components/backend/DatasourceNavigator/icons"
import { IntegrationNames } from "constants"
import { onMount } from "svelte"
import rowListScreen from "templates/rowListScreen"
import DatasourceTemplateRow from "./DatasourceTemplateRow.svelte"
export let mode
export let onCancel
export let onConfirm
export let initialScreens = []
let selectedScreens = [...initialScreens]
let selectedSources = []
$: filteredSources = $datasources.list?.filter(datasource => {
return datasource.source !== IntegrationNames.REST && datasource["entities"]
})
const toggleSelection = datasource => {
const { resourceId } = datasource
if (selectedScreens.find(s => s.resourceId === resourceId)) {
selectedScreens = selectedScreens.filter(
screen => screen.resourceId !== resourceId
const exists = selectedSources.find(
d => d.resourceId === datasource.resourceId
)
if (exists) {
selectedSources = selectedSources.filter(
d => d.resourceId === datasource.resourceId
)
} else {
selectedScreens = [
...selectedScreens,
rowListScreen([datasource], mode)[0],
]
selectedSources = [...selectedSources, datasource]
}
}
const confirmDatasourceSelection = async () => {
await onConfirm({
templates: selectedScreens,
datasources: selectedSources,
})
}
@ -54,7 +50,7 @@
cancelText="Back"
onConfirm={confirmDatasourceSelection}
{onCancel}
disabled={!selectedScreens.length}
disabled={!selectedSources.length}
size="L"
>
<Body size="S">
@ -85,8 +81,8 @@
resourceId: table._id,
type: "table",
}}
{@const selected = selectedScreens.find(
screen => screen.resourceId === tableDS.resourceId
{@const selected = selectedSources.find(
datasource => datasource.resourceId === tableDS.resourceId
)}
<DatasourceTemplateRow
on:click={() => toggleSelection(tableDS)}
@ -103,7 +99,7 @@
tableId: view.tableId,
type: "viewV2",
}}
{@const selected = selectedScreens.find(
{@const selected = selectedSources.find(
x => x.resourceId === viewDS.resourceId
)}
<DatasourceTemplateRow

123
packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte Normal file
View file

@ -0,0 +1,123 @@
<script>
import { ModalContent, Layout, Body, Icon } from "@budibase/bbui"
import { createEventDispatcher } from "svelte"
export let onCancel = () => {}
export let onConfirm = () => {}
export let type
const dispatch = createEventDispatcher()
</script>
<span>
<ModalContent
title="Select form type"
confirmText="Done"
cancelText="Back"
{onConfirm}
{onCancel}
disabled={!type}
size="L"
>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<Layout noPadding gap="S">
<div
class="form-type"
class:selected={type === "Create"}
on:click={() => {
dispatch("select", "Create")
}}
>
<div class="form-type-wrap">
<div class="form-type-content">
<Body noPadding>Create a new row</Body>
<Body size="S">
For capturing and storing new data from your users
</Body>
</div>
{#if type === "Create"}
<span class="form-type-check">
<Icon size="S" name="CheckmarkCircle" />
</span>
{/if}
</div>
</div>
<div
class="form-type"
class:selected={type === "Update"}
on:click={() => {
dispatch("select", "Update")
}}
>
<div class="form-type-wrap">
<div class="form-type-content">
<Body noPadding>Update an existing row</Body>
<Body size="S">For viewing and updating existing data</Body>
</div>
{#if type === "Update"}
<span class="form-type-check">
<Icon size="S" name="CheckmarkCircle" />
</span>
{/if}
</div>
</div>
<div
class="form-type"
class:selected={type === "View"}
on:click={() => {
dispatch("select", "View")
}}
>
<div class="form-type-wrap">
<div class="form-type-content">
<Body noPadding>View an existing row</Body>
<Body size="S">For a read only view of your data</Body>
</div>
{#if type === "View"}
<span class="form-type-check">
<Icon size="S" name="CheckmarkCircle" />
</span>
{/if}
</div>
</div>
</Layout>
</ModalContent>
</span>
<style>
.form-type {
cursor: pointer;
gap: var(--spacing-s);
padding: var(--spacing-m) var(--spacing-xl);
background: var(--spectrum-alias-background-color-secondary);
transition: 0.3s all;
border: 1px solid var(--spectrum-global-color-gray-300);
border-radius: 4px;
display: flex;
flex-direction: column;
}
.selected,
.form-type:hover {
background: var(--spectrum-alias-background-color-tertiary);
}
.form-type-wrap {
display: flex;
align-items: center;
justify-content: space-between;
}
.form-type :global(p:nth-child(2)) {
color: var(--grey-6);
}
.form-type-check {
margin-left: auto;
}
.form-type-check :global(.spectrum-Icon) {
color: var(--spectrum-global-color-green-600);
}
.form-type-content {
gap: var(--spacing-s);
display: flex;
flex-direction: column;
}
</style>

3
packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte
View file

@ -9,6 +9,7 @@
export let onCancel
export let screenUrl
export let screenAccessRole
export let confirmText = "Done"
let error
@ -40,7 +41,7 @@
<ModalContent
title="Access"
confirmText="Done"
{confirmText}
cancelText="Back"
{onConfirm}
{onCancel}

0
packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/blank.png → packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/images/blank.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 32 KiB

After

Width:  |  Height:  |  Size: 32 KiB

BIN
packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/images/form.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 22 KiB

0
packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/grid.png → packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/images/grid.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

0
packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/table.png → packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/images/table.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 26 KiB

After

Width:  |  Height:  |  Size: 26 KiB

17
packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte
View file

@ -1,9 +1,10 @@
<script>
import { Body } from "@budibase/bbui"
import CreationPage from "components/common/CreationPage.svelte"
import blankImage from "./blank.png"
import tableImage from "./table.png"
import gridImage from "./grid.png"
import blankImage from "./images/blank.png"
import tableImage from "./images/table.png"
import gridImage from "./images/grid.png"
import formImage from "./images/form.png"
import CreateScreenModal from "./CreateScreenModal.svelte"
import { screenStore } from "stores/builder"
@ -56,6 +57,16 @@
<Body size="XS">View and manipulate rows on a grid</Body>
</div>
</div>
<div class="card" on:click={() => createScreenModal.show("form")}>
<div class="image">
<img alt="" src={formImage} />
</div>
<div class="text">
<Body size="S">Form</Body>
<Body size="XS">Capture data from your users</Body>
</div>
</div>
</div>
</CreationPage>
</div>

2
packages/builder/src/stores/builder/app.js
View file

@ -29,6 +29,7 @@ export const INITIAL_APP_META_STATE = {
initialised: false,
hasAppPackage: false,
usedPlugins: null,
automations: {},
routes: {},
}
@ -63,6 +64,7 @@ export class AppMetaStore extends BudiStore {
...app.features,
},
initialised: true,
automations: app.automations || {},
hasAppPackage: true,
}))
}

31
packages/builder/src/stores/builder/builder.js
View file

@ -2,12 +2,11 @@ import { get } from "svelte/store"
import { createBuilderWebsocket } from "./websocket.js"
import { BuilderSocketEvent } from "@budibase/shared-core"
import BudiStore from "./BudiStore"
import { previewStore } from "./preview.js"
import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
export const INITIAL_BUILDER_STATE = {
previousTopNavPath: {},
highlightedSettingKey: null,
highlightedSetting: null,
propertyFocus: null,
builderSidePanel: false,
onboarding: false,
@ -26,7 +25,6 @@ export class BuilderStore extends BudiStore {
this.reset = this.reset.bind(this)
this.highlightSetting = this.highlightSetting.bind(this)
this.propertyFocus = this.propertyFocus.bind(this)
this.hover = this.hover.bind(this)
this.hideBuilderSidePanel = this.hideBuilderSidePanel.bind(this)
this.showBuilderSidePanel = this.showBuilderSidePanel.bind(this)
this.setPreviousTopNavPath = this.setPreviousTopNavPath.bind(this)
@ -58,10 +56,10 @@ export class BuilderStore extends BudiStore {
this.websocket = null
}
highlightSetting(key) {
highlightSetting(key, type) {
this.update(state => ({
...state,
highlightedSettingKey: key,
highlightedSetting: key ? { key, type: type || "info" } : null,
}))
}
@ -135,25 +133,20 @@ export class BuilderStore extends BudiStore {
}))
}
startTour(tourKey) {
endBuilderOnboarding() {
this.update(state => ({
...state,
tourKey: tourKey,
onboarding: false,
}))
}
hover(componentId, notifyClient = true) {
const store = get(this.store)
if (componentId === store.hoveredComponentId) {
return
}
this.update(state => {
state.hoveredComponentId = componentId
return state
})
if (notifyClient) {
previewStore.sendEvent("hover-component", componentId)
}
setTour(tourKey) {
this.update(state => ({
...state,
tourStepKey: null,
tourNodes: null,
tourKey: tourKey,
}))
}
}

34
packages/builder/src/stores/builder/tests/builder.test.js
View file

@ -88,14 +88,42 @@ describe("Builder store", () => {
)
})
it("Sync a highlighted setting key to state", ctx => {
expect(ctx.test.store.highlightedSettingKey).toBeNull()
it("Sync a highlighted setting key to state. Default to info type", ctx => {
expect(ctx.test.store.highlightedSetting).toBeNull()
ctx.test.builderStore.highlightSetting("testing")
expect(ctx.test.store).toStrictEqual({
...INITIAL_BUILDER_STATE,
highlightedSettingKey: "testing",
highlightedSetting: {
key: "testing",
type: "info",
},
})
})
it("Sync a highlighted setting key to state. Use provided type", ctx => {
expect(ctx.test.store.highlightedSetting).toBeNull()
ctx.test.builderStore.highlightSetting("testing", "error")
expect(ctx.test.store).toStrictEqual({
...INITIAL_BUILDER_STATE,
highlightedSetting: {
key: "testing",
type: "error",
},
})
})
it("Sync a highlighted setting key to state. Unset when no value is passed", ctx => {
expect(ctx.test.store.highlightedSetting).toBeNull()
ctx.test.builderStore.highlightSetting("testing", "error")
ctx.test.builderStore.highlightSetting()
expect(ctx.test.store).toStrictEqual({
...INITIAL_BUILDER_STATE,
})
})

43
packages/builder/src/templates/formScreen.js Normal file
View file

@ -0,0 +1,43 @@
import { Screen } from "./Screen"
import { Component } from "./Component"
import sanitizeUrl from "helpers/sanitizeUrl"
export const FORM_TEMPLATE = "FORM_TEMPLATE"
export const formUrl = datasource => sanitizeUrl(`/${datasource.label}-form`)
// Mode not really necessary
export default function (datasources, config) {
if (!Array.isArray(datasources)) {
return []
}
return datasources.map(datasource => {
return {
name: `${datasource.label} - Form`,
create: () => createScreen(datasource, config),
id: FORM_TEMPLATE,
resourceId: datasource.resourceId,
}
})
}
const generateMultistepFormBlock = (dataSource, { actionType } = {}) => {
const multistepFormBlock = new Component(
"@budibase/standard-components/multistepformblock"
)
multistepFormBlock
.customProps({
actionType,
dataSource,
steps: [{}],
})
.instanceName(`${dataSource.label} - Multistep Form block`)
return multistepFormBlock
}
const createScreen = (datasource, config) => {
return new Screen()
.route(formUrl(datasource))
.instanceName(`${datasource.label} - Form`)
.addChild(generateMultistepFormBlock(datasource, config))
.json()
}

7
packages/builder/src/templates/index.js
View file

@ -1,7 +1,11 @@
import rowListScreen from "./rowListScreen"
import createFromScratchScreen from "./createFromScratchScreen"
import formScreen from "./formScreen"
const allTemplates = datasources => [...rowListScreen(datasources)]
const allTemplates = datasources => [
...rowListScreen(datasources),
...formScreen(datasources),
]
// Allows us to apply common behaviour to all create() functions
const createTemplateOverride = template => () => {
@ -19,6 +23,7 @@ export default datasources => {
})
const fromScratch = enrichTemplate(createFromScratchScreen)
const tableTemplates = allTemplates(datasources).map(enrichTemplate)
return [
fromScratch,
...tableTemplates.sort((templateA, templateB) => {

32
packages/client/manifest.json
View file

@ -525,6 +525,38 @@
"barTitle": "Disable button",
"key": "disabled"
},
{
"type": "icon",
"label": "Icon",
"key": "icon"
},
{
"type": "select",
"label": "Gap",
"key": "gap",
"showInBar": true,
"barStyle": "picker",
"dependsOn": "icon",
"options": [
{
"label": "None",
"value": "N"
},
{
"label": "Small",
"value": "S"
},
{
"label": "Medium",
"value": "M"
},
{
"label": "Large",
"value": "L"
}
],
"defaultValue": "M"
},
{
"type": "event",
"label": "On click",

24
packages/client/src/components/app/Button.svelte
View file

@ -13,9 +13,10 @@
export let size = "M"
export let type = "cta"
export let quiet = false
export let icon = null
export let gap = "M"
// For internal use only for now - not defined in the manifest
export let icon = null
export let active = false
const handleOnClick = async () => {
@ -47,7 +48,7 @@
{#key $component.editing}
<button
class={`spectrum-Button spectrum-Button--size${size} spectrum-Button--${type}`}
class={`spectrum-Button spectrum-Button--size${size} spectrum-Button--${type} gap-${gap}`}
class:spectrum-Button--quiet={quiet}
disabled={disabled || handlingOnClick}
use:styleable={$component.styles}
@ -58,15 +59,7 @@
class:active
>
{#if icon}
<svg
class:hasText={componentText?.length > 0}
class="spectrum-Icon spectrum-Icon--size{size.toUpperCase()}"
focusable="false"
aria-hidden="true"
aria-label={icon}
>
<use xlink:href="#spectrum-icon-18-{icon}" />
</svg>
<i class="{icon} {size}" />
{/if}
{componentText}
</button>
@ -92,4 +85,13 @@
.active {
color: var(--spectrum-global-color-blue-600);
}
.gap-S {
gap: 8px;
}
.gap-M {
gap: 16px;
}
.gap-L {
gap: 32px;
}
</style>

4
packages/client/src/components/app/ButtonGroup.svelte
View file

@ -20,7 +20,7 @@
wrap: true,
}}
>
{#each buttons as { text, type, quiet, disabled, onClick, size }}
{#each buttons as { text, type, quiet, disabled, onClick, size, icon, gap }}
<BlockComponent
type="button"
props={{
@ -29,6 +29,8 @@
type,
quiet,
disabled,
icon,
gap,
size: size || "M",
}}
/>

4
packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte
View file

@ -92,9 +92,9 @@
{#if schemaLoaded}
<Button
onClick={openEditor}
icon="Properties"
icon="ri-filter-3-line"
text="Filter"
{size}
size="XL"
type="secondary"
quiet
active={filters?.length > 0}

2
packages/pro

@ -1 +1 @@
Subproject commit 678c913246bacb398fbda2ad73a8e1bb562983fd
Subproject commit cce3ba87b8aaa00e6e3fa6df9888c35722a12c45

17
packages/server/__mocks__/node-fetch.ts
View file

@ -8,6 +8,7 @@ module FetchMock {
let mockSearch = false
const func = async (url: any, opts: any) => {
const { host, pathname } = new URL(url)
function json(body: any, status = 200) {
return {
status,
@ -34,7 +35,7 @@ module FetchMock {
}
}
if (url.includes("/api/global")) {
if (pathname.includes("/api/global")) {
const user = {
email: "test@example.com",
_id: "us_test@example.com",
@ -47,31 +48,31 @@ module FetchMock {
global: false,
},
}
return url.endsWith("/users") && opts.method === "GET"
return pathname.endsWith("/users") && opts.method === "GET"
? json([user])
: json(user)
}
// mocked data based on url
else if (url.includes("api/apps")) {
else if (pathname.includes("api/apps")) {
return json({
app1: {
url: "/app1",
},
})
} else if (url.includes("example.com")) {
} else if (host.includes("example.com")) {
return json({
body: opts.body,
url,
method: opts.method,
})
} else if (url.includes("invalid.com")) {
} else if (host.includes("invalid.com")) {
return json(
{
invalid: true,
},
404
)
} else if (mockSearch && url.includes("_search")) {
} else if (mockSearch && pathname.includes("_search")) {
const body = opts.body
const parts = body.split("tableId:")
let tableId
@ -90,7 +91,7 @@ module FetchMock {
],
bookmark: "test",
})
} else if (url.includes("google.com")) {
} else if (host.includes("google.com")) {
return json({
url,
opts,
@ -177,7 +178,7 @@ module FetchMock {
} else if (url === "https://www.googleapis.com/oauth2/v4/token") {
// any valid response
return json({})
} else if (url.includes("failonce.com")) {
} else if (host.includes("failonce.com")) {
failCount++
if (failCount === 1) {
return json({ message: "error" }, 500)

2
packages/server/package.json
View file

@ -149,7 +149,7 @@
"rimraf": "3.0.2",
"supertest": "6.3.3",
"swagger-jsdoc": "6.1.0",
"testcontainers": "10.6.0",
"testcontainers": "10.7.2",
"timekeeper": "2.2.0",
"ts-node": "10.8.1",
"tsconfig-paths": "4.0.0",

15
packages/server/src/api/controllers/role.ts
View file

@ -106,6 +106,21 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
)
role._rev = result.rev
ctx.body = role
const devDb = context.getDevAppDB()
const prodDb = context.getProdAppDB()
if (await prodDb.exists()) {
const replication = new dbCore.Replication({
source: devDb.name,
target: prodDb.name,
})
await replication.replicate({
filter: (doc: any, params: any) => {
return doc._id && doc._id.startsWith("role_")
},
})
}
}
export async function destroy(ctx: UserCtx<void, DestroyRoleResponse>) {

52
packages/server/src/api/controllers/row/alias.ts
View file

@ -1,12 +1,27 @@
import {
QueryJson,
SearchFilters,
Table,
Row,
Datasource,
DatasourcePlusQueryResponse,
Operation,
QueryJson,
Row,
SearchFilters,
} from "@budibase/types"
import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { getSQLClient } from "../../../sdk/app/rows/utils"
import { cloneDeep } from "lodash"
import sdk from "../../../sdk"
import { makeExternalQuery } from "../../../integrations/base/query"
import { SqlClient } from "../../../integrations/utils"
const WRITE_OPERATIONS: Operation[] = [
Operation.CREATE,
Operation.UPDATE,
Operation.DELETE,
]
const DISABLED_WRITE_CLIENTS: SqlClient[] = [
SqlClient.MY_SQL,
SqlClient.MS_SQL,
SqlClient.ORACLE,
]
class CharSequence {
static alphabet = "abcdefghijklmnopqrstuvwxyz"
@ -43,6 +58,25 @@ export default class AliasTables {
this.charSeq = new CharSequence()
}
isAliasingEnabled(json: QueryJson, datasource: Datasource) {
const fieldLength = json.resource?.fields?.length
if (!fieldLength || fieldLength <= 0) {
return false
}
try {
const sqlClient = getSQLClient(datasource)
const isWrite = WRITE_OPERATIONS.includes(json.endpoint.operation)
const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient)
if (isWrite && isDisabledClient) {
return false
}
} catch (err) {
// if we can't get an SQL client, we can't alias
return false
}
return true
}
getAlias(tableName: string) {
if (this.aliases[tableName]) {
return this.aliases[tableName]
@ -111,8 +145,10 @@ export default class AliasTables {
}
async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
const fieldLength = json.resource?.fields?.length
const aliasingEnabled = fieldLength && fieldLength > 0
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
const aliasingEnabled = this.isAliasingEnabled(json, datasource)
if (aliasingEnabled) {
json = cloneDeep(json)
// run through the query json to update anywhere a table may be used
@ -158,7 +194,7 @@ export default class AliasTables {
}
json.tableAliases = invertedTableAliases
}
const response = await getDatasourceAndQuery(json)
const response = await makeExternalQuery(datasource, json)
if (Array.isArray(response) && aliasingEnabled) {
return this.reverse(response)
} else {

2
packages/server/src/api/controllers/user.ts
View file

@ -1,6 +1,6 @@
import { generateUserFlagID, InternalTables } from "../../db/utils"
import { getFullUser } from "../../utilities/users"
import { context } from "@budibase/backend-core"
import { cache, context } from "@budibase/backend-core"
import {
ContextUserMetadata,
Ctx,

80
packages/server/src/api/routes/tests/application.spec.ts
View file

@ -16,8 +16,9 @@ import * as setup from "./utilities"
import { AppStatus } from "../../../db/utils"
import { events, utils, context } from "@budibase/backend-core"
import env from "../../../environment"
import type { App } from "@budibase/types"
import { type App } from "@budibase/types"
import tk from "timekeeper"
import * as uuid from "uuid"
describe("/applications", () => {
let config = setup.getConfig()
@ -251,7 +252,7 @@ describe("/applications", () => {
describe("permissions", () => {
it("should only return apps a user has access to", async () => {
const user = await config.createUser({
let user = await config.createUser({
builder: { global: false },
admin: { global: false },
})
@ -260,6 +261,81 @@ describe("/applications", () => {
const apps = await config.api.application.fetch()
expect(apps).toHaveLength(0)
})
user = await config.globalUser({
...user,
builder: {
apps: [config.getProdAppId()],
},
})
await config.withUser(user, async () => {
const apps = await config.api.application.fetch()
expect(apps).toHaveLength(1)
})
})
it("should only return apps a user has access to through a custom role", async () => {
let user = await config.createUser({
builder: { global: false },
admin: { global: false },
})
await config.withUser(user, async () => {
const apps = await config.api.application.fetch()
expect(apps).toHaveLength(0)
})
const role = await config.api.roles.save({
name: "Test",
inherits: "PUBLIC",
permissionId: "read_only",
version: "name",
})
user = await config.globalUser({
...user,
roles: {
[config.getProdAppId()]: role.name,
},
})
await config.withUser(user, async () => {
const apps = await config.api.application.fetch()
expect(apps).toHaveLength(1)
})
})
it.only("should only return apps a user has access to through a custom role on a group", async () => {
let user = await config.createUser({
builder: { global: false },
admin: { global: false },
})
await config.withUser(user, async () => {
const apps = await config.api.application.fetch()
expect(apps).toHaveLength(0)
})
const roleName = uuid.v4().replace(/-/g, "")
const role = await config.api.roles.save({
name: roleName,
inherits: "PUBLIC",
permissionId: "read_only",
version: "name",
})
const group = await config.createGroup(role._id!)
user = await config.globalUser({
...user,
userGroups: [group._id!],
})
await config.withUser(user, async () => {
const apps = await config.api.application.fetch()
expect(apps).toHaveLength(1)
})
})
})
})

10
packages/server/src/integrations/base/sql.ts
View file

@ -435,10 +435,12 @@ class InternalBuilder {
aliases?: QueryJson["tableAliases"]
): Knex.QueryBuilder {
const tableName = endpoint.entityId
const tableAliased = aliases?.[tableName]
? `${tableName} as ${aliases?.[tableName]}`
: tableName
let query = knex(tableAliased)
const tableAlias = aliases?.[tableName]
let table: string | Record<string, string> = tableName
if (tableAlias) {
table = { [tableAlias]: tableName }
}
let query = knex(table)
if (endpoint.schema) {
query = query.withSchema(endpoint.schema)
}

48
packages/server/src/integrations/index.ts
View file

@ -14,7 +14,12 @@ import firebase from "./firebase"
import redis from "./redis"
import snowflake from "./snowflake"
import oracle from "./oracle"
import { SourceName, Integration, PluginType } from "@budibase/types"
import {
SourceName,
Integration,
PluginType,
IntegrationBase,
} from "@budibase/types"
import { getDatasourcePlugin } from "../utilities/fileSystem"
import env from "../environment"
import cloneDeep from "lodash/cloneDeep"
@ -40,25 +45,28 @@ const DEFINITIONS: Record<SourceName, Integration | undefined> = {
[SourceName.BUDIBASE]: undefined,
}
const INTEGRATIONS: Record<SourceName, any> = {
[SourceName.POSTGRES]: postgres.integration,
[SourceName.DYNAMODB]: dynamodb.integration,
[SourceName.MONGODB]: mongodb.integration,
[SourceName.ELASTICSEARCH]: elasticsearch.integration,
[SourceName.COUCHDB]: couchdb.integration,
[SourceName.SQL_SERVER]: sqlServer.integration,
[SourceName.S3]: s3.integration,
[SourceName.AIRTABLE]: airtable.integration,
[SourceName.MYSQL]: mysql.integration,
[SourceName.ARANGODB]: arangodb.integration,
[SourceName.REST]: rest.integration,
[SourceName.FIRESTORE]: firebase.integration,
[SourceName.GOOGLE_SHEETS]: googlesheets.integration,
[SourceName.REDIS]: redis.integration,
[SourceName.SNOWFLAKE]: snowflake.integration,
[SourceName.ORACLE]: undefined,
[SourceName.BUDIBASE]: undefined,
}
type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase
const INTEGRATIONS: Record<SourceName, IntegrationBaseConstructor | undefined> =
{
[SourceName.POSTGRES]: postgres.integration,
[SourceName.DYNAMODB]: dynamodb.integration,
[SourceName.MONGODB]: mongodb.integration,
[SourceName.ELASTICSEARCH]: elasticsearch.integration,
[SourceName.COUCHDB]: couchdb.integration,
[SourceName.SQL_SERVER]: sqlServer.integration,
[SourceName.S3]: s3.integration,
[SourceName.AIRTABLE]: airtable.integration,
[SourceName.MYSQL]: mysql.integration,
[SourceName.ARANGODB]: arangodb.integration,
[SourceName.REST]: rest.integration,
[SourceName.FIRESTORE]: firebase.integration,
[SourceName.GOOGLE_SHEETS]: googlesheets.integration,
[SourceName.REDIS]: redis.integration,
[SourceName.SNOWFLAKE]: snowflake.integration,
[SourceName.ORACLE]: undefined,
[SourceName.BUDIBASE]: undefined,
}
// optionally add oracle integration if the oracle binary can be installed
if (

110
packages/server/src/integrations/tests/sqlAlias.spec.ts
View file

@ -1,4 +1,4 @@
import { QueryJson } from "@budibase/types"
import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types"
import { join } from "path"
import Sql from "../base/sql"
import { SqlClient } from "../utils"
@ -198,6 +198,114 @@ describe("Captures of real examples", () => {
})
})
describe("check aliasing is disabled/enabled", () => {
const tables = ["tableA", "tableB"]
function getDatasource(source: SourceName): Datasource {
return {
source,
type: "datasource",
isSQL: true,
}
}
function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson {
return {
endpoint: { datasourceId: "", entityId: "", operation: op },
resource: {
fields,
},
}
}
it("should check for Postgres aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.POSTGRES)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(true)
})
it("should check for MS-SQL aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.SQL_SERVER)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(false)
})
it("should check for MySQL aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.MYSQL)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(false)
})
it("should check for Oracle aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.ORACLE)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(false)
})
it("should disable aliasing for non-SQL datasources", () => {
const aliasing = new AliasTables(tables)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), {
source: SourceName.GOOGLE_SHEETS,
type: "datasource",
isSQL: false,
})
)
})
it("should disable when no fields", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.POSTGRES)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource)
).toEqual(false)
})
})
describe("check some edge cases", () => {
const tableNames = ["hello", "world"]

36
packages/server/src/sdk/app/rows/utils.ts
View file

@ -1,17 +1,51 @@
import cloneDeep from "lodash/cloneDeep"
import validateJs from "validate.js"
import {
Datasource,
DatasourcePlusQueryResponse,
FieldType,
QueryJson,
Row,
SourceName,
Table,
TableSchema,
DatasourcePlusQueryResponse,
} from "@budibase/types"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Format } from "../../../api/controllers/view/exporters"
import sdk from "../.."
import { isRelationshipColumn } from "../../../db/utils"
import { SqlClient } from "../../../integrations/utils"
const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
[SourceName.POSTGRES]: SqlClient.POSTGRES,
[SourceName.MYSQL]: SqlClient.MY_SQL,
[SourceName.SQL_SERVER]: SqlClient.MS_SQL,
[SourceName.ORACLE]: SqlClient.ORACLE,
[SourceName.DYNAMODB]: undefined,
[SourceName.MONGODB]: undefined,
[SourceName.ELASTICSEARCH]: undefined,
[SourceName.COUCHDB]: undefined,
[SourceName.S3]: undefined,
[SourceName.AIRTABLE]: undefined,
[SourceName.ARANGODB]: undefined,
[SourceName.REST]: undefined,
[SourceName.FIRESTORE]: undefined,
[SourceName.GOOGLE_SHEETS]: undefined,
[SourceName.REDIS]: undefined,
[SourceName.SNOWFLAKE]: undefined,
[SourceName.BUDIBASE]: undefined,
}
export function getSQLClient(datasource: Datasource): SqlClient {
if (!datasource.isSQL) {
throw new Error("Cannot get SQL Client for non-SQL datasource")
}
const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source]
if (lookup) {
return lookup
}
throw new Error("Unable to determine client for SQL datasource")
}
export async function getDatasourceAndQuery(
json: QueryJson

1
packages/server/src/tests/jestEnv.ts
View file

@ -10,3 +10,4 @@ process.env.MOCK_REDIS = "1"
process.env.PLATFORM_URL = "http://localhost:10000"
process.env.REDIS_PASSWORD = "budibase"
process.env.BUDIBASE_VERSION = "0.0.0+jest"
process.env.WORKER_URL = "http://localhost:10000"

28
packages/server/src/tests/utilities/TestConfiguration.ts
View file

@ -299,11 +299,11 @@ export default class TestConfiguration {
}
}
withUser(user: User, f: () => Promise<void>) {
async withUser(user: User, f: () => Promise<void>) {
const oldUser = this.user
this.user = user
try {
return f()
return await f()
} finally {
this.user = oldUser
}
@ -347,7 +347,7 @@ export default class TestConfiguration {
lastName = generator.last(),
builder = { global: true },
admin = { global: false },
email = generator.email(),
email = generator.email({ domain: "example.com" }),
tenantId = this.getTenantId(),
roles = {},
} = config
@ -363,6 +363,7 @@ export default class TestConfiguration {
_id,
...existing,
...config,
_rev: existing._rev,
email,
roles,
tenantId,
@ -372,11 +373,12 @@ export default class TestConfiguration {
admin,
}
await sessions.createASession(_id, {
sessionId: "sessionid",
sessionId: this.sessionIdForUser(_id),
tenantId: this.getTenantId(),
csrfToken: this.csrfToken,
})
const resp = await db.put(user)
await cache.user.invalidateUser(_id)
return {
_rev: resp.rev,
...user,
@ -384,9 +386,7 @@ export default class TestConfiguration {
}
async createUser(user: Partial<User> = {}): Promise<User> {
const resp = await this.globalUser(user)
await cache.user.invalidateUser(resp._id!)
return resp
return await this.globalUser(user)
}
async createGroup(roleId: string = roles.BUILTIN_ROLE_IDS.BASIC) {
@ -416,6 +416,10 @@ export default class TestConfiguration {
})
}
sessionIdForUser(userId: string): string {
return `sessionid-${userId}`
}
async login({
roleId,
userId,
@ -442,13 +446,13 @@ export default class TestConfiguration {
})
}
await sessions.createASession(userId, {
sessionId: "sessionid",
sessionId: this.sessionIdForUser(userId),
tenantId: this.getTenantId(),
})
// have to fake this
const authObj = {
userId,
sessionId: "sessionid",
sessionId: this.sessionIdForUser(userId),
tenantId: this.getTenantId(),
}
const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
@ -470,7 +474,7 @@ export default class TestConfiguration {
const user = this.getUser()
const authObj: AuthToken = {
userId: user._id!,
sessionId: "sessionid",
sessionId: this.sessionIdForUser(user._id!),
tenantId,
}
const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
@ -508,7 +512,7 @@ export default class TestConfiguration {
async basicRoleHeaders() {
return await this.roleHeaders({
email: generator.email(),
email: generator.email({ domain: "example.com" }),
builder: false,
prodApp: true,
roleId: roles.BUILTIN_ROLE_IDS.BASIC,
@ -516,7 +520,7 @@ export default class TestConfiguration {
}
async roleHeaders({
email = generator.email(),
email = generator.email({ domain: "example.com" }),
roleId = roles.BUILTIN_ROLE_IDS.ADMIN,
builder = false,
prodApp = true,

3
packages/server/src/tests/utilities/api/index.ts
View file

@ -11,6 +11,7 @@ import { BackupAPI } from "./backup"
import { AttachmentAPI } from "./attachment"
import { UserAPI } from "./user"
import { QueryAPI } from "./query"
import { RoleAPI } from "./role"
export default class API {
table: TableAPI
@ -25,6 +26,7 @@ export default class API {
attachment: AttachmentAPI
user: UserAPI
query: QueryAPI
roles: RoleAPI
constructor(config: TestConfiguration) {
this.table = new TableAPI(config)
@ -39,5 +41,6 @@ export default class API {
this.attachment = new AttachmentAPI(config)
this.user = new UserAPI(config)
this.query = new QueryAPI(config)
this.roles = new RoleAPI(config)
}
}

41
packages/server/src/tests/utilities/api/role.ts Normal file
View file

@ -0,0 +1,41 @@
import {
AccessibleRolesResponse,
FetchRolesResponse,
FindRoleResponse,
SaveRoleRequest,
SaveRoleResponse,
} from "@budibase/types"
import { Expectations, TestAPI } from "./base"
export class RoleAPI extends TestAPI {
fetch = async (expectations?: Expectations) => {
return await this._get<FetchRolesResponse>(`/api/roles`, {
expectations,
})
}
find = async (roleId: string, expectations?: Expectations) => {
return await this._get<FindRoleResponse>(`/api/roles/${roleId}`, {
expectations,
})
}
save = async (body: SaveRoleRequest, expectations?: Expectations) => {
return await this._post<SaveRoleResponse>(`/api/roles`, {
body,
expectations,
})
}
destroy = async (roleId: string, expectations?: Expectations) => {
return await this._delete(`/api/roles/${roleId}`, {
expectations,
})
}
accesssible = async (expectations?: Expectations) => {
return await this._get<AccessibleRolesResponse>(`/api/roles/accessible`, {
expectations,
})
}
}

49
packages/shared-core/src/filters.ts
View file

@ -390,24 +390,41 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
}
)
// Match a document against all criteria
const docMatch = (doc: any) => {
return (
stringMatch(doc) &&
fuzzyMatch(doc) &&
rangeMatch(doc) &&
equalMatch(doc) &&
notEqualMatch(doc) &&
emptyMatch(doc) &&
notEmptyMatch(doc) &&
oneOf(doc) &&
contains(doc) &&
containsAny(doc) &&
notContains(doc)
)
}
const filterFunctions: Record<SearchQueryOperators, (doc: any) => boolean> =
{
string: stringMatch,
fuzzy: fuzzyMatch,
range: rangeMatch,
equal: equalMatch,
notEqual: notEqualMatch,
empty: emptyMatch,
notEmpty: notEmptyMatch,
oneOf: oneOf,
contains: contains,
containsAny: containsAny,
notContains: notContains,
}
// Process all docs
const activeFilterKeys: SearchQueryOperators[] = Object.entries(query || {})
.filter(
([key, value]: [string, any]) =>
!["allOr", "onEmptyFilter"].includes(key) &&
value &&
Object.keys(value as Record<string, any>).length > 0
)
.map(([key]) => key as any)
const results: boolean[] = activeFilterKeys.map(filterKey => {
return filterFunctions[filterKey]?.(doc) ?? false
})
if (query!.allOr) {
return results.some(result => result === true)
} else {
return results.every(result => result === true)
}
}
return docs.filter(docMatch)
}

129
packages/shared-core/src/tests/filters.test.ts
View file

@ -47,10 +47,7 @@ describe("runLuceneQuery", () => {
},
]
function buildQuery(
filterKey: string,
value: { [key: string]: any }
): SearchQuery {
function buildQuery(filters: { [filterKey: string]: any }): SearchQuery {
const query: SearchQuery = {
string: {},
fuzzy: {},
@ -63,8 +60,13 @@ describe("runLuceneQuery", () => {
notContains: {},
oneOf: {},
containsAny: {},
allOr: false,
}
query[filterKey as SearchQueryOperators] = value
for (const filterKey in filters) {
query[filterKey as SearchQueryOperators] = filters[filterKey]
}
return query
}
@ -73,16 +75,17 @@ describe("runLuceneQuery", () => {
})
it("should return matching rows for equal filter", () => {
const query = buildQuery("equal", {
order_status: 4,
const query = buildQuery({
equal: { order_status: 4 },
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2])
})
it("should return matching row for notEqual filter", () => {
const query = buildQuery("notEqual", {
order_status: 4,
const query = buildQuery({
notEqual: { order_status: 4 },
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
})
@ -90,48 +93,56 @@ describe("runLuceneQuery", () => {
expect(
runLuceneQuery(
docs,
buildQuery("fuzzy", {
description: "sm",
buildQuery({
fuzzy: { description: "sm" },
})
).map(row => row.description)
).toEqual(["Small box"])
expect(
runLuceneQuery(
docs,
buildQuery("string", {
description: "SM",
buildQuery({
string: { description: "SM" },
})
).map(row => row.description)
).toEqual(["Small box"])
})
it("should return rows within a range filter", () => {
const query = buildQuery("range", {
customer_id: {
low: 500,
high: 1000,
const query = buildQuery({
range: {
customer_id: {
low: 500,
high: 1000,
},
},
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
})
it("should return rows with numeric strings within a range filter", () => {
const query = buildQuery("range", {
customer_id: {
low: "500",
high: "1000",
const query = buildQuery({
range: {
customer_id: {
low: "500",
high: "1000",
},
},
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
})
it("should return rows with ISO date strings within a range filter", () => {
const query = buildQuery("range", {
order_date: {
low: "2016-01-04T00:00:00.000Z",
high: "2016-01-11T00:00:00.000Z",
const query = buildQuery({
range: {
order_date: {
low: "2016-01-04T00:00:00.000Z",
high: "2016-01-11T00:00:00.000Z",
},
},
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2])
})
@ -150,40 +161,88 @@ describe("runLuceneQuery", () => {
label: "",
},
]
const query = buildQuery("range", {
order_date: {
low: "2016-01-04T00:00:00.000Z",
high: "2016-01-11T00:00:00.000Z",
const query = buildQuery({
range: {
order_date: {
low: "2016-01-04T00:00:00.000Z",
high: "2016-01-11T00:00:00.000Z",
},
},
})
expect(runLuceneQuery(docs, query)).toEqual(docs)
})
it("should return rows with matches on empty filter", () => {
const query = buildQuery("empty", {
label: null,
const query = buildQuery({
empty: {
label: null,
},
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1])
})
it("should return rows with matches on notEmpty filter", () => {
const query = buildQuery("notEmpty", {
label: null,
const query = buildQuery({
notEmpty: {
label: null,
},
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2, 3])
})
test.each([[523, 259], "523,259"])(
"should return rows with matches on numeric oneOf filter",
input => {
let query = buildQuery("oneOf", {
customer_id: input,
const query = buildQuery({
oneOf: {
customer_id: input,
},
})
expect(runLuceneQuery(docs, query).map(row => row.customer_id)).toEqual([
259, 523,
])
}
)
test.each([
[false, []],
[true, [1, 2, 3]],
])("should return %s if allOr is %s ", (allOr, expectedResult) => {
const query = buildQuery({
allOr,
oneOf: { staff_id: [10] },
contains: { description: ["box"] },
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual(
expectedResult
)
})
it("should return matching results if allOr is true and only one filter matches with different operands", () => {
const query = buildQuery({
allOr: true,
equal: { order_status: 4 },
oneOf: { label: ["FRAGILE"] },
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2])
})
it("should handle when a value is null or undefined", () => {
const query = buildQuery({
allOr: true,
equal: { order_status: null },
oneOf: { label: ["FRAGILE"] },
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2])
})
})
describe("buildLuceneQuery", () => {

1
packages/types/src/api/web/auth.ts
View file

@ -18,6 +18,7 @@ export interface UpdateSelfRequest {
password?: string
forceResetPassword?: boolean
onboardedAt?: string
tours?: Record<string, Date>
}
export interface UpdateSelfResponse {

1
packages/types/src/documents/app/role.ts
View file

@ -5,4 +5,5 @@ export interface Role extends Document {
inherits?: string
permissions: { [key: string]: string[] }
version?: string
name: string
}

1
packages/types/src/documents/global/user.ts
View file

@ -55,6 +55,7 @@ export interface User extends Document {
dayPassRecordedAt?: string
userGroups?: string[]
onboardedAt?: string
tours?: Record<string, Date>
scimInfo?: { isSync: true } & Record<string, any>
ssoId?: string
}

1
packages/types/src/sdk/locks.ts
View file

@ -23,6 +23,7 @@ export enum LockName {
APP_MIGRATION = "app_migrations",
PROCESS_AUTO_COLUMNS = "process_auto_columns",
PROCESS_USER_INVITE = "process_user_invite",
PERSIST_DOC_WRITETHROUGH = "persist_doc_writethrough",
}
export type LockOptions = {

2
packages/worker/src/api/routes/global/tests/groups.spec.ts
View file

@ -147,7 +147,7 @@ describe("/api/global/groups", () => {
await Promise.all(
Array.from({ length: 30 }).map(async (_, i) => {
const email = `user${i}@${generator.domain()}`
const email = `user${i}@example.com`
const user = await config.api.users.saveUser({
...structures.users.user(),
email,

1
packages/worker/src/api/routes/validation/users.ts
View file

@ -26,6 +26,7 @@ export const buildSelfSaveValidation = () => {
firstName: OPTIONAL_STRING,
lastName: OPTIONAL_STRING,
onboardedAt: Joi.string().optional(),
tours: Joi.object().optional(),
}
return auth.joiValidator.body(Joi.object(schema).required().unknown(false))
}

2
qa-core/src/account-api/tests/accounts/accounts.cloud.spec.ts
View file

@ -84,7 +84,7 @@ describe("Accounts", () => {
})
it("searches by email", async () => {
const email = generator.email()
const email = generator.email({ domain: "example.com" })
// Empty result
const [_, emptyBody] = await config.api.accounts.search(email, "email")

2
qa-core/src/public-api/fixtures/users.ts
View file

@ -4,7 +4,7 @@ import { generator } from "../../shared"
export const generateUser = (
overrides: Partial<User> = {}
): CreateUserParams => ({
email: generator.email(),
email: generator.email({ domain: "example.com" }),
roles: {
[generator.string({ length: 32, alpha: true, numeric: true })]:
generator.word(),

83
yarn.lock
View file

@ -5303,10 +5303,10 @@
dependencies:
"@types/node" "*"
"@types/dockerode@^3.3.21":
version "3.3.23"
resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-3.3.23.tgz#07b2084013d01e14d5d97856446f4d9c9f27c223"
integrity sha512-Lz5J+NFgZS4cEVhquwjIGH4oQwlVn2h7LXD3boitujBnzOE5o7s9H8hchEjoDK2SlRsJTogdKnQeiJgPPKLIEw==
"@types/dockerode@^3.3.24":
version "3.3.24"
resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-3.3.24.tgz#bea354a4fcd0824a80fd5ea5ede3e8cda71137a7"
integrity sha512-679y69OYusf7Fr2HtdjXPUF6hnHxSA9K4EsuagsMuPno/XpJHjXxCOy2I5YL8POnWbzjsQAi0pyKIYM9HSpQog==
dependencies:
"@types/docker-modem" "*"
"@types/node" "*"
@ -7027,7 +7027,7 @@ ast-types@0.9.6:
resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9"
integrity sha512-qEdtR2UH78yyHX/AUNfXmJTlM48XoFZKBdwi1nzkI1mJL21cmbu0cvjxjpkXJ5NENMq42H+hNs8VLJcqXLerBQ==
async-lock@^1.4.0:
async-lock@^1.4.1:
version "1.4.1"
resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.4.1.tgz#56b8718915a9b68b10fce2f2a9a3dddf765ef53f"
integrity sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==
@ -7263,6 +7263,33 @@ balanced-match@^1.0.0:
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
bare-events@^2.0.0, bare-events@^2.2.0:
version "2.2.1"
resolved "https://registry.yarnpkg.com/bare-events/-/bare-events-2.2.1.tgz#7b6d421f26a7a755e20bf580b727c84b807964c1"
integrity sha512-9GYPpsPFvrWBkelIhOhTWtkeZxVxZOdb3VnFTCzlOo3OjvmTvzLoZFUT8kNFACx0vJej6QPney1Cf9BvzCNE/A==
bare-fs@^2.1.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/bare-fs/-/bare-fs-2.2.1.tgz#c1985d8d3e07a178956b072d3af67cb8c1fa9391"
integrity sha512-+CjmZANQDFZWy4PGbVdmALIwmt33aJg8qTkVjClU6X4WmZkTPBDxRHiBn7fpqEWEfF3AC2io++erpViAIQbSjg==
dependencies:
bare-events "^2.0.0"
bare-os "^2.0.0"
bare-path "^2.0.0"
streamx "^2.13.0"
bare-os@^2.0.0, bare-os@^2.1.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/bare-os/-/bare-os-2.2.0.tgz#24364692984d0bd507621754781b31d7872736b2"
integrity sha512-hD0rOPfYWOMpVirTACt4/nK8mC55La12K5fY1ij8HAdfQakD62M+H4o4tpfKzVGLgRDTuk3vjA4GqGXXCeFbag==
bare-path@^2.0.0, bare-path@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-2.1.0.tgz#830f17fd39842813ca77d211ebbabe238a88cb4c"
integrity sha512-DIIg7ts8bdRKwJRJrUMy/PICEaQZaPGZ26lsSx9MJSwIhSrcdHn7/C8W+XmnG/rKi6BaRcz+JO00CjZteybDtw==
dependencies:
bare-os "^2.1.0"
base62@^1.1.0:
version "1.2.8"
resolved "https://registry.yarnpkg.com/base62/-/base62-1.2.8.tgz#1264cb0fb848d875792877479dbe8bae6bae3428"
@ -9615,10 +9642,10 @@ docker-compose@^0.23.5, docker-compose@^0.23.6:
dependencies:
yaml "^1.10.2"
docker-compose@^0.24.2:
version "0.24.3"
resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.24.3.tgz#298d7bb4aaf37b3b45d0e4ef55c7f58ccc39cca9"
integrity sha512-x3/QN3AIOMe7j2c8f/jcycizMft7dl8MluoB9OGPAYCyKHHiPUFqI9GjCcsU0kYy24vYKMCcfR6+5ZaEyQlrxg==
docker-compose@^0.24.6:
version "0.24.6"
resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.24.6.tgz#d1f490a641bdb7ccc07c4d446b264f026f9a1f15"
integrity sha512-VidlUyNzXMaVsuM79sjSvwC4nfojkP2VneL+Zfs538M2XFnffZDhx6veqnz/evCNIYGyz5O+1fgL6+g0NLWTBA==
dependencies:
yaml "^2.2.2"
@ -20179,6 +20206,16 @@ stream-to-array@^2.3.0:
dependencies:
any-promise "^1.1.0"
streamx@^2.13.0:
version "2.16.1"
resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.16.1.tgz#2b311bd34832f08aa6bb4d6a80297c9caef89614"
integrity sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ==
dependencies:
fast-fifo "^1.1.0"
queue-tick "^1.0.1"
optionalDependencies:
bare-events "^2.2.0"
streamx@^2.15.0:
version "2.15.6"
resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.15.6.tgz#28bf36997ebc7bf6c08f9eba958735231b833887"
@ -20730,14 +20767,16 @@ tar-fs@2.1.1, tar-fs@^2.0.0, tar-fs@^2.1.0:
pump "^3.0.0"
tar-stream "^2.1.4"
tar-fs@^3.0.4:
version "3.0.4"
resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.4.tgz#a21dc60a2d5d9f55e0089ccd78124f1d3771dbbf"
integrity sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==
tar-fs@^3.0.5:
version "3.0.5"
resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.5.tgz#f954d77767e4e6edf973384e1eb95f8f81d64ed9"
integrity sha512-JOgGAmZyMgbqpLwct7ZV8VzkEB6pxXFBVErLtb+XCOqzc6w1xiWKI9GVd6bwk68EX7eJ4DWmfXVmq8K2ziZTGg==
dependencies:
mkdirp-classic "^0.5.2"
pump "^3.0.0"
tar-stream "^3.1.5"
optionalDependencies:
bare-fs "^2.1.1"
bare-path "^2.1.0"
tar-fs@~2.0.1:
version "2.0.1"
@ -20875,25 +20914,25 @@ test-exclude@^6.0.0:
glob "^7.1.4"
minimatch "^3.0.4"
testcontainers@10.6.0:
version "10.6.0"
resolved "https://registry.yarnpkg.com/testcontainers/-/testcontainers-10.6.0.tgz#205ad9148e68ff5c43a6209a30b12965acbe89d8"
integrity sha512-FDJ3o3J8IMu1V7Uc6lNZ2MAD8+BV4HdpR/Vf5mHtgYHKdn6k1EbGFwtnvVNOxanJ99FCjf/EU8eA5ZQ4yjlsGA==
testcontainers@10.7.2, testcontainers@^10.7.2:
version "10.7.2"
resolved "https://registry.yarnpkg.com/testcontainers/-/testcontainers-10.7.2.tgz#619e93200dd47f174b307b40fa830cf023b74c25"
integrity sha512-7d+LVd/4YKp/cutiVMLL5cnj/8p8oYELAVRRyNUM4FyUDz1OLQuwW868nDl7Vd1ZAQxzGeCR+F86FlR9Yw9fMA==
dependencies:
"@balena/dockerignore" "^1.0.2"
"@types/dockerode" "^3.3.21"
"@types/dockerode" "^3.3.24"
archiver "^5.3.2"
async-lock "^1.4.0"
async-lock "^1.4.1"
byline "^5.0.0"
debug "^4.3.4"
docker-compose "^0.24.2"
docker-compose "^0.24.6"
dockerode "^3.3.5"
get-port "^5.1.1"
node-fetch "^2.7.0"
proper-lockfile "^4.1.2"
properties-reader "^2.3.0"
ssh-remote-port-forward "^1.0.4"
tar-fs "^3.0.4"
tar-fs "^3.0.5"
tmp "^0.2.1"
testcontainers@4.7.0: