From ff98ba5a0a5a7aad323dd0f4e4217eadc4f52067 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 4 Apr 2023 00:25:15 +0100 Subject: [PATCH 01/19] Adding a document update queue based on the events which can be used to track when certain documents have changed, using this for users and groups to detect when a re-sync is needed. --- packages/backend-core/src/docUpdates/index.ts | 1 + .../backend-core/src/docUpdates/updates.ts | 62 +++++++++++++++++++ .../processors/DocumentUpdateProcessor.ts | 42 +++++++++++++ .../src/events/processors/index.ts | 3 + packages/backend-core/src/index.ts | 1 + packages/backend-core/src/queue/constants.ts | 1 + .../server/src/sdk/app/applications/sync.ts | 16 ++++- packages/server/src/startup.ts | 1 + packages/types/src/sdk/events/event.ts | 16 +++++ 9 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 packages/backend-core/src/docUpdates/index.ts create mode 100644 packages/backend-core/src/docUpdates/updates.ts create mode 100644 packages/backend-core/src/events/processors/DocumentUpdateProcessor.ts diff --git a/packages/backend-core/src/docUpdates/index.ts b/packages/backend-core/src/docUpdates/index.ts new file mode 100644 index 0000000000..c6c80aad8a --- /dev/null +++ b/packages/backend-core/src/docUpdates/index.ts @@ -0,0 +1 @@ +export * from "./updates" diff --git a/packages/backend-core/src/docUpdates/updates.ts b/packages/backend-core/src/docUpdates/updates.ts new file mode 100644 index 0000000000..478d29f4eb --- /dev/null +++ b/packages/backend-core/src/docUpdates/updates.ts @@ -0,0 +1,62 @@ +import { createQueue, JobQueue } from "../queue" +import BullQueue from "bull" +import { DocumentType, SEPARATOR } from "../constants" +import { doInContext, doInTenant } from "../context" + +type DocUpdateEvent = { + id: string + tenantId: string + appId?: string +} + +type Processor = (update: DocUpdateEvent) => Promise + +const processors: { types: DocumentType[]; processor: Processor }[] = [] +let queue: BullQueue.Queue +let processingPromise: Promise + +export function init() { + queue = createQueue(JobQueue.DOC_UPDATE) +} + +export async function shutdown() { + if (queue) { + await queue.close() + } +} + +export async function update(opts: DocUpdateEvent) { + if (!queue) { + init() + } + await queue.add(opts) +} + +async function handleJob(data: DocUpdateEvent) { + for (let { types, processor } of processors) { + if (types.find(type => data.id.startsWith(`${type}${SEPARATOR}`))) { + const context = data.appId || data.tenantId + const contextFn = data.appId ? doInContext : doInTenant + await contextFn(context, async () => { + await processor(data) + }) + } + } +} + +export async function process(types: DocumentType[], processor: Processor) { + if (!queue) { + init() + } + // add to processor list + processors.push({ + types, + processor, + }) + // if not processing in this instance, kick it off + if (!processingPromise) { + processingPromise = queue.process(async job => { + await handleJob(job.data) + }) + } +} diff --git a/packages/backend-core/src/events/processors/DocumentUpdateProcessor.ts b/packages/backend-core/src/events/processors/DocumentUpdateProcessor.ts new file mode 100644 index 0000000000..496da7e923 --- /dev/null +++ b/packages/backend-core/src/events/processors/DocumentUpdateProcessor.ts @@ -0,0 +1,42 @@ +import { Event, Identity, Group, DocumentUpdateEvents } from "@budibase/types" +import { EventProcessor } from "./types" +import * as docUpdates from "../../docUpdates" +import { getTenantId } from "../../context" + +export default class DocumentUpdateProcessor implements EventProcessor { + async processEvent( + event: Event, + identity: Identity, + properties: any, + timestamp?: string + ): Promise { + // only user and group IDs supported right now - no app documents yet + if (DocumentUpdateEvents.indexOf(event) !== -1 && identity.tenantId) { + await docUpdates.update({ + id: this.getId(properties), + tenantId: getTenantId(), + }) + } + } + + getId(properties: any) { + let possibleProps = ["groupId", "userId"] + for (let prop of possibleProps) { + if (properties[prop]) { + return properties[prop] + } + } + } + + async identify(identity: Identity, timestamp?: string | number) { + // no-op + } + + async identifyGroup(group: Group, timestamp?: string | number) { + // no-op + } + + shutdown(): void { + docUpdates.shutdown() + } +} diff --git a/packages/backend-core/src/events/processors/index.ts b/packages/backend-core/src/events/processors/index.ts index 6646764e47..3582838d31 100644 --- a/packages/backend-core/src/events/processors/index.ts +++ b/packages/backend-core/src/events/processors/index.ts @@ -1,12 +1,14 @@ import AnalyticsProcessor from "./AnalyticsProcessor" import LoggingProcessor from "./LoggingProcessor" import AuditLogsProcessor from "./AuditLogsProcessor" +import DocumentUpdateProcessor from "./DocumentUpdateProcessor" import Processors from "./Processors" import { AuditLogFn } from "@budibase/types" export const analyticsProcessor = new AnalyticsProcessor() const loggingProcessor = new LoggingProcessor() const auditLogsProcessor = new AuditLogsProcessor() +const documentUpdateProcessor = new DocumentUpdateProcessor() export function init(auditingFn: AuditLogFn) { return AuditLogsProcessor.init(auditingFn) @@ -16,4 +18,5 @@ export const processors = new Processors([ analyticsProcessor, loggingProcessor, auditLogsProcessor, + documentUpdateProcessor, ]) diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 30072196ba..40233b3827 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -27,6 +27,7 @@ export * as errors from "./errors" export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" +export * as docUpdates from "./docUpdates" export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index 9261ed1176..72d2e4742c 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -2,4 +2,5 @@ export enum JobQueue { AUTOMATION = "automationQueue", APP_BACKUP = "appBackupQueue", AUDIT_LOG = "auditLogQueue", + DOC_UPDATE = "docUpdateQueue", } diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 6fb3576ae6..682fd7b519 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -1,7 +1,21 @@ import env from "../../../environment" -import { db as dbCore, context } from "@budibase/backend-core" +import { + db as dbCore, + context, + docUpdates, + constants, +} from "@budibase/backend-core" import sdk from "../../" +export function initUserGroupSync() { + const types = [constants.DocumentType.USER, constants.DocumentType.GROUP] + docUpdates.process(types, async update => { + console.log("syncing - ", JSON.stringify(update)) + // TODO: make the sync smarter + await sdk.users.syncGlobalUsers() + }) +} + export async function syncApp( appId: string, opts?: { automationOnly?: boolean } diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index 6cdbf87c2c..77164f9a75 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -64,6 +64,7 @@ export async function startup(app?: any, server?: any) { eventEmitter.emitPort(env.PORT) fileSystem.init() await redis.init() + sdk.applications.initUserGroupSync() // run migrations on startup if not done via http // not recommended in a clustered environment diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 0d59576435..92965fa533 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -186,6 +186,22 @@ export enum Event { AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded", } +export const DocumentUpdateEvents: Event[] = [ + Event.USER_CREATED, + Event.USER_UPDATED, + Event.USER_DELETED, + Event.USER_PERMISSION_ADMIN_ASSIGNED, + Event.USER_PERMISSION_ADMIN_REMOVED, + Event.USER_PERMISSION_BUILDER_ASSIGNED, + Event.USER_PERMISSION_BUILDER_REMOVED, + Event.USER_GROUP_CREATED, + Event.USER_GROUP_UPDATED, + Event.USER_GROUP_DELETED, + Event.USER_GROUP_USERS_ADDED, + Event.USER_GROUP_USERS_REMOVED, + Event.USER_GROUP_PERMISSIONS_EDITED, +] + // all events that are not audited have been added to this record as undefined, this means // that Typescript can protect us against new events being added and auditing of those // events not being considered. This might be a little ugly, but provides a level of From a2cabb01d5253c4bfeb52b877915878dbe017334 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 4 Apr 2023 13:53:30 +0100 Subject: [PATCH 02/19] Removing user app sync calls from worker - no longer required. --- packages/worker/src/sdk/users/users.ts | 10 ----- packages/worker/src/utilities/appService.ts | 46 --------------------- 2 files changed, 56 deletions(-) delete mode 100644 packages/worker/src/utilities/appService.ts diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index f05c6b98d2..efc683302e 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -1,5 +1,4 @@ import env from "../../environment" -import * as apps from "../../utilities/appService" import * as eventHelpers from "./events" import { accounts, @@ -29,7 +28,6 @@ import { PlatformUser, PlatformUserByEmail, RowResponse, - SearchUsersRequest, User, SaveUserOpts, } from "@budibase/types" @@ -275,9 +273,6 @@ export const save = async ( await platform.users.addUser(tenantId, builtUser._id!, builtUser.email) await cache.user.invalidateUser(response.id) - // let server know to sync user - await apps.syncUserInApps(_id, dbUser) - await Promise.all(groupPromises) // finally returned the saved user from the db @@ -424,7 +419,6 @@ export const bulkCreate = async ( // instead of relying on looping tenant creation await platform.users.addUser(tenantId, user._id, user.email) await eventHelpers.handleSaveEvents(user, undefined) - await apps.syncUserInApps(user._id) } const saved = usersToBulkSave.map(user => { @@ -563,8 +557,6 @@ export const destroy = async (id: string) => { await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) await sessions.invalidateSessions(userId, { reason: "deletion" }) - // let server know to sync user - await apps.syncUserInApps(userId, dbUser) } const bulkDeleteProcessing = async (dbUser: User) => { @@ -573,8 +565,6 @@ const bulkDeleteProcessing = async (dbUser: User) => { await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) await sessions.invalidateSessions(userId, { reason: "bulk-deletion" }) - // let server know to sync user - await apps.syncUserInApps(userId, dbUser) } export const invite = async ( diff --git a/packages/worker/src/utilities/appService.ts b/packages/worker/src/utilities/appService.ts deleted file mode 100644 index 8f411d58fa..0000000000 --- a/packages/worker/src/utilities/appService.ts +++ /dev/null @@ -1,46 +0,0 @@ -import fetch from "node-fetch" -import { - constants, - tenancy, - logging, - env as coreEnv, -} from "@budibase/backend-core" -import { checkSlashesInUrl } from "../utilities" -import env from "../environment" -import { SyncUserRequest, User } from "@budibase/types" - -async function makeAppRequest(url: string, method: string, body: any) { - if (env.isTest()) { - return - } - const request: any = { headers: {} } - request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY - if (tenancy.isTenantIdSet()) { - request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId() - } - if (body) { - request.headers["Content-Type"] = "application/json" - request.body = JSON.stringify(body) - } - request.method = method - - // add x-budibase-correlation-id header - logging.correlation.setHeader(request.headers) - - return fetch(checkSlashesInUrl(env.APPS_URL + url), request) -} - -export async function syncUserInApps(userId: string, previousUser?: User) { - const body: SyncUserRequest = { - previousUser, - } - - const response = await makeAppRequest( - `/api/users/metadata/sync/${userId}`, - "POST", - body - ) - if (response && response.status !== 200) { - throw "Unable to sync user." - } -} From e699f4684a7423f1809a221be6192c434c624d6d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 4 Apr 2023 18:03:56 +0100 Subject: [PATCH 03/19] Updating the global user sync to be more accurate and also remove old user metadata from apps that users don't have access to anymore. --- packages/server/src/api/controllers/user.ts | 106 ++----------- packages/server/src/api/routes/user.ts | 5 - .../server/src/sdk/app/applications/sync.ts | 143 +++++++++++++++--- packages/server/src/sdk/users/utils.ts | 23 ++- packages/server/src/utilities/global.ts | 11 +- 5 files changed, 151 insertions(+), 137 deletions(-) diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index 1ae1a68824..b66f11bc1c 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -1,98 +1,12 @@ import { generateUserMetadataID, generateUserFlagID } from "../../db/utils" import { InternalTables } from "../../db/utils" -import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global" +import { getGlobalUsers } from "../../utilities/global" import { getFullUser } from "../../utilities/users" -import { - context, - roles as rolesCore, - db as dbCore, -} from "@budibase/backend-core" -import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types" +import { context } from "@budibase/backend-core" +import { UserCtx } from "@budibase/types" import sdk from "../../sdk" -export async function syncUser(ctx: Ctx) { - let deleting = false, - user: User | any - const userId = ctx.params.id - - const previousUser = ctx.request.body?.previousUser - - try { - user = (await getRawGlobalUser(userId)) as User - } catch (err: any) { - if (err && err.status === 404) { - user = {} - deleting = true - } else { - throw err - } - } - - let previousApps = previousUser - ? Object.keys(previousUser.roles).map(appId => appId) - : [] - - const roles = deleting ? {} : user.roles - // remove props which aren't useful to metadata - delete user.password - delete user.forceResetPassword - delete user.roles - // run through all production appIDs in the users roles - let prodAppIds - // if they are a builder then get all production app IDs - if ((user.builder && user.builder.global) || deleting) { - prodAppIds = await dbCore.getProdAppIDs() - } else { - prodAppIds = Object.entries(roles) - .filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC) - .map(([appId]) => appId) - } - for (let prodAppId of new Set([...prodAppIds, ...previousApps])) { - const roleId = roles[prodAppId] - const deleteFromApp = !roleId - const devAppId = dbCore.getDevelopmentAppID(prodAppId) - for (let appId of [prodAppId, devAppId]) { - if (!(await dbCore.dbExists(appId))) { - continue - } - await context.doInAppContext(appId, async () => { - const db = context.getAppDB() - const metadataId = generateUserMetadataID(userId) - let metadata - try { - metadata = await db.get(metadataId) - } catch (err) { - if (deleteFromApp) { - return - } - metadata = { - tableId: InternalTables.USER_METADATA, - } - } - - if (deleteFromApp) { - await db.remove(metadata) - return - } - - // assign the roleId for the metadata doc - if (roleId) { - metadata.roleId = roleId - } - let combined = sdk.users.combineMetadataAndUser(user, metadata) - // if its null then there was no updates required - if (combined) { - await db.put(combined) - } - }) - } - } - ctx.body = { - message: "User synced.", - } -} - -export async function fetchMetadata(ctx: BBContext) { +export async function fetchMetadata(ctx: UserCtx) { const global = await getGlobalUsers() const metadata = await sdk.users.rawUserMetadata() const users = [] @@ -111,7 +25,7 @@ export async function fetchMetadata(ctx: BBContext) { ctx.body = users } -export async function updateSelfMetadata(ctx: BBContext) { +export async function updateSelfMetadata(ctx: UserCtx) { // overwrite the ID with current users ctx.request.body._id = ctx.user?._id // make sure no stale rev @@ -121,7 +35,7 @@ export async function updateSelfMetadata(ctx: BBContext) { await updateMetadata(ctx) } -export async function updateMetadata(ctx: BBContext) { +export async function updateMetadata(ctx: UserCtx) { const db = context.getAppDB() const user = ctx.request.body // this isn't applicable to the user @@ -133,7 +47,7 @@ export async function updateMetadata(ctx: BBContext) { ctx.body = await db.put(metadata) } -export async function destroyMetadata(ctx: BBContext) { +export async function destroyMetadata(ctx: UserCtx) { const db = context.getAppDB() try { const dbUser = await db.get(ctx.params.id) @@ -146,11 +60,11 @@ export async function destroyMetadata(ctx: BBContext) { } } -export async function findMetadata(ctx: BBContext) { +export async function findMetadata(ctx: UserCtx) { ctx.body = await getFullUser(ctx, ctx.params.id) } -export async function setFlag(ctx: BBContext) { +export async function setFlag(ctx: UserCtx) { const userId = ctx.user?._id const { flag, value } = ctx.request.body if (!flag) { @@ -169,7 +83,7 @@ export async function setFlag(ctx: BBContext) { ctx.body = { message: "Flag set successfully" } } -export async function getFlags(ctx: BBContext) { +export async function getFlags(ctx: UserCtx) { const userId = ctx.user?._id const docId = generateUserFlagID(userId!) const db = context.getAppDB() diff --git a/packages/server/src/api/routes/user.ts b/packages/server/src/api/routes/user.ts index 14deb111e6..24f33140a6 100644 --- a/packages/server/src/api/routes/user.ts +++ b/packages/server/src/api/routes/user.ts @@ -32,11 +32,6 @@ router authorized(PermissionType.USER, PermissionLevel.WRITE), controller.destroyMetadata ) - .post( - "/api/users/metadata/sync/:id", - authorized(PermissionType.USER, PermissionLevel.WRITE), - controller.syncUser - ) .post( "/api/users/flags", authorized(PermissionType.USER, PermissionLevel.WRITE), diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 682fd7b519..3572e047e6 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -4,15 +4,114 @@ import { context, docUpdates, constants, + logging, + roles, } from "@budibase/backend-core" +import { User, ContextUser } from "@budibase/types" import sdk from "../../" +import { getGlobalUsers, updateAppRole } from "../../../utilities/global" +import { generateUserMetadataID, InternalTables } from "../../../db/utils" + +type DeletedUser = { _id: string; deleted: boolean } + +async function syncUsersToApp(appId: string, users: (User | DeletedUser)[]) { + if (!(await dbCore.dbExists(appId))) { + return + } + await context.doInAppContext(appId, async () => { + const db = context.getAppDB() + for (let user of users) { + let ctxUser = user as ContextUser + let deletedUser = false + const metadataId = generateUserMetadataID(user._id!) + if ((user as DeletedUser).deleted) { + deletedUser = true + } + + // make sure role is correct + if (!deletedUser) { + ctxUser = updateAppRole(ctxUser, { appId }) + } + let roleId = ctxUser.roleId + if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) { + roleId = undefined + } + + let metadata + try { + metadata = await db.get(metadataId) + } catch (err: any) { + if (err.status !== 404) { + throw err + } + // no metadata and user is to be deleted, can skip + // no role - user isn't in app anyway + if (!roleId) { + continue + } else if (!deletedUser) { + // doesn't exist yet, creating it + metadata = { + tableId: InternalTables.USER_METADATA, + } + } + } + + // the user doesn't exist, or doesn't have a role anymore + // get rid of their metadata + if (deletedUser || !roleId) { + await db.remove(metadata) + continue + } + + // assign the roleId for the metadata doc + if (roleId) { + metadata.roleId = roleId + } + + let combined = sdk.users.combineMetadataAndUser(ctxUser, metadata) + // if no combined returned, there are no updates to make + if (combined) { + await db.put(combined) + } + } + }) +} + +async function syncUsersToAllApps(userIds: string[]) { + // list of users, if one has been deleted it will be undefined in array + const users = (await getGlobalUsers(userIds)) as User[] + const finalUsers: (User | DeletedUser)[] = [] + for (let userId of userIds) { + if (!users.find(user => user._id === userId)) { + finalUsers.push({ _id: userId, deleted: true }) + } + } + const devAppIds = await dbCore.getDevAppIDs() + let promises = [] + for (let devAppId of devAppIds) { + const prodAppId = dbCore.getProdAppID(devAppId) + for (let appId of [prodAppId, devAppId]) { + promises.push(syncUsersToApp(appId, finalUsers)) + } + } + const resp = await Promise.allSettled(promises) + const failed = resp.filter(promise => promise.status === "rejected") + if (failed.length > 0) { + const reasons = failed.map(fail => (fail as PromiseRejectedResult).reason) + logging.logAlert("Failed to sync users to apps", reasons) + } +} export function initUserGroupSync() { const types = [constants.DocumentType.USER, constants.DocumentType.GROUP] docUpdates.process(types, async update => { - console.log("syncing - ", JSON.stringify(update)) - // TODO: make the sync smarter - await sdk.users.syncGlobalUsers() + const docId = update.id + const isGroup = docId.startsWith(constants.DocumentType.GROUP) + if (isGroup) { + // TODO: get the group, get users in the group then run the function + } else { + await syncUsersToAllApps([docId]) + } }) } @@ -37,32 +136,28 @@ export async function syncApp( // specific case, want to make sure setup is skipped const prodDb = context.getProdAppDB({ skip_setup: true }) const exists = await prodDb.exists() - if (!exists) { - // the database doesn't exist. Don't replicate - return { - message: "App sync not required, app not deployed.", - } - } - const replication = new dbCore.Replication({ - source: prodAppId, - target: appId, - }) let error - try { - const replOpts = replication.appReplicateOpts() - if (opts?.automationOnly) { - replOpts.filter = (doc: any) => - doc._id.startsWith(dbCore.DocumentType.AUTOMATION) + if (exists) { + const replication = new dbCore.Replication({ + source: prodAppId, + target: appId, + }) + try { + const replOpts = replication.appReplicateOpts() + if (opts?.automationOnly) { + replOpts.filter = (doc: any) => + doc._id.startsWith(dbCore.DocumentType.AUTOMATION) + } + await replication.replicate(replOpts) + } catch (err) { + error = err + } finally { + await replication.close() } - await replication.replicate(replOpts) - } catch (err) { - error = err - } finally { - await replication.close() } - // sync the users + // sync the users - kept for safe keeping await sdk.users.syncGlobalUsers() if (error) { diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index 9b9ea04c56..a3982fd45b 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -1,12 +1,13 @@ import { getGlobalUsers } from "../../utilities/global" import { context, roles as rolesCore } from "@budibase/backend-core" import { + getGlobalIDFromUserMetadataID, generateUserMetadataID, getUserMetadataParams, InternalTables, } from "../../db/utils" import { isEqual } from "lodash" -import { ContextUser, UserMetadata } from "@budibase/types" +import { ContextUser, UserMetadata, User } from "@budibase/types" export function combineMetadataAndUser( user: ContextUser, @@ -37,6 +38,10 @@ export function combineMetadataAndUser( if (found) { newDoc._rev = found._rev } + // clear fields that shouldn't be in metadata + delete newDoc.password + delete newDoc.forceResetPassword + delete newDoc.roles if (found == null || !isEqual(newDoc, found)) { return { ...found, @@ -60,10 +65,9 @@ export async function rawUserMetadata() { export async function syncGlobalUsers() { // sync user metadata const db = context.getAppDB() - const [users, metadata] = await Promise.all([ - getGlobalUsers(), - rawUserMetadata(), - ]) + const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()]) + const users = resp[0] as User[] + const metadata = resp[1] as UserMetadata[] const toWrite = [] for (let user of users) { const combined = combineMetadataAndUser(user, metadata) @@ -71,5 +75,14 @@ export async function syncGlobalUsers() { toWrite.push(combined) } } + for (let data of metadata) { + if (!data._id) { + continue + } + const globalId = getGlobalIDFromUserMetadataID(data._id) + if (!users.find(user => user._id === globalId)) { + toWrite.push({ ...data, _deleted: true }) + } + } await db.bulkDocs(toWrite) } diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index a75fcc0b30..0debb68d54 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -94,16 +94,13 @@ export async function getGlobalUser(userId: string) { return processUser(user, { appId }) } -export async function getGlobalUsers(users?: ContextUser[]) { +export async function getGlobalUsers(userIds?: string[]) { const appId = context.getAppId() const db = tenancy.getGlobalDB() const allGroups = await groups.fetch() let globalUsers - if (users) { - const globalIds = users.map(user => - getGlobalIDFromUserMetadataID(user._id!) - ) - globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map( + if (userIds) { + globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( row => row.doc ) } else { @@ -134,7 +131,7 @@ export async function getGlobalUsers(users?: ContextUser[]) { } export async function getGlobalUsersFromMetadata(users: ContextUser[]) { - const globalUsers = await getGlobalUsers(users) + const globalUsers = await getGlobalUsers(users.map(user => user._id!)) return users.map(user => { const globalUser = globalUsers.find( globalUser => globalUser && user._id?.includes(globalUser._id) From 70c9d06832b63fcff55a8ccdbaa9a3432efca30f Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Wed, 5 Apr 2023 20:51:47 +0100 Subject: [PATCH 04/19] Adding group management to background user/group sync. --- packages/server/src/sdk/app/applications/sync.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 3572e047e6..ce59740f2a 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -8,6 +8,7 @@ import { roles, } from "@budibase/backend-core" import { User, ContextUser } from "@budibase/types" +import { sdk as proSdk } from "@budibase/pro" import sdk from "../../" import { getGlobalUsers, updateAppRole } from "../../../utilities/global" import { generateUserMetadataID, InternalTables } from "../../../db/utils" @@ -107,10 +108,15 @@ export function initUserGroupSync() { docUpdates.process(types, async update => { const docId = update.id const isGroup = docId.startsWith(constants.DocumentType.GROUP) + let userIds: string[] if (isGroup) { - // TODO: get the group, get users in the group then run the function + const group = await proSdk.groups.get(docId) + userIds = group.users?.map(user => user._id) || [] } else { - await syncUsersToAllApps([docId]) + userIds = [docId] + } + if (userIds.length > 0) { + await syncUsersToAllApps(userIds) } }) } From 0771ec55fbee2578daf2e42bff476ad764432ad1 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Wed, 5 Apr 2023 21:22:50 +0100 Subject: [PATCH 05/19] Small change to make sure no duplicates ever occur. --- packages/server/src/sdk/users/utils.ts | 7 ++++++- packages/types/src/documents/app/user.ts | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index a3982fd45b..7f7f0b4809 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -75,14 +75,19 @@ export async function syncGlobalUsers() { toWrite.push(combined) } } + let foundEmails: string[] = [] for (let data of metadata) { if (!data._id) { continue } + const alreadyExisting = data.email && foundEmails.indexOf(data.email) !== -1 const globalId = getGlobalIDFromUserMetadataID(data._id) - if (!users.find(user => user._id === globalId)) { + if (!users.find(user => user._id === globalId) || alreadyExisting) { toWrite.push({ ...data, _deleted: true }) } + if (data.email) { + foundEmails.push(data.email) + } } await db.bulkDocs(toWrite) } diff --git a/packages/types/src/documents/app/user.ts b/packages/types/src/documents/app/user.ts index b5f31ca349..4defd4a414 100644 --- a/packages/types/src/documents/app/user.ts +++ b/packages/types/src/documents/app/user.ts @@ -2,4 +2,5 @@ import { Document } from "../document" export interface UserMetadata extends Document { roleId: string + email?: string } From 58d0a8210897a72b06d959aeee218d8c29fa20b9 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Wed, 5 Apr 2023 22:35:01 +0100 Subject: [PATCH 06/19] First sync test, checking that a user is created correctly. --- .../server/src/sdk/app/applications/sync.ts | 11 +++- .../sdk/app/applications/tests/sync.spec.ts | 61 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 packages/server/src/sdk/app/applications/tests/sync.spec.ts diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index ce59740f2a..4a0c782ebd 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -83,8 +83,11 @@ async function syncUsersToAllApps(userIds: string[]) { const users = (await getGlobalUsers(userIds)) as User[] const finalUsers: (User | DeletedUser)[] = [] for (let userId of userIds) { - if (!users.find(user => user._id === userId)) { + const user = users.find(user => user._id === userId) + if (!user) { finalUsers.push({ _id: userId, deleted: true }) + } else { + finalUsers.push(user) } } const devAppIds = await dbCore.getDevAppIDs() @@ -103,7 +106,7 @@ async function syncUsersToAllApps(userIds: string[]) { } } -export function initUserGroupSync() { +export function initUserGroupSync(updateCb?: () => void) { const types = [constants.DocumentType.USER, constants.DocumentType.GROUP] docUpdates.process(types, async update => { const docId = update.id @@ -118,6 +121,10 @@ export function initUserGroupSync() { if (userIds.length > 0) { await syncUsersToAllApps(userIds) } + // used to tracking when updates have occurred + if (updateCb) { + updateCb() + } }) } diff --git a/packages/server/src/sdk/app/applications/tests/sync.spec.ts b/packages/server/src/sdk/app/applications/tests/sync.spec.ts new file mode 100644 index 0000000000..9ad6fe2eef --- /dev/null +++ b/packages/server/src/sdk/app/applications/tests/sync.spec.ts @@ -0,0 +1,61 @@ +import TestConfiguration from "../../../../tests/utilities/TestConfiguration" +import { events, context, roles, db as dbCore } from "@budibase/backend-core" +import { initUserGroupSync } from "../sync" +import { rawUserMetadata } from "../../../users/utils" +import EventEmitter from "events" +import { UserMetadata, UserRoles } from "@budibase/types" + +const config = new TestConfiguration() +let app +const ROLE_ID = roles.BUILTIN_ROLE_IDS.BASIC + +const emitter = new EventEmitter() + +function updateCb() { + emitter.emit("update") +} + +function waitForUpdate() { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject() + }, 5000) + emitter.on("update", () => { + clearTimeout(timeout) + resolve() + }) + }) +} + +beforeAll(async () => { + app = await config.init("syncApp") + initUserGroupSync(updateCb) +}) + +async function createUser(email: string, roles: UserRoles, appId?: string) { + const user = await config.createUser({ email, roles }) + await context.doInContext(appId || config.appId!, async () => { + await events.user.created(user) + }) +} + +async function getUserMetadata(appId?: string): Promise { + return context.doInContext(appId || config.appId!, async () => { + return await rawUserMetadata() + }) +} + +function buildRoles(appId?: string) { + const prodAppId = dbCore.getProdAppID(appId || config.appId!) + return { [prodAppId]: ROLE_ID } +} + +describe("app user/group sync", () => { + it("should be able to sync a new user", async () => { + const email = "test@test.com" + await createUser(email, buildRoles()) + await waitForUpdate() + const metadata = await getUserMetadata() + expect(metadata.find(data => data.email === email)).toBeDefined() + }) +}) From e623820478a84043678de36a298b05075b8ddb92 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Apr 2023 17:33:52 +0100 Subject: [PATCH 07/19] Adding test cases. --- .../server/src/sdk/app/applications/tests/sync.spec.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/server/src/sdk/app/applications/tests/sync.spec.ts b/packages/server/src/sdk/app/applications/tests/sync.spec.ts index 9ad6fe2eef..a98297dbe3 100644 --- a/packages/server/src/sdk/app/applications/tests/sync.spec.ts +++ b/packages/server/src/sdk/app/applications/tests/sync.spec.ts @@ -58,4 +58,12 @@ describe("app user/group sync", () => { const metadata = await getUserMetadata() expect(metadata.find(data => data.email === email)).toBeDefined() }) + + it("should be able to sync a group", async () => { + + }) + + it("should be able to remove user", async () => { + + }) }) From 23e99ca4b7061deba2d67ca172e24fa050b09480 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Apr 2023 19:59:05 +0100 Subject: [PATCH 08/19] Adding group and user tests for user sync. --- packages/backend-core/src/db/views.ts | 20 +++- .../server/src/sdk/app/applications/sync.ts | 59 ++++++---- .../sdk/app/applications/tests/sync.spec.ts | 101 ++++++++++++++---- .../src/tests/utilities/TestConfiguration.ts | 28 +++++ packages/server/src/utilities/global.ts | 28 +++-- 5 files changed, 180 insertions(+), 56 deletions(-) diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 8a2c2e7efd..f0057cd7c3 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -42,7 +42,11 @@ async function removeDeprecated(db: Database, viewName: ViewName) { } } -export async function createView(db: any, viewJs: string, viewName: string) { +export async function createView( + db: any, + viewJs: string, + viewName: string +): Promise { let designDoc try { designDoc = (await db.get(DESIGN_DB)) as DesignDocument @@ -57,7 +61,15 @@ export async function createView(db: any, viewJs: string, viewName: string) { ...designDoc.views, [viewName]: view, } - await db.put(designDoc) + try { + await db.put(designDoc) + } catch (err: any) { + if (err.status === 409) { + return await createView(db, viewJs, viewName) + } else { + throw err + } + } } export const createNewUserEmailView = async () => { @@ -135,6 +147,10 @@ export const queryView = async ( await removeDeprecated(db, viewName) await createFunc() return queryView(viewName, params, db, createFunc, opts) + } else if (err.status === 409) { + // can happen when multiple queries occur at once, view couldn't be created + // other design docs being updated, re-run + return queryView(viewName, params, db, createFunc, opts) } else { throw err } diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 4a0c782ebd..66fd5d2d59 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -7,15 +7,19 @@ import { logging, roles, } from "@budibase/backend-core" -import { User, ContextUser } from "@budibase/types" +import { User, ContextUser, UserGroup } from "@budibase/types" import { sdk as proSdk } from "@budibase/pro" import sdk from "../../" -import { getGlobalUsers, updateAppRole } from "../../../utilities/global" +import { getGlobalUsers, processUser } from "../../../utilities/global" import { generateUserMetadataID, InternalTables } from "../../../db/utils" type DeletedUser = { _id: string; deleted: boolean } -async function syncUsersToApp(appId: string, users: (User | DeletedUser)[]) { +async function syncUsersToApp( + appId: string, + users: (User | DeletedUser)[], + groups: UserGroup[] +) { if (!(await dbCore.dbExists(appId))) { return } @@ -31,7 +35,7 @@ async function syncUsersToApp(appId: string, users: (User | DeletedUser)[]) { // make sure role is correct if (!deletedUser) { - ctxUser = updateAppRole(ctxUser, { appId }) + ctxUser = await processUser(ctxUser, { appId, groups }) } let roleId = ctxUser.roleId if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) { @@ -80,7 +84,10 @@ async function syncUsersToApp(appId: string, users: (User | DeletedUser)[]) { async function syncUsersToAllApps(userIds: string[]) { // list of users, if one has been deleted it will be undefined in array - const users = (await getGlobalUsers(userIds)) as User[] + const users = (await getGlobalUsers(userIds, { + noProcessing: true, + })) as User[] + const groups = await proSdk.groups.fetch() const finalUsers: (User | DeletedUser)[] = [] for (let userId of userIds) { const user = users.find(user => user._id === userId) @@ -95,7 +102,7 @@ async function syncUsersToAllApps(userIds: string[]) { for (let devAppId of devAppIds) { const prodAppId = dbCore.getProdAppID(devAppId) for (let appId of [prodAppId, devAppId]) { - promises.push(syncUsersToApp(appId, finalUsers)) + promises.push(syncUsersToApp(appId, finalUsers, groups)) } } const resp = await Promise.allSettled(promises) @@ -106,24 +113,32 @@ async function syncUsersToAllApps(userIds: string[]) { } } -export function initUserGroupSync(updateCb?: () => void) { +export function initUserGroupSync(updateCb?: (docId: string) => void) { const types = [constants.DocumentType.USER, constants.DocumentType.GROUP] docUpdates.process(types, async update => { - const docId = update.id - const isGroup = docId.startsWith(constants.DocumentType.GROUP) - let userIds: string[] - if (isGroup) { - const group = await proSdk.groups.get(docId) - userIds = group.users?.map(user => user._id) || [] - } else { - userIds = [docId] - } - if (userIds.length > 0) { - await syncUsersToAllApps(userIds) - } - // used to tracking when updates have occurred - if (updateCb) { - updateCb() + try { + const docId = update.id + const isGroup = docId.startsWith(constants.DocumentType.GROUP) + let userIds: string[] + if (isGroup) { + const group = await proSdk.groups.get(docId) + userIds = group.users?.map(user => user._id) || [] + } else { + userIds = [docId] + } + if (userIds.length > 0) { + await syncUsersToAllApps(userIds) + } + if (updateCb) { + updateCb(docId) + } + } catch (err: any) { + // if something not found - no changes to perform + if (err?.status === 404) { + return + } else { + logging.logAlert("Failed to perform user/group app sync", err) + } } }) } diff --git a/packages/server/src/sdk/app/applications/tests/sync.spec.ts b/packages/server/src/sdk/app/applications/tests/sync.spec.ts index a98297dbe3..a2ea2a485a 100644 --- a/packages/server/src/sdk/app/applications/tests/sync.spec.ts +++ b/packages/server/src/sdk/app/applications/tests/sync.spec.ts @@ -1,26 +1,32 @@ import TestConfiguration from "../../../../tests/utilities/TestConfiguration" -import { events, context, roles, db as dbCore } from "@budibase/backend-core" +import { events, context, roles, constants } from "@budibase/backend-core" import { initUserGroupSync } from "../sync" import { rawUserMetadata } from "../../../users/utils" import EventEmitter from "events" -import { UserMetadata, UserRoles } from "@budibase/types" +import { UserGroup, UserMetadata, UserRoles, User } from "@budibase/types" const config = new TestConfiguration() -let app +let app, group: UserGroup, groupUser: User const ROLE_ID = roles.BUILTIN_ROLE_IDS.BASIC const emitter = new EventEmitter() -function updateCb() { - emitter.emit("update") +function updateCb(docId: string) { + const isGroup = docId.startsWith(constants.DocumentType.GROUP) + if (isGroup) { + emitter.emit("update-group") + } else { + emitter.emit("update-user") + } } -function waitForUpdate() { +function waitForUpdate(opts: { group?: boolean }) { return new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject() }, 5000) - emitter.on("update", () => { + const event = opts?.group ? "update-group" : "update-user" + emitter.on(event, () => { clearTimeout(timeout) resolve() }) @@ -32,38 +38,89 @@ beforeAll(async () => { initUserGroupSync(updateCb) }) -async function createUser(email: string, roles: UserRoles, appId?: string) { +async function createUser(email: string, roles: UserRoles) { const user = await config.createUser({ email, roles }) - await context.doInContext(appId || config.appId!, async () => { + await context.doInContext(config.appId!, async () => { await events.user.created(user) }) + return user +} + +async function removeUserRole(user: User) { + const final = await config.globalUser({ + ...user, + id: user._id, + roles: {}, + builder: false, + admin: false, + }) + await context.doInContext(config.appId!, async () => { + await events.user.updated(final) + }) } -async function getUserMetadata(appId?: string): Promise { - return context.doInContext(appId || config.appId!, async () => { +async function createGroupAndUser(email: string) { + groupUser = await config.createUser({ + email, + roles: {}, + builder: false, + admin: false, + }) + group = await config.createGroup() + await config.addUserToGroup(group._id!, groupUser._id!) +} + +async function removeUserFromGroup() { + await config.removeUserFromGroup(group._id!, groupUser._id!) + return context.doInContext(config.appId!, async () => { + await events.user.updated(groupUser) + }) +} + +async function getUserMetadata(): Promise { + return context.doInContext(config.appId!, async () => { return await rawUserMetadata() }) } -function buildRoles(appId?: string) { - const prodAppId = dbCore.getProdAppID(appId || config.appId!) - return { [prodAppId]: ROLE_ID } +function buildRoles() { + return { [config.prodAppId!]: ROLE_ID } } describe("app user/group sync", () => { - it("should be able to sync a new user", async () => { - const email = "test@test.com" - await createUser(email, buildRoles()) - await waitForUpdate() + const groupEmail = "test2@test.com", + normalEmail = "test@test.com" + async function checkEmail( + email: string, + opts?: { group?: boolean; notFound?: boolean } + ) { + await waitForUpdate(opts || {}) const metadata = await getUserMetadata() - expect(metadata.find(data => data.email === email)).toBeDefined() + const found = metadata.find(data => data.email === email) + if (opts?.notFound) { + expect(found).toBeUndefined() + } else { + expect(found).toBeDefined() + } + } + + it("should be able to sync a new user, add then remove", async () => { + const user = await createUser(normalEmail, buildRoles()) + await checkEmail(normalEmail) + await removeUserRole(user) + await checkEmail(normalEmail, { notFound: true }) }) it("should be able to sync a group", async () => { - + await createGroupAndUser(groupEmail) + await checkEmail(groupEmail, { group: true }) }) - it("should be able to remove user", async () => { - + it("should be able to remove user from group", async () => { + if (!group) { + await createGroupAndUser(groupEmail) + } + await removeUserFromGroup() + await checkEmail(groupEmail, { notFound: true }) }) }) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index ca48dd7f86..f5887e6558 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -49,6 +49,7 @@ import { SearchFilters, UserRoles, } from "@budibase/types" +import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/src/security/roles" type DefaultUserValues = { globalUserId: string @@ -306,6 +307,33 @@ class TestConfiguration { } } + async createGroup(roleId: string = BUILTIN_ROLE_IDS.BASIC) { + return context.doInTenant(this.tenantId!, async () => { + const baseGroup = structures.userGroups.userGroup() + baseGroup.roles = { + [this.prodAppId]: roleId, + } + const { id, rev } = await pro.sdk.groups.save(baseGroup) + return { + _id: id, + _rev: rev, + ...baseGroup, + } + }) + } + + async addUserToGroup(groupId: string, userId: string) { + return context.doInTenant(this.tenantId!, async () => { + await pro.sdk.groups.addUsers(groupId, [userId]) + }) + } + + async removeUserFromGroup(groupId: string, userId: string) { + return context.doInTenant(this.tenantId!, async () => { + await pro.sdk.groups.removeUsers(groupId, [userId]) + }) + } + async login({ roleId, userId, builder, prodApp = false }: any = {}) { const appId = prodApp ? this.prodAppId : this.appId return context.doInAppContext(appId, async () => { diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index 0debb68d54..21e86a28b9 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -9,6 +9,7 @@ import { import env from "../environment" import { groups } from "@budibase/pro" import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types" +import { global } from "yargs" export function updateAppRole( user: ContextUser, @@ -16,7 +17,7 @@ export function updateAppRole( ) { appId = appId || context.getAppId() - if (!user || !user.roles) { + if (!user || (!user.roles && !user.userGroups)) { return user } // if in an multi-tenancy environment make sure roles are never updated @@ -27,7 +28,7 @@ export function updateAppRole( return user } // always use the deployed app - if (appId) { + if (appId && user.roles) { user.roleId = user.roles[dbCore.getProdAppID(appId)] } // if a role wasn't found then either set as admin (builder) or public (everyone else) @@ -60,7 +61,7 @@ async function checkGroupRoles( return user } -async function processUser( +export async function processUser( user: ContextUser, opts: { appId?: string; groups?: UserGroup[] } = {} ) { @@ -94,10 +95,12 @@ export async function getGlobalUser(userId: string) { return processUser(user, { appId }) } -export async function getGlobalUsers(userIds?: string[]) { +export async function getGlobalUsers( + userIds?: string[], + opts?: { noProcessing?: boolean } +) { const appId = context.getAppId() const db = tenancy.getGlobalDB() - const allGroups = await groups.fetch() let globalUsers if (userIds) { globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( @@ -123,11 +126,16 @@ export async function getGlobalUsers(userIds?: string[]) { return globalUsers } - // pass in the groups, meaning we don't actually need to retrieve them for - // each user individually - return Promise.all( - globalUsers.map(user => processUser(user, { groups: allGroups })) - ) + if (opts?.noProcessing) { + return globalUsers + } else { + // pass in the groups, meaning we don't actually need to retrieve them for + // each user individually + const allGroups = await groups.fetch() + return Promise.all( + globalUsers.map(user => processUser(user, { groups: allGroups })) + ) + } } export async function getGlobalUsersFromMetadata(users: ContextUser[]) { From b2d7f24ff30c0fc5f3a403f7ad82e9e46cb5c764 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Apr 2023 20:02:37 +0100 Subject: [PATCH 09/19] Adding builder test case. --- .../src/sdk/app/applications/tests/sync.spec.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/applications/tests/sync.spec.ts b/packages/server/src/sdk/app/applications/tests/sync.spec.ts index a2ea2a485a..90493c541d 100644 --- a/packages/server/src/sdk/app/applications/tests/sync.spec.ts +++ b/packages/server/src/sdk/app/applications/tests/sync.spec.ts @@ -38,8 +38,13 @@ beforeAll(async () => { initUserGroupSync(updateCb) }) -async function createUser(email: string, roles: UserRoles) { - const user = await config.createUser({ email, roles }) +async function createUser(email: string, roles: UserRoles, builder?: boolean) { + const user = await config.createUser({ + email, + roles, + builder: builder || false, + admin: false, + }) await context.doInContext(config.appId!, async () => { await events.user.created(user) }) @@ -123,4 +128,9 @@ describe("app user/group sync", () => { await removeUserFromGroup() await checkEmail(groupEmail, { notFound: true }) }) + + it("should be able to handle builder users", async () => { + await createUser("test3@test.com", {}, true) + await checkEmail("test3@test.com") + }) }) From 727aba9177db2d79b88593e611316599aeb19fc9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Apr 2023 11:19:28 +0100 Subject: [PATCH 10/19] Fixing build. --- .../src/api/controllers/row/internal.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 039f03c015..a0bebc2490 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -30,7 +30,6 @@ import { finaliseRow, updateRelatedFormula } from "./staticFormula" import { csv, json, jsonWithSchema, Format } from "../view/exporters" import { apiFileReturn } from "../../../utilities/fileSystem" import { - Ctx, UserCtx, Database, LinkDocumentValue, @@ -72,7 +71,7 @@ async function getView(db: Database, viewName: string) { return viewInfo } -async function getRawTableData(ctx: Ctx, db: Database, tableId: string) { +async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) { let rows if (tableId === InternalTables.USER_METADATA) { await userController.fetchMetadata(ctx) @@ -188,7 +187,7 @@ export async function save(ctx: UserCtx) { }) } -export async function fetchView(ctx: Ctx) { +export async function fetchView(ctx: UserCtx) { const viewName = decodeURIComponent(ctx.params.viewName) // if this is a table view being looked for just transfer to that @@ -255,7 +254,7 @@ export async function fetchView(ctx: Ctx) { return rows } -export async function fetch(ctx: Ctx) { +export async function fetch(ctx: UserCtx) { const db = context.getAppDB() const tableId = ctx.params.tableId @@ -264,7 +263,7 @@ export async function fetch(ctx: Ctx) { return outputProcessing(table, rows) } -export async function find(ctx: Ctx) { +export async function find(ctx: UserCtx) { const db = dbCore.getDB(ctx.appId) const table = await db.get(ctx.params.tableId) let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId) @@ -272,7 +271,7 @@ export async function find(ctx: Ctx) { return row } -export async function destroy(ctx: Ctx) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const { _id } = ctx.request.body let row = await db.get(_id) @@ -308,7 +307,7 @@ export async function destroy(ctx: Ctx) { return { response, row } } -export async function bulkDestroy(ctx: Ctx) { +export async function bulkDestroy(ctx: UserCtx) { const db = context.getAppDB() const tableId = ctx.params.tableId const table = await db.get(tableId) @@ -347,7 +346,7 @@ export async function bulkDestroy(ctx: Ctx) { return { response: { ok: true }, rows: processedRows } } -export async function search(ctx: Ctx) { +export async function search(ctx: UserCtx) { // Fetch the whole table when running in cypress, as search doesn't work if (!env.COUCH_DB_URL && env.isCypress()) { return { rows: await fetch(ctx) } @@ -387,7 +386,7 @@ export async function search(ctx: Ctx) { return response } -export async function exportRows(ctx: Ctx) { +export async function exportRows(ctx: UserCtx) { const db = context.getAppDB() const table = await db.get(ctx.params.tableId) const rowIds = ctx.request.body.rows @@ -439,7 +438,7 @@ export async function exportRows(ctx: Ctx) { } } -export async function fetchEnrichedRow(ctx: Ctx) { +export async function fetchEnrichedRow(ctx: UserCtx) { const db = context.getAppDB() const tableId = ctx.params.tableId const rowId = ctx.params.rowId From 7da330624fb46ec54b844bb600a3392a5441ce95 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Apr 2023 12:17:29 +0100 Subject: [PATCH 11/19] Fixing build (again). --- packages/server/src/api/controllers/row/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 2cf3b5472f..e96a4fe6ee 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -5,7 +5,7 @@ import { context } from "@budibase/backend-core" import { makeExternalQuery } from "../../../integrations/base/query" import { Row, Table } from "@budibase/types" import { Format } from "../view/exporters" -import { Ctx } from "@budibase/types" +import { UserCtx } from "@budibase/types" import sdk from "../../../sdk" const validateJs = require("validate.js") const { cloneDeep } = require("lodash/fp") @@ -26,7 +26,7 @@ export async function getDatasourceAndQuery(json: any) { return makeExternalQuery(datasource, json) } -export async function findRow(ctx: Ctx, tableId: string, rowId: string) { +export async function findRow(ctx: UserCtx, tableId: string, rowId: string) { const db = context.getAppDB() let row // TODO remove special user case in future From 8f8843aece28d161f46eca1b968782b679adaddf Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Apr 2023 12:53:25 +0100 Subject: [PATCH 12/19] Updating test cases based on new user sync and removing old endpoint testcases (endpoint removed). --- .../server/src/api/routes/tests/user.spec.js | 37 ------------------- .../server/src/sdk/users/tests/utils.spec.ts | 2 +- 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/packages/server/src/api/routes/tests/user.spec.js b/packages/server/src/api/routes/tests/user.spec.js index 0a2b02364a..e8ffd8df2b 100644 --- a/packages/server/src/api/routes/tests/user.spec.js +++ b/packages/server/src/api/routes/tests/user.spec.js @@ -205,41 +205,4 @@ describe("/users", () => { expect(res.body.message).toEqual("Flag set successfully") }) }) - - describe("syncUser", () => { - it("should sync the user", async () => { - let user = await config.createUser() - await config.createApp("New App") - let res = await request - .post(`/api/users/metadata/sync/${user._id}`) - .set(config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.message).toEqual("User synced.") - }) - - it("should sync the user when a previous user is specified", async () => { - const app1 = await config.createApp("App 1") - const app2 = await config.createApp("App 2") - - let user = await config.createUser({ - builder: false, - admin: true, - roles: { [app1.appId]: "ADMIN" }, - }) - let res = await request - .post(`/api/users/metadata/sync/${user._id}`) - .set(config.defaultHeaders()) - .send({ - previousUser: { - ...user, - roles: { ...user.roles, [app2.appId]: "BASIC" }, - }, - }) - .expect(200) - .expect("Content-Type", /json/) - - expect(res.body.message).toEqual("User synced.") - }) - }) }) diff --git a/packages/server/src/sdk/users/tests/utils.spec.ts b/packages/server/src/sdk/users/tests/utils.spec.ts index 11c2c53643..9e7b078ab5 100644 --- a/packages/server/src/sdk/users/tests/utils.spec.ts +++ b/packages/server/src/sdk/users/tests/utils.spec.ts @@ -121,7 +121,7 @@ describe("syncGlobalUsers", () => { await syncGlobalUsers() const metadata = await rawUserMetadata() - expect(metadata).toHaveLength(1) + expect(metadata).toHaveLength(0) }) }) }) From f87974caf1cd2717e2459db514feb7954f094bd6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Apr 2023 13:10:51 +0100 Subject: [PATCH 13/19] Removing duplicate test case. --- .../server/src/sdk/users/tests/utils.spec.ts | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/packages/server/src/sdk/users/tests/utils.spec.ts b/packages/server/src/sdk/users/tests/utils.spec.ts index 9e7b078ab5..5c6777df59 100644 --- a/packages/server/src/sdk/users/tests/utils.spec.ts +++ b/packages/server/src/sdk/users/tests/utils.spec.ts @@ -125,35 +125,4 @@ describe("syncGlobalUsers", () => { }) }) }) - - it("app users are removed when app is removed from user group", async () => { - await config.doInTenant(async () => { - const group = await proSdk.groups.save(structures.userGroups.userGroup()) - const user1 = await config.createUser({ admin: false, builder: false }) - const user2 = await config.createUser({ admin: false, builder: false }) - await proSdk.groups.updateGroupApps(group.id, { - appsToAdd: [ - { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC }, - ], - }) - await proSdk.groups.addUsers(group.id, [user1._id, user2._id]) - - await config.doInContext(config.appId, async () => { - await syncGlobalUsers() - expect(await rawUserMetadata()).toHaveLength(3) - - await proSdk.groups.removeUsers(group.id, [user1._id]) - await syncGlobalUsers() - - const metadata = await rawUserMetadata() - expect(metadata).toHaveLength(2) - - expect(metadata).not.toContainEqual( - expect.objectContaining({ - _id: db.generateUserMetadataID(user1._id), - }) - ) - }) - }) - }) }) From 2629817741af50df75cbc41ea7a9168d80eef4b8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Apr 2023 13:30:47 +0100 Subject: [PATCH 14/19] Fixing qa-core message now that sync always occurs (dev and prod). --- qa-core/src/internal-api/tests/applications/publish.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa-core/src/internal-api/tests/applications/publish.spec.ts b/qa-core/src/internal-api/tests/applications/publish.spec.ts index 3d5fa7c598..e614a0f2a4 100644 --- a/qa-core/src/internal-api/tests/applications/publish.spec.ts +++ b/qa-core/src/internal-api/tests/applications/publish.spec.ts @@ -36,7 +36,7 @@ describe("Internal API - Application creation, update, publish and delete", () = const [syncResponse, sync] = await config.api.apps.sync(app.appId!) expect(sync).toEqual({ - message: "App sync not required, app not deployed.", + message: "App sync completed successfully.", }) }) From ef5bcc4b6661cd0807ed5ba3f9bb09e6084fa5aa Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Sat, 15 Apr 2023 00:37:22 +0100 Subject: [PATCH 15/19] Main body of PR comments, switching event handling to be a publisher of async events, a generic event; then adding a new async document update processor, which can later be converted to be part of the event processing pipeline. --- packages/backend-core/src/docUpdates/index.ts | 30 ++++++++- .../backend-core/src/docUpdates/updates.ts | 62 ------------------- .../src/events/asyncEvents/index.ts | 2 + .../src/events/asyncEvents/publisher.ts | 12 ++++ .../src/events/asyncEvents/queue.ts | 22 +++++++ .../backend-core/src/events/documentId.ts | 56 +++++++++++++++++ packages/backend-core/src/events/events.ts | 12 +++- packages/backend-core/src/events/index.ts | 2 + .../processors/DocumentUpdateProcessor.ts | 42 ------------- .../src/events/processors/Processors.ts | 12 +++- .../async/DocumentUpdateProcessor.ts | 44 +++++++++++++ .../src/events/processors/index.ts | 3 - .../src/events/processors/types.ts | 6 +- packages/server/src/events/docUpdates.ts | 42 +++++++++++++ packages/server/src/events/index.ts | 1 + .../server/src/sdk/app/applications/sync.ts | 32 +--------- .../sdk/app/applications/tests/sync.spec.ts | 4 +- packages/server/src/startup.ts | 4 +- packages/types/src/sdk/events/event.ts | 8 ++- 19 files changed, 245 insertions(+), 151 deletions(-) delete mode 100644 packages/backend-core/src/docUpdates/updates.ts create mode 100644 packages/backend-core/src/events/asyncEvents/index.ts create mode 100644 packages/backend-core/src/events/asyncEvents/publisher.ts create mode 100644 packages/backend-core/src/events/asyncEvents/queue.ts create mode 100644 packages/backend-core/src/events/documentId.ts delete mode 100644 packages/backend-core/src/events/processors/DocumentUpdateProcessor.ts create mode 100644 packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts create mode 100644 packages/server/src/events/docUpdates.ts diff --git a/packages/backend-core/src/docUpdates/index.ts b/packages/backend-core/src/docUpdates/index.ts index c6c80aad8a..3971f8de12 100644 --- a/packages/backend-core/src/docUpdates/index.ts +++ b/packages/backend-core/src/docUpdates/index.ts @@ -1 +1,29 @@ -export * from "./updates" +import { asyncEventQueue, init as initQueue } from "../events/asyncEvents" +import { + ProcessorMap, + default as DocumentUpdateProcessor, +} from "../events/processors/async/DocumentUpdateProcessor" + +let processingPromise: Promise +let documentProcessor: DocumentUpdateProcessor + +export function init(processors: ProcessorMap) { + if (!asyncEventQueue) { + initQueue() + } + if (!documentProcessor) { + documentProcessor = new DocumentUpdateProcessor(processors) + } + // if not processing in this instance, kick it off + if (!processingPromise) { + processingPromise = asyncEventQueue.process(async job => { + const { event, identity, properties, timestamp } = job.data + await documentProcessor.processEvent( + event, + identity, + properties, + timestamp + ) + }) + } +} diff --git a/packages/backend-core/src/docUpdates/updates.ts b/packages/backend-core/src/docUpdates/updates.ts deleted file mode 100644 index 478d29f4eb..0000000000 --- a/packages/backend-core/src/docUpdates/updates.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { createQueue, JobQueue } from "../queue" -import BullQueue from "bull" -import { DocumentType, SEPARATOR } from "../constants" -import { doInContext, doInTenant } from "../context" - -type DocUpdateEvent = { - id: string - tenantId: string - appId?: string -} - -type Processor = (update: DocUpdateEvent) => Promise - -const processors: { types: DocumentType[]; processor: Processor }[] = [] -let queue: BullQueue.Queue -let processingPromise: Promise - -export function init() { - queue = createQueue(JobQueue.DOC_UPDATE) -} - -export async function shutdown() { - if (queue) { - await queue.close() - } -} - -export async function update(opts: DocUpdateEvent) { - if (!queue) { - init() - } - await queue.add(opts) -} - -async function handleJob(data: DocUpdateEvent) { - for (let { types, processor } of processors) { - if (types.find(type => data.id.startsWith(`${type}${SEPARATOR}`))) { - const context = data.appId || data.tenantId - const contextFn = data.appId ? doInContext : doInTenant - await contextFn(context, async () => { - await processor(data) - }) - } - } -} - -export async function process(types: DocumentType[], processor: Processor) { - if (!queue) { - init() - } - // add to processor list - processors.push({ - types, - processor, - }) - // if not processing in this instance, kick it off - if (!processingPromise) { - processingPromise = queue.process(async job => { - await handleJob(job.data) - }) - } -} diff --git a/packages/backend-core/src/events/asyncEvents/index.ts b/packages/backend-core/src/events/asyncEvents/index.ts new file mode 100644 index 0000000000..65f7f8f58b --- /dev/null +++ b/packages/backend-core/src/events/asyncEvents/index.ts @@ -0,0 +1,2 @@ +export * from "./queue" +export * from "./publisher" diff --git a/packages/backend-core/src/events/asyncEvents/publisher.ts b/packages/backend-core/src/events/asyncEvents/publisher.ts new file mode 100644 index 0000000000..4e44c4ddc5 --- /dev/null +++ b/packages/backend-core/src/events/asyncEvents/publisher.ts @@ -0,0 +1,12 @@ +import { AsyncEvents } from "@budibase/types" +import { EventPayload, asyncEventQueue, init } from "./queue" + +export async function publishAsyncEvent(payload: EventPayload) { + if (!asyncEventQueue) { + init() + } + const { event, identity } = payload + if (AsyncEvents.indexOf(event) !== -1 && identity.tenantId) { + await asyncEventQueue.add(payload) + } +} diff --git a/packages/backend-core/src/events/asyncEvents/queue.ts b/packages/backend-core/src/events/asyncEvents/queue.ts new file mode 100644 index 0000000000..c0b9b368b8 --- /dev/null +++ b/packages/backend-core/src/events/asyncEvents/queue.ts @@ -0,0 +1,22 @@ +import BullQueue from "bull" +import { createQueue, JobQueue } from "../../queue" +import { Event, Identity } from "@budibase/types" + +export interface EventPayload { + event: Event + identity: Identity + properties: any + timestamp?: string | number +} + +export let asyncEventQueue: BullQueue.Queue + +export function init() { + asyncEventQueue = createQueue(JobQueue.DOC_UPDATE) +} + +export async function shutdown() { + if (asyncEventQueue) { + await asyncEventQueue.close() + } +} diff --git a/packages/backend-core/src/events/documentId.ts b/packages/backend-core/src/events/documentId.ts new file mode 100644 index 0000000000..887e56f07b --- /dev/null +++ b/packages/backend-core/src/events/documentId.ts @@ -0,0 +1,56 @@ +import { + Event, + UserCreatedEvent, + UserUpdatedEvent, + UserDeletedEvent, + UserPermissionAssignedEvent, + UserPermissionRemovedEvent, + GroupCreatedEvent, + GroupUpdatedEvent, + GroupDeletedEvent, + GroupUsersAddedEvent, + GroupUsersDeletedEvent, + GroupPermissionsEditedEvent, +} from "@budibase/types" + +const getEventProperties: Record< + string, + (properties: any) => string | undefined +> = { + [Event.USER_CREATED]: (properties: UserCreatedEvent) => properties.userId, + [Event.USER_UPDATED]: (properties: UserUpdatedEvent) => properties.userId, + [Event.USER_DELETED]: (properties: UserDeletedEvent) => properties.userId, + [Event.USER_PERMISSION_ADMIN_ASSIGNED]: ( + properties: UserPermissionAssignedEvent + ) => properties.userId, + [Event.USER_PERMISSION_ADMIN_REMOVED]: ( + properties: UserPermissionRemovedEvent + ) => properties.userId, + [Event.USER_PERMISSION_BUILDER_ASSIGNED]: ( + properties: UserPermissionAssignedEvent + ) => properties.userId, + [Event.USER_PERMISSION_BUILDER_REMOVED]: ( + properties: UserPermissionRemovedEvent + ) => properties.userId, + [Event.USER_GROUP_CREATED]: (properties: GroupCreatedEvent) => + properties.groupId, + [Event.USER_GROUP_UPDATED]: (properties: GroupUpdatedEvent) => + properties.groupId, + [Event.USER_GROUP_DELETED]: (properties: GroupDeletedEvent) => + properties.groupId, + [Event.USER_GROUP_USERS_ADDED]: (properties: GroupUsersAddedEvent) => + properties.groupId, + [Event.USER_GROUP_USERS_REMOVED]: (properties: GroupUsersDeletedEvent) => + properties.groupId, + [Event.USER_GROUP_PERMISSIONS_EDITED]: ( + properties: GroupPermissionsEditedEvent + ) => properties.groupId, +} + +export function getDocumentId(event: Event, properties: any) { + const extractor = getEventProperties[event] + if (!extractor) { + throw new Error("Event does not have a method of document ID extraction") + } + return extractor(properties) +} diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index c2f7cf66ec..12cef21dff 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -1,7 +1,8 @@ -import { Event, AuditedEventFriendlyName } from "@budibase/types" +import { Event } from "@budibase/types" import { processors } from "./processors" import identification from "./identification" import * as backfill from "./backfill" +import { publishAsyncEvent } from "./asyncEvents" export const publishEvent = async ( event: Event, @@ -12,6 +13,15 @@ export const publishEvent = async ( const identity = await identification.getCurrentIdentity() const backfilling = await backfill.isBackfillingEvent(event) + + // send off async events if required + await publishAsyncEvent({ + event, + identity, + properties, + timestamp, + }) + // no backfill - send the event and exit if (!backfilling) { await processors.processEvent(event, identity, properties, timestamp) diff --git a/packages/backend-core/src/events/index.ts b/packages/backend-core/src/events/index.ts index d0d59a5b22..a238d72bac 100644 --- a/packages/backend-core/src/events/index.ts +++ b/packages/backend-core/src/events/index.ts @@ -6,6 +6,8 @@ export * as backfillCache from "./backfill" import { processors } from "./processors" +export function initAsyncEvents() {} + export const shutdown = () => { processors.shutdown() console.log("Events shutdown") diff --git a/packages/backend-core/src/events/processors/DocumentUpdateProcessor.ts b/packages/backend-core/src/events/processors/DocumentUpdateProcessor.ts deleted file mode 100644 index 496da7e923..0000000000 --- a/packages/backend-core/src/events/processors/DocumentUpdateProcessor.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Event, Identity, Group, DocumentUpdateEvents } from "@budibase/types" -import { EventProcessor } from "./types" -import * as docUpdates from "../../docUpdates" -import { getTenantId } from "../../context" - -export default class DocumentUpdateProcessor implements EventProcessor { - async processEvent( - event: Event, - identity: Identity, - properties: any, - timestamp?: string - ): Promise { - // only user and group IDs supported right now - no app documents yet - if (DocumentUpdateEvents.indexOf(event) !== -1 && identity.tenantId) { - await docUpdates.update({ - id: this.getId(properties), - tenantId: getTenantId(), - }) - } - } - - getId(properties: any) { - let possibleProps = ["groupId", "userId"] - for (let prop of possibleProps) { - if (properties[prop]) { - return properties[prop] - } - } - } - - async identify(identity: Identity, timestamp?: string | number) { - // no-op - } - - async identifyGroup(group: Group, timestamp?: string | number) { - // no-op - } - - shutdown(): void { - docUpdates.shutdown() - } -} diff --git a/packages/backend-core/src/events/processors/Processors.ts b/packages/backend-core/src/events/processors/Processors.ts index 4baedd909f..72de945d44 100644 --- a/packages/backend-core/src/events/processors/Processors.ts +++ b/packages/backend-core/src/events/processors/Processors.ts @@ -25,7 +25,9 @@ export default class Processor implements EventProcessor { timestamp?: string | number ): Promise { for (const eventProcessor of this.processors) { - await eventProcessor.identify(identity, timestamp) + if (eventProcessor.identify) { + await eventProcessor.identify(identity, timestamp) + } } } @@ -34,13 +36,17 @@ export default class Processor implements EventProcessor { timestamp?: string | number ): Promise { for (const eventProcessor of this.processors) { - await eventProcessor.identifyGroup(identity, timestamp) + if (eventProcessor.identifyGroup) { + await eventProcessor.identifyGroup(identity, timestamp) + } } } shutdown() { for (const eventProcessor of this.processors) { - eventProcessor.shutdown() + if (eventProcessor.shutdown) { + eventProcessor.shutdown() + } } } } diff --git a/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts new file mode 100644 index 0000000000..a65152cc70 --- /dev/null +++ b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts @@ -0,0 +1,44 @@ +import { EventProcessor } from "../types" +import { Event, Identity, DocUpdateEvent } from "@budibase/types" +import { DocumentType, SEPARATOR } from "../../../constants" +import { doInTenant } from "../../../context" +import { getDocumentId } from "../../documentId" +import { shutdown } from "../../asyncEvents" + +export type Processor = (update: DocUpdateEvent) => Promise +export type ProcessorMap = { types: DocumentType[]; processor: Processor }[] + +export default class DocumentUpdateProcessor implements EventProcessor { + processors: ProcessorMap = [] + + constructor(processors: ProcessorMap) { + this.processors = processors + } + + async processEvent( + event: Event, + identity: Identity, + properties: any, + timestamp?: string | number + ) { + const tenantId = identity.tenantId + const docId = getDocumentId(event, properties) + if (!tenantId || !docId) { + return + } + for (let { types, processor } of this.processors) { + if (types.find(type => docId.startsWith(`${type}${SEPARATOR}`))) { + await doInTenant(tenantId, async () => { + await processor({ + id: docId, + tenantId, + }) + }) + } + } + } + + shutdown() { + return shutdown() + } +} diff --git a/packages/backend-core/src/events/processors/index.ts b/packages/backend-core/src/events/processors/index.ts index 3582838d31..6646764e47 100644 --- a/packages/backend-core/src/events/processors/index.ts +++ b/packages/backend-core/src/events/processors/index.ts @@ -1,14 +1,12 @@ import AnalyticsProcessor from "./AnalyticsProcessor" import LoggingProcessor from "./LoggingProcessor" import AuditLogsProcessor from "./AuditLogsProcessor" -import DocumentUpdateProcessor from "./DocumentUpdateProcessor" import Processors from "./Processors" import { AuditLogFn } from "@budibase/types" export const analyticsProcessor = new AnalyticsProcessor() const loggingProcessor = new LoggingProcessor() const auditLogsProcessor = new AuditLogsProcessor() -const documentUpdateProcessor = new DocumentUpdateProcessor() export function init(auditingFn: AuditLogFn) { return AuditLogsProcessor.init(auditingFn) @@ -18,5 +16,4 @@ export const processors = new Processors([ analyticsProcessor, loggingProcessor, auditLogsProcessor, - documentUpdateProcessor, ]) diff --git a/packages/backend-core/src/events/processors/types.ts b/packages/backend-core/src/events/processors/types.ts index f4066fe248..33db6d8932 100644 --- a/packages/backend-core/src/events/processors/types.ts +++ b/packages/backend-core/src/events/processors/types.ts @@ -12,7 +12,7 @@ export interface EventProcessor { properties: any, timestamp?: string | number ): Promise - identify(identity: Identity, timestamp?: string | number): Promise - identifyGroup(group: Group, timestamp?: string | number): Promise - shutdown(): void + identify?(identity: Identity, timestamp?: string | number): Promise + identifyGroup?(group: Group, timestamp?: string | number): Promise + shutdown?(): void } diff --git a/packages/server/src/events/docUpdates.ts b/packages/server/src/events/docUpdates.ts new file mode 100644 index 0000000000..fcd90f64c4 --- /dev/null +++ b/packages/server/src/events/docUpdates.ts @@ -0,0 +1,42 @@ +import { constants, docUpdates, logging } from "@budibase/backend-core" +import { sdk as proSdk } from "@budibase/pro" +import { DocUpdateEvent } from "@budibase/types" +import { syncUsersToAllApps } from "../sdk/app/applications/sync" + +type UpdateCallback = (docId: string) => void + +function userGroupUpdates(updateCb?: UpdateCallback) { + const types = [constants.DocumentType.USER, constants.DocumentType.GROUP] + const processor = async (update: DocUpdateEvent) => { + try { + const docId = update.id + const isGroup = docId.startsWith(constants.DocumentType.GROUP) + let userIds: string[] + if (isGroup) { + const group = await proSdk.groups.get(docId) + userIds = group.users?.map(user => user._id) || [] + } else { + userIds = [docId] + } + if (userIds.length > 0) { + await syncUsersToAllApps(userIds) + } + if (updateCb) { + updateCb(docId) + } + } catch (err: any) { + // if something not found - no changes to perform + if (err?.status === 404) { + return + } else { + logging.logAlert("Failed to perform user/group app sync", err) + } + } + } + return { types, processor } +} + +export function init(updateCb?: UpdateCallback) { + const processors = [userGroupUpdates(updateCb)] + docUpdates.init(processors) +} diff --git a/packages/server/src/events/index.ts b/packages/server/src/events/index.ts index fad7bcaa9a..23c3f3e512 100644 --- a/packages/server/src/events/index.ts +++ b/packages/server/src/events/index.ts @@ -2,4 +2,5 @@ import BudibaseEmitter from "./BudibaseEmitter" const emitter = new BudibaseEmitter() +export { init } from "./docUpdates" export default emitter diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 66fd5d2d59..d0a1d78428 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -82,7 +82,7 @@ async function syncUsersToApp( }) } -async function syncUsersToAllApps(userIds: string[]) { +export async function syncUsersToAllApps(userIds: string[]) { // list of users, if one has been deleted it will be undefined in array const users = (await getGlobalUsers(userIds, { noProcessing: true, @@ -113,36 +113,6 @@ async function syncUsersToAllApps(userIds: string[]) { } } -export function initUserGroupSync(updateCb?: (docId: string) => void) { - const types = [constants.DocumentType.USER, constants.DocumentType.GROUP] - docUpdates.process(types, async update => { - try { - const docId = update.id - const isGroup = docId.startsWith(constants.DocumentType.GROUP) - let userIds: string[] - if (isGroup) { - const group = await proSdk.groups.get(docId) - userIds = group.users?.map(user => user._id) || [] - } else { - userIds = [docId] - } - if (userIds.length > 0) { - await syncUsersToAllApps(userIds) - } - if (updateCb) { - updateCb(docId) - } - } catch (err: any) { - // if something not found - no changes to perform - if (err?.status === 404) { - return - } else { - logging.logAlert("Failed to perform user/group app sync", err) - } - } - }) -} - export async function syncApp( appId: string, opts?: { automationOnly?: boolean } diff --git a/packages/server/src/sdk/app/applications/tests/sync.spec.ts b/packages/server/src/sdk/app/applications/tests/sync.spec.ts index 90493c541d..3b412a7fa7 100644 --- a/packages/server/src/sdk/app/applications/tests/sync.spec.ts +++ b/packages/server/src/sdk/app/applications/tests/sync.spec.ts @@ -1,6 +1,6 @@ import TestConfiguration from "../../../../tests/utilities/TestConfiguration" import { events, context, roles, constants } from "@budibase/backend-core" -import { initUserGroupSync } from "../sync" +import { init } from "../../../../events" import { rawUserMetadata } from "../../../users/utils" import EventEmitter from "events" import { UserGroup, UserMetadata, UserRoles, User } from "@budibase/types" @@ -35,7 +35,7 @@ function waitForUpdate(opts: { group?: boolean }) { beforeAll(async () => { app = await config.init("syncApp") - initUserGroupSync(updateCb) + init(updateCb) }) async function createUser(email: string, roles: UserRoles, builder?: boolean) { diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index 80cdc3c792..2fd59b7a8e 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -10,7 +10,7 @@ import fs from "fs" import { watch } from "./watch" import * as automations from "./automations" import * as fileSystem from "./utilities/fileSystem" -import eventEmitter from "./events" +import { default as eventEmitter, init as eventInit } from "./events" import * as migrations from "./migrations" import * as bullboard from "./automations/bullboard" import * as pro from "@budibase/pro" @@ -63,7 +63,7 @@ export async function startup(app?: any, server?: any) { eventEmitter.emitPort(env.PORT) fileSystem.init() await redis.init() - sdk.applications.initUserGroupSync() + eventInit() // run migrations on startup if not done via http // not recommended in a clustered environment diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 92965fa533..0b2ed3ce7c 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -186,7 +186,7 @@ export enum Event { AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded", } -export const DocumentUpdateEvents: Event[] = [ +export const AsyncEvents: Event[] = [ Event.USER_CREATED, Event.USER_UPDATED, Event.USER_DELETED, @@ -399,3 +399,9 @@ export interface BaseEvent { } export type TableExportFormat = "json" | "csv" + +export type DocUpdateEvent = { + id: string + tenantId: string + appId?: string +} From 453d5cc0e6bcf8d1905bf892a5b81241b4759552 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 17 Apr 2023 14:03:54 +0100 Subject: [PATCH 16/19] Further PR comments. --- .../src/events/asyncEvents/queue.ts | 2 +- packages/backend-core/src/events/events.ts | 17 ++++++++--------- .../async/DocumentUpdateProcessor.ts | 7 +++---- .../src/events/processors/types.ts | 19 +------------------ packages/backend-core/src/queue/constants.ts | 2 +- .../server/src/events/docUpdates/index.ts | 1 + .../src/events/docUpdates/processors.ts | 9 +++++++++ .../syncUsers.ts} | 19 ++++++------------- packages/types/src/sdk/events/event.ts | 17 ++++++++++++++++- yarn.lock | 12 ++++++------ 10 files changed, 52 insertions(+), 53 deletions(-) create mode 100644 packages/server/src/events/docUpdates/index.ts create mode 100644 packages/server/src/events/docUpdates/processors.ts rename packages/server/src/events/{docUpdates.ts => docUpdates/syncUsers.ts} (60%) diff --git a/packages/backend-core/src/events/asyncEvents/queue.ts b/packages/backend-core/src/events/asyncEvents/queue.ts index c0b9b368b8..196fd359b3 100644 --- a/packages/backend-core/src/events/asyncEvents/queue.ts +++ b/packages/backend-core/src/events/asyncEvents/queue.ts @@ -12,7 +12,7 @@ export interface EventPayload { export let asyncEventQueue: BullQueue.Queue export function init() { - asyncEventQueue = createQueue(JobQueue.DOC_UPDATE) + asyncEventQueue = createQueue(JobQueue.SYSTEM_EVENT_QUEUE) } export async function shutdown() { diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index 12cef21dff..f02b9fdf32 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -13,17 +13,16 @@ export const publishEvent = async ( const identity = await identification.getCurrentIdentity() const backfilling = await backfill.isBackfillingEvent(event) - - // send off async events if required - await publishAsyncEvent({ - event, - identity, - properties, - timestamp, - }) - // no backfill - send the event and exit if (!backfilling) { + // send off async events if required + await publishAsyncEvent({ + event, + identity, + properties, + timestamp, + }) + // now handle the main sync event processing pipeline await processors.processEvent(event, identity, properties, timestamp) return } diff --git a/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts index a65152cc70..d64ac1d41d 100644 --- a/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts +++ b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts @@ -1,12 +1,11 @@ import { EventProcessor } from "../types" import { Event, Identity, DocUpdateEvent } from "@budibase/types" -import { DocumentType, SEPARATOR } from "../../../constants" import { doInTenant } from "../../../context" import { getDocumentId } from "../../documentId" import { shutdown } from "../../asyncEvents" export type Processor = (update: DocUpdateEvent) => Promise -export type ProcessorMap = { types: DocumentType[]; processor: Processor }[] +export type ProcessorMap = { events: Event[]; processor: Processor }[] export default class DocumentUpdateProcessor implements EventProcessor { processors: ProcessorMap = [] @@ -26,8 +25,8 @@ export default class DocumentUpdateProcessor implements EventProcessor { if (!tenantId || !docId) { return } - for (let { types, processor } of this.processors) { - if (types.find(type => docId.startsWith(`${type}${SEPARATOR}`))) { + for (let { events, processor } of this.processors) { + if (events.includes(event)) { await doInTenant(tenantId, async () => { await processor({ id: docId, diff --git a/packages/backend-core/src/events/processors/types.ts b/packages/backend-core/src/events/processors/types.ts index 33db6d8932..5256a1bc62 100644 --- a/packages/backend-core/src/events/processors/types.ts +++ b/packages/backend-core/src/events/processors/types.ts @@ -1,18 +1 @@ -import { Event, Identity, Group } from "@budibase/types" - -export enum EventProcessorType { - POSTHOG = "posthog", - LOGGING = "logging", -} - -export interface EventProcessor { - processEvent( - event: Event, - identity: Identity, - properties: any, - timestamp?: string | number - ): Promise - identify?(identity: Identity, timestamp?: string | number): Promise - identifyGroup?(group: Group, timestamp?: string | number): Promise - shutdown?(): void -} +export { EventProcessor } from "@budibase/types" diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index 72d2e4742c..e1ffcfee36 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -2,5 +2,5 @@ export enum JobQueue { AUTOMATION = "automationQueue", APP_BACKUP = "appBackupQueue", AUDIT_LOG = "auditLogQueue", - DOC_UPDATE = "docUpdateQueue", + SYSTEM_EVENT_QUEUE = "systemEventQueue", } diff --git a/packages/server/src/events/docUpdates/index.ts b/packages/server/src/events/docUpdates/index.ts new file mode 100644 index 0000000000..fa7a623108 --- /dev/null +++ b/packages/server/src/events/docUpdates/index.ts @@ -0,0 +1 @@ +export * from "./processors" diff --git a/packages/server/src/events/docUpdates/processors.ts b/packages/server/src/events/docUpdates/processors.ts new file mode 100644 index 0000000000..8f3738f0dc --- /dev/null +++ b/packages/server/src/events/docUpdates/processors.ts @@ -0,0 +1,9 @@ +import userGroupProcessor from "./syncUsers" +import { docUpdates } from "@budibase/backend-core" + +export type UpdateCallback = (docId: string) => void + +export function init(updateCb?: UpdateCallback) { + const processors = [userGroupProcessor(updateCb)] + docUpdates.init(processors) +} diff --git a/packages/server/src/events/docUpdates.ts b/packages/server/src/events/docUpdates/syncUsers.ts similarity index 60% rename from packages/server/src/events/docUpdates.ts rename to packages/server/src/events/docUpdates/syncUsers.ts index fcd90f64c4..7957178168 100644 --- a/packages/server/src/events/docUpdates.ts +++ b/packages/server/src/events/docUpdates/syncUsers.ts @@ -1,12 +1,10 @@ -import { constants, docUpdates, logging } from "@budibase/backend-core" +import { constants, logging } from "@budibase/backend-core" import { sdk as proSdk } from "@budibase/pro" -import { DocUpdateEvent } from "@budibase/types" -import { syncUsersToAllApps } from "../sdk/app/applications/sync" +import { DocUpdateEvent, UserGroupSyncEvents } from "@budibase/types" +import { syncUsersToAllApps } from "../../sdk/app/applications/sync" +import { UpdateCallback } from "./processors" -type UpdateCallback = (docId: string) => void - -function userGroupUpdates(updateCb?: UpdateCallback) { - const types = [constants.DocumentType.USER, constants.DocumentType.GROUP] +export default function process(updateCb?: UpdateCallback) { const processor = async (update: DocUpdateEvent) => { try { const docId = update.id @@ -33,10 +31,5 @@ function userGroupUpdates(updateCb?: UpdateCallback) { } } } - return { types, processor } -} - -export function init(updateCb?: UpdateCallback) { - const processors = [userGroupUpdates(updateCb)] - docUpdates.init(processors) + return { events: UserGroupSyncEvents, processor } } diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 0b2ed3ce7c..c4990f869b 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -1,4 +1,5 @@ import { Hosting } from "../hosting" +import { Group, Identity } from "./identification" export enum Event { // USER @@ -186,7 +187,7 @@ export enum Event { AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded", } -export const AsyncEvents: Event[] = [ +export const UserGroupSyncEvents: Event[] = [ Event.USER_CREATED, Event.USER_UPDATED, Event.USER_DELETED, @@ -202,6 +203,8 @@ export const AsyncEvents: Event[] = [ Event.USER_GROUP_PERMISSIONS_EDITED, ] +export const AsyncEvents: Event[] = [...UserGroupSyncEvents] + // all events that are not audited have been added to this record as undefined, this means // that Typescript can protect us against new events being added and auditing of those // events not being considered. This might be a little ugly, but provides a level of @@ -405,3 +408,15 @@ export type DocUpdateEvent = { tenantId: string appId?: string } + +export interface EventProcessor { + processEvent( + event: Event, + identity: Identity, + properties: any, + timestamp?: string | number + ): Promise + identify?(identity: Identity, timestamp?: string | number): Promise + identifyGroup?(group: Group, timestamp?: string | number): Promise + shutdown?(): void +} diff --git a/yarn.lock b/yarn.lock index c1b5be3b11..f5cd450ddc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,15 +1486,15 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@2.5.5-alpha.0": - version "2.5.5-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.5-alpha.0.tgz#28b075a96efb564328a4972cae9ea6c9a5f3aabc" - integrity sha512-98fLnvHWVy7ASEFC98bo6Qdd55SjC7yrJNuf7FUYZbeFwpmwwRxlWnWFTa0ctKWB5p2LToARWBns3TqgnUr/zQ== +"@budibase/pro@2.5.5-alpha.1": + version "2.5.5-alpha.1" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.5-alpha.1.tgz#6d9476ce724bd3e405cb2e0198c4b83b168e65bc" + integrity sha512-fEuropk/0aH1+lELX6wdFa8UUpE+SMtlvBbsgNR2ulvLgLLPEYhI2gINlhiItMWWgTtxNtele3hOs1VWhn0o2A== dependencies: - "@budibase/backend-core" "2.5.5-alpha.0" + "@budibase/backend-core" "2.5.5-alpha.1" "@budibase/shared-core" "2.4.44-alpha.1" "@budibase/string-templates" "2.4.44-alpha.1" - "@budibase/types" "2.5.5-alpha.0" + "@budibase/types" "2.5.5-alpha.1" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" From b4e6cbce7aced11f2d4acbbb16113640d417c0e6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 17 Apr 2023 15:59:59 +0100 Subject: [PATCH 17/19] Some final updates to get tests passing again, issue with publisher tenant ID being the 'unique' format which cannot actually be used. --- packages/backend-core/src/events/identification.ts | 1 + .../src/events/processors/async/DocumentUpdateProcessor.ts | 2 +- packages/server/src/events/docUpdates/processors.ts | 5 +++++ packages/server/src/sdk/app/applications/tests/sync.spec.ts | 3 ++- packages/types/src/sdk/events/identification.ts | 2 ++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 9534fb293d..c85eb16a77 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -65,6 +65,7 @@ const getCurrentIdentity = async (): Promise => { hosting, installationId, tenantId, + realTenantId: context.getTenantId(), environment, } } else if (identityType === IdentityType.USER) { diff --git a/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts index d64ac1d41d..54304ee21b 100644 --- a/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts +++ b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts @@ -20,7 +20,7 @@ export default class DocumentUpdateProcessor implements EventProcessor { properties: any, timestamp?: string | number ) { - const tenantId = identity.tenantId + const tenantId = identity.realTenantId const docId = getDocumentId(event, properties) if (!tenantId || !docId) { return diff --git a/packages/server/src/events/docUpdates/processors.ts b/packages/server/src/events/docUpdates/processors.ts index 8f3738f0dc..53036970e0 100644 --- a/packages/server/src/events/docUpdates/processors.ts +++ b/packages/server/src/events/docUpdates/processors.ts @@ -2,8 +2,13 @@ import userGroupProcessor from "./syncUsers" import { docUpdates } from "@budibase/backend-core" export type UpdateCallback = (docId: string) => void +let started = false export function init(updateCb?: UpdateCallback) { + if (started) { + return + } const processors = [userGroupProcessor(updateCb)] docUpdates.init(processors) + started = true } diff --git a/packages/server/src/sdk/app/applications/tests/sync.spec.ts b/packages/server/src/sdk/app/applications/tests/sync.spec.ts index 3b412a7fa7..8609e59a2f 100644 --- a/packages/server/src/sdk/app/applications/tests/sync.spec.ts +++ b/packages/server/src/sdk/app/applications/tests/sync.spec.ts @@ -20,6 +20,8 @@ function updateCb(docId: string) { } } +init(updateCb) + function waitForUpdate(opts: { group?: boolean }) { return new Promise((resolve, reject) => { const timeout = setTimeout(() => { @@ -35,7 +37,6 @@ function waitForUpdate(opts: { group?: boolean }) { beforeAll(async () => { app = await config.init("syncApp") - init(updateCb) }) async function createUser(email: string, roles: UserRoles, builder?: boolean) { diff --git a/packages/types/src/sdk/events/identification.ts b/packages/types/src/sdk/events/identification.ts index 627254882e..7c7a2be8e0 100644 --- a/packages/types/src/sdk/events/identification.ts +++ b/packages/types/src/sdk/events/identification.ts @@ -46,6 +46,8 @@ export interface Identity { environment: string installationId?: string tenantId?: string + // usable - no unique format + realTenantId?: string hostInfo?: HostInfo } From 443319f4a51b91ae57b56a3916effdc8251c87d6 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 17 Apr 2023 16:55:49 +0000 Subject: [PATCH 18/19] v2.5.6-alpha.2 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 6 +++--- packages/builder/package.json | 12 ++++++------ packages/cli/package.json | 8 ++++---- packages/client/package.json | 12 ++++++------ packages/frontend-core/package.json | 6 +++--- packages/sdk/package.json | 2 +- packages/server/package.json | 12 ++++++------ packages/shared-core/package.json | 4 ++-- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 13 files changed, 40 insertions(+), 40 deletions(-) diff --git a/lerna.json b/lerna.json index 5c8280fef2..0abab04b41 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "npmClient": "yarn", "useWorkspaces": true, "packages": ["packages/*"], diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 77aac3b557..cf63eaf3e0 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/types": "2.5.6-alpha.2", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 760772cffe..5d60f15208 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", + "@budibase/shared-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/builder/package.json b/packages/builder/package.json index 0fe2cfdeb3..0483aef7e2 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,11 +58,11 @@ } }, "dependencies": { - "@budibase/bbui": "2.5.6-alpha.1", - "@budibase/client": "2.5.6-alpha.1", - "@budibase/frontend-core": "2.5.6-alpha.1", - "@budibase/shared-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", + "@budibase/bbui": "2.5.6-alpha.2", + "@budibase/client": "2.5.6-alpha.2", + "@budibase/frontend-core": "2.5.6-alpha.2", + "@budibase/shared-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 49d8d2308a..663a392a87 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "dist/index.js", "bin": { @@ -29,9 +29,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/backend-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", + "@budibase/types": "2.5.6-alpha.2", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index de27e5da83..ec13b77386 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,11 +19,11 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "2.5.6-alpha.1", - "@budibase/frontend-core": "2.5.6-alpha.1", - "@budibase/shared-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/bbui": "2.5.6-alpha.2", + "@budibase/frontend-core": "2.5.6-alpha.2", + "@budibase/shared-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", + "@budibase/types": "2.5.6-alpha.2", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index c41c3adb28..82cba430ec 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,13 +1,13 @@ { "name": "@budibase/frontend-core", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "2.5.6-alpha.1", - "@budibase/shared-core": "2.5.6-alpha.1", + "@budibase/bbui": "2.5.6-alpha.2", + "@budibase/shared-core": "2.5.6-alpha.2", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 50c4ec597d..2128d4fbd0 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index b78e0ad4e1..29e99e3b4e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -45,12 +45,12 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "2.5.6-alpha.1", - "@budibase/client": "2.5.6-alpha.1", + "@budibase/backend-core": "2.5.6-alpha.2", + "@budibase/client": "2.5.6-alpha.2", "@budibase/pro": "2.5.6-alpha.1", - "@budibase/shared-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/shared-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", + "@budibase/types": "2.5.6-alpha.2", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index 9eb0c26ebc..adff1822a4 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/shared-core", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Shared data utils", "main": "dist/cjs/src/index.js", "types": "dist/mjs/src/index.d.ts", @@ -20,7 +20,7 @@ "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\"" }, "dependencies": { - "@budibase/types": "2.5.6-alpha.1" + "@budibase/types": "2.5.6-alpha.2" }, "devDependencies": { "concurrently": "^7.6.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index f4def12c14..01840f3546 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index e36d039375..f7ede811a1 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase types", "main": "dist/cjs/index.js", "types": "dist/mjs/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index 972437389b..c2b00bc8f0 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -37,10 +37,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "2.5.6-alpha.1", + "@budibase/backend-core": "2.5.6-alpha.2", "@budibase/pro": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/string-templates": "2.5.6-alpha.2", + "@budibase/types": "2.5.6-alpha.2", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From a8f884085406034cc4125c175ffdb65c47573662 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 17 Apr 2023 16:57:29 +0000 Subject: [PATCH 19/19] Update pro version to 2.5.6-alpha.2 --- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 29e99e3b4e..373edee379 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -47,7 +47,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "2.5.6-alpha.2", "@budibase/client": "2.5.6-alpha.2", - "@budibase/pro": "2.5.6-alpha.1", + "@budibase/pro": "2.5.6-alpha.2", "@budibase/shared-core": "2.5.6-alpha.2", "@budibase/string-templates": "2.5.6-alpha.2", "@budibase/types": "2.5.6-alpha.2", diff --git a/packages/worker/package.json b/packages/worker/package.json index c2b00bc8f0..f67a569935 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -38,7 +38,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "2.5.6-alpha.2", - "@budibase/pro": "2.5.6-alpha.1", + "@budibase/pro": "2.5.6-alpha.2", "@budibase/string-templates": "2.5.6-alpha.2", "@budibase/types": "2.5.6-alpha.2", "@koa/router": "8.0.8",