diff --git a/lerna.json b/lerna.json index 5c8280fef2..0abab04b41 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "npmClient": "yarn", "useWorkspaces": true, "packages": ["packages/*"], diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 77aac3b557..cf63eaf3e0 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/types": "2.5.6-alpha.2", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 8a2c2e7efd..f0057cd7c3 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -42,7 +42,11 @@ async function removeDeprecated(db: Database, viewName: ViewName) { } } -export async function createView(db: any, viewJs: string, viewName: string) { +export async function createView( + db: any, + viewJs: string, + viewName: string +): Promise { let designDoc try { designDoc = (await db.get(DESIGN_DB)) as DesignDocument @@ -57,7 +61,15 @@ export async function createView(db: any, viewJs: string, viewName: string) { ...designDoc.views, [viewName]: view, } - await db.put(designDoc) + try { + await db.put(designDoc) + } catch (err: any) { + if (err.status === 409) { + return await createView(db, viewJs, viewName) + } else { + throw err + } + } } export const createNewUserEmailView = async () => { @@ -135,6 +147,10 @@ export const queryView = async ( await removeDeprecated(db, viewName) await createFunc() return queryView(viewName, params, db, createFunc, opts) + } else if (err.status === 409) { + // can happen when multiple queries occur at once, view couldn't be created + // other design docs being updated, re-run + return queryView(viewName, params, db, createFunc, opts) } else { throw err } diff --git a/packages/backend-core/src/docUpdates/index.ts b/packages/backend-core/src/docUpdates/index.ts new file mode 100644 index 0000000000..3971f8de12 --- /dev/null +++ b/packages/backend-core/src/docUpdates/index.ts @@ -0,0 +1,29 @@ +import { asyncEventQueue, init as initQueue } from "../events/asyncEvents" +import { + ProcessorMap, + default as DocumentUpdateProcessor, +} from "../events/processors/async/DocumentUpdateProcessor" + +let processingPromise: Promise +let documentProcessor: DocumentUpdateProcessor + +export function init(processors: ProcessorMap) { + if (!asyncEventQueue) { + initQueue() + } + if (!documentProcessor) { + documentProcessor = new DocumentUpdateProcessor(processors) + } + // if not processing in this instance, kick it off + if (!processingPromise) { + processingPromise = asyncEventQueue.process(async job => { + const { event, identity, properties, timestamp } = job.data + await documentProcessor.processEvent( + event, + identity, + properties, + timestamp + ) + }) + } +} diff --git a/packages/backend-core/src/events/asyncEvents/index.ts b/packages/backend-core/src/events/asyncEvents/index.ts new file mode 100644 index 0000000000..65f7f8f58b --- /dev/null +++ b/packages/backend-core/src/events/asyncEvents/index.ts @@ -0,0 +1,2 @@ +export * from "./queue" +export * from "./publisher" diff --git a/packages/backend-core/src/events/asyncEvents/publisher.ts b/packages/backend-core/src/events/asyncEvents/publisher.ts new file mode 100644 index 0000000000..4e44c4ddc5 --- /dev/null +++ b/packages/backend-core/src/events/asyncEvents/publisher.ts @@ -0,0 +1,12 @@ +import { AsyncEvents } from "@budibase/types" +import { EventPayload, asyncEventQueue, init } from "./queue" + +export async function publishAsyncEvent(payload: EventPayload) { + if (!asyncEventQueue) { + init() + } + const { event, identity } = payload + if (AsyncEvents.indexOf(event) !== -1 && identity.tenantId) { + await asyncEventQueue.add(payload) + } +} diff --git a/packages/backend-core/src/events/asyncEvents/queue.ts b/packages/backend-core/src/events/asyncEvents/queue.ts new file mode 100644 index 0000000000..196fd359b3 --- /dev/null +++ b/packages/backend-core/src/events/asyncEvents/queue.ts @@ -0,0 +1,22 @@ +import BullQueue from "bull" +import { createQueue, JobQueue } from "../../queue" +import { Event, Identity } from "@budibase/types" + +export interface EventPayload { + event: Event + identity: Identity + properties: any + timestamp?: string | number +} + +export let asyncEventQueue: BullQueue.Queue + +export function init() { + asyncEventQueue = createQueue(JobQueue.SYSTEM_EVENT_QUEUE) +} + +export async function shutdown() { + if (asyncEventQueue) { + await asyncEventQueue.close() + } +} diff --git a/packages/backend-core/src/events/documentId.ts b/packages/backend-core/src/events/documentId.ts new file mode 100644 index 0000000000..887e56f07b --- /dev/null +++ b/packages/backend-core/src/events/documentId.ts @@ -0,0 +1,56 @@ +import { + Event, + UserCreatedEvent, + UserUpdatedEvent, + UserDeletedEvent, + UserPermissionAssignedEvent, + UserPermissionRemovedEvent, + GroupCreatedEvent, + GroupUpdatedEvent, + GroupDeletedEvent, + GroupUsersAddedEvent, + GroupUsersDeletedEvent, + GroupPermissionsEditedEvent, +} from "@budibase/types" + +const getEventProperties: Record< + string, + (properties: any) => string | undefined +> = { + [Event.USER_CREATED]: (properties: UserCreatedEvent) => properties.userId, + [Event.USER_UPDATED]: (properties: UserUpdatedEvent) => properties.userId, + [Event.USER_DELETED]: (properties: UserDeletedEvent) => properties.userId, + [Event.USER_PERMISSION_ADMIN_ASSIGNED]: ( + properties: UserPermissionAssignedEvent + ) => properties.userId, + [Event.USER_PERMISSION_ADMIN_REMOVED]: ( + properties: UserPermissionRemovedEvent + ) => properties.userId, + [Event.USER_PERMISSION_BUILDER_ASSIGNED]: ( + properties: UserPermissionAssignedEvent + ) => properties.userId, + [Event.USER_PERMISSION_BUILDER_REMOVED]: ( + properties: UserPermissionRemovedEvent + ) => properties.userId, + [Event.USER_GROUP_CREATED]: (properties: GroupCreatedEvent) => + properties.groupId, + [Event.USER_GROUP_UPDATED]: (properties: GroupUpdatedEvent) => + properties.groupId, + [Event.USER_GROUP_DELETED]: (properties: GroupDeletedEvent) => + properties.groupId, + [Event.USER_GROUP_USERS_ADDED]: (properties: GroupUsersAddedEvent) => + properties.groupId, + [Event.USER_GROUP_USERS_REMOVED]: (properties: GroupUsersDeletedEvent) => + properties.groupId, + [Event.USER_GROUP_PERMISSIONS_EDITED]: ( + properties: GroupPermissionsEditedEvent + ) => properties.groupId, +} + +export function getDocumentId(event: Event, properties: any) { + const extractor = getEventProperties[event] + if (!extractor) { + throw new Error("Event does not have a method of document ID extraction") + } + return extractor(properties) +} diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index c2f7cf66ec..f02b9fdf32 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -1,7 +1,8 @@ -import { Event, AuditedEventFriendlyName } from "@budibase/types" +import { Event } from "@budibase/types" import { processors } from "./processors" import identification from "./identification" import * as backfill from "./backfill" +import { publishAsyncEvent } from "./asyncEvents" export const publishEvent = async ( event: Event, @@ -14,6 +15,14 @@ export const publishEvent = async ( const backfilling = await backfill.isBackfillingEvent(event) // no backfill - send the event and exit if (!backfilling) { + // send off async events if required + await publishAsyncEvent({ + event, + identity, + properties, + timestamp, + }) + // now handle the main sync event processing pipeline await processors.processEvent(event, identity, properties, timestamp) return } diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 9534fb293d..c85eb16a77 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -65,6 +65,7 @@ const getCurrentIdentity = async (): Promise => { hosting, installationId, tenantId, + realTenantId: context.getTenantId(), environment, } } else if (identityType === IdentityType.USER) { diff --git a/packages/backend-core/src/events/index.ts b/packages/backend-core/src/events/index.ts index d0d59a5b22..a238d72bac 100644 --- a/packages/backend-core/src/events/index.ts +++ b/packages/backend-core/src/events/index.ts @@ -6,6 +6,8 @@ export * as backfillCache from "./backfill" import { processors } from "./processors" +export function initAsyncEvents() {} + export const shutdown = () => { processors.shutdown() console.log("Events shutdown") diff --git a/packages/backend-core/src/events/processors/Processors.ts b/packages/backend-core/src/events/processors/Processors.ts index 4baedd909f..72de945d44 100644 --- a/packages/backend-core/src/events/processors/Processors.ts +++ b/packages/backend-core/src/events/processors/Processors.ts @@ -25,7 +25,9 @@ export default class Processor implements EventProcessor { timestamp?: string | number ): Promise { for (const eventProcessor of this.processors) { - await eventProcessor.identify(identity, timestamp) + if (eventProcessor.identify) { + await eventProcessor.identify(identity, timestamp) + } } } @@ -34,13 +36,17 @@ export default class Processor implements EventProcessor { timestamp?: string | number ): Promise { for (const eventProcessor of this.processors) { - await eventProcessor.identifyGroup(identity, timestamp) + if (eventProcessor.identifyGroup) { + await eventProcessor.identifyGroup(identity, timestamp) + } } } shutdown() { for (const eventProcessor of this.processors) { - eventProcessor.shutdown() + if (eventProcessor.shutdown) { + eventProcessor.shutdown() + } } } } diff --git a/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts new file mode 100644 index 0000000000..54304ee21b --- /dev/null +++ b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts @@ -0,0 +1,43 @@ +import { EventProcessor } from "../types" +import { Event, Identity, DocUpdateEvent } from "@budibase/types" +import { doInTenant } from "../../../context" +import { getDocumentId } from "../../documentId" +import { shutdown } from "../../asyncEvents" + +export type Processor = (update: DocUpdateEvent) => Promise +export type ProcessorMap = { events: Event[]; processor: Processor }[] + +export default class DocumentUpdateProcessor implements EventProcessor { + processors: ProcessorMap = [] + + constructor(processors: ProcessorMap) { + this.processors = processors + } + + async processEvent( + event: Event, + identity: Identity, + properties: any, + timestamp?: string | number + ) { + const tenantId = identity.realTenantId + const docId = getDocumentId(event, properties) + if (!tenantId || !docId) { + return + } + for (let { events, processor } of this.processors) { + if (events.includes(event)) { + await doInTenant(tenantId, async () => { + await processor({ + id: docId, + tenantId, + }) + }) + } + } + } + + shutdown() { + return shutdown() + } +} diff --git a/packages/backend-core/src/events/processors/types.ts b/packages/backend-core/src/events/processors/types.ts index f4066fe248..5256a1bc62 100644 --- a/packages/backend-core/src/events/processors/types.ts +++ b/packages/backend-core/src/events/processors/types.ts @@ -1,18 +1 @@ -import { Event, Identity, Group } from "@budibase/types" - -export enum EventProcessorType { - POSTHOG = "posthog", - LOGGING = "logging", -} - -export interface EventProcessor { - processEvent( - event: Event, - identity: Identity, - properties: any, - timestamp?: string | number - ): Promise - identify(identity: Identity, timestamp?: string | number): Promise - identifyGroup(group: Group, timestamp?: string | number): Promise - shutdown(): void -} +export { EventProcessor } from "@budibase/types" diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 30072196ba..40233b3827 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -27,6 +27,7 @@ export * as errors from "./errors" export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" +export * as docUpdates from "./docUpdates" export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index 9261ed1176..e1ffcfee36 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -2,4 +2,5 @@ export enum JobQueue { AUTOMATION = "automationQueue", APP_BACKUP = "appBackupQueue", AUDIT_LOG = "auditLogQueue", + SYSTEM_EVENT_QUEUE = "systemEventQueue", } diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 0c008589d4..7fb0466dcd 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", + "@budibase/shared-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/builder/package.json b/packages/builder/package.json index 0fe2cfdeb3..0483aef7e2 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,11 +58,11 @@ } }, "dependencies": { - "@budibase/bbui": "2.5.6-alpha.1", - "@budibase/client": "2.5.6-alpha.1", - "@budibase/frontend-core": "2.5.6-alpha.1", - "@budibase/shared-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", + "@budibase/bbui": "2.5.6-alpha.2", + "@budibase/client": "2.5.6-alpha.2", + "@budibase/frontend-core": "2.5.6-alpha.2", + "@budibase/shared-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 49d8d2308a..663a392a87 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "dist/index.js", "bin": { @@ -29,9 +29,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/backend-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", + "@budibase/types": "2.5.6-alpha.2", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index de27e5da83..ec13b77386 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,11 +19,11 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "2.5.6-alpha.1", - "@budibase/frontend-core": "2.5.6-alpha.1", - "@budibase/shared-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/bbui": "2.5.6-alpha.2", + "@budibase/frontend-core": "2.5.6-alpha.2", + "@budibase/shared-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", + "@budibase/types": "2.5.6-alpha.2", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 140e43a388..b2fb8a3b37 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,13 +1,13 @@ { "name": "@budibase/frontend-core", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "2.5.6-alpha.1", - "@budibase/shared-core": "2.5.6-alpha.1", + "@budibase/bbui": "2.5.6-alpha.2", + "@budibase/shared-core": "2.5.6-alpha.2", "dayjs": "^1.11.7", "lodash": "^4.17.21", "socket.io-client": "^4.6.1", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 50c4ec597d..2128d4fbd0 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index d86df9ec71..f3210b0785 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -45,12 +45,12 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "2.5.6-alpha.1", - "@budibase/client": "2.5.6-alpha.1", - "@budibase/pro": "2.5.6-alpha.1", - "@budibase/shared-core": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/backend-core": "2.5.6-alpha.2", + "@budibase/client": "2.5.6-alpha.2", + "@budibase/pro": "2.5.6-alpha.2", + "@budibase/shared-core": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", + "@budibase/types": "2.5.6-alpha.2", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 039f03c015..a0bebc2490 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -30,7 +30,6 @@ import { finaliseRow, updateRelatedFormula } from "./staticFormula" import { csv, json, jsonWithSchema, Format } from "../view/exporters" import { apiFileReturn } from "../../../utilities/fileSystem" import { - Ctx, UserCtx, Database, LinkDocumentValue, @@ -72,7 +71,7 @@ async function getView(db: Database, viewName: string) { return viewInfo } -async function getRawTableData(ctx: Ctx, db: Database, tableId: string) { +async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) { let rows if (tableId === InternalTables.USER_METADATA) { await userController.fetchMetadata(ctx) @@ -188,7 +187,7 @@ export async function save(ctx: UserCtx) { }) } -export async function fetchView(ctx: Ctx) { +export async function fetchView(ctx: UserCtx) { const viewName = decodeURIComponent(ctx.params.viewName) // if this is a table view being looked for just transfer to that @@ -255,7 +254,7 @@ export async function fetchView(ctx: Ctx) { return rows } -export async function fetch(ctx: Ctx) { +export async function fetch(ctx: UserCtx) { const db = context.getAppDB() const tableId = ctx.params.tableId @@ -264,7 +263,7 @@ export async function fetch(ctx: Ctx) { return outputProcessing(table, rows) } -export async function find(ctx: Ctx) { +export async function find(ctx: UserCtx) { const db = dbCore.getDB(ctx.appId) const table = await db.get(ctx.params.tableId) let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId) @@ -272,7 +271,7 @@ export async function find(ctx: Ctx) { return row } -export async function destroy(ctx: Ctx) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const { _id } = ctx.request.body let row = await db.get(_id) @@ -308,7 +307,7 @@ export async function destroy(ctx: Ctx) { return { response, row } } -export async function bulkDestroy(ctx: Ctx) { +export async function bulkDestroy(ctx: UserCtx) { const db = context.getAppDB() const tableId = ctx.params.tableId const table = await db.get(tableId) @@ -347,7 +346,7 @@ export async function bulkDestroy(ctx: Ctx) { return { response: { ok: true }, rows: processedRows } } -export async function search(ctx: Ctx) { +export async function search(ctx: UserCtx) { // Fetch the whole table when running in cypress, as search doesn't work if (!env.COUCH_DB_URL && env.isCypress()) { return { rows: await fetch(ctx) } @@ -387,7 +386,7 @@ export async function search(ctx: Ctx) { return response } -export async function exportRows(ctx: Ctx) { +export async function exportRows(ctx: UserCtx) { const db = context.getAppDB() const table = await db.get(ctx.params.tableId) const rowIds = ctx.request.body.rows @@ -439,7 +438,7 @@ export async function exportRows(ctx: Ctx) { } } -export async function fetchEnrichedRow(ctx: Ctx) { +export async function fetchEnrichedRow(ctx: UserCtx) { const db = context.getAppDB() const tableId = ctx.params.tableId const rowId = ctx.params.rowId diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 2cf3b5472f..e96a4fe6ee 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -5,7 +5,7 @@ import { context } from "@budibase/backend-core" import { makeExternalQuery } from "../../../integrations/base/query" import { Row, Table } from "@budibase/types" import { Format } from "../view/exporters" -import { Ctx } from "@budibase/types" +import { UserCtx } from "@budibase/types" import sdk from "../../../sdk" const validateJs = require("validate.js") const { cloneDeep } = require("lodash/fp") @@ -26,7 +26,7 @@ export async function getDatasourceAndQuery(json: any) { return makeExternalQuery(datasource, json) } -export async function findRow(ctx: Ctx, tableId: string, rowId: string) { +export async function findRow(ctx: UserCtx, tableId: string, rowId: string) { const db = context.getAppDB() let row // TODO remove special user case in future diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index 1ae1a68824..b66f11bc1c 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -1,98 +1,12 @@ import { generateUserMetadataID, generateUserFlagID } from "../../db/utils" import { InternalTables } from "../../db/utils" -import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global" +import { getGlobalUsers } from "../../utilities/global" import { getFullUser } from "../../utilities/users" -import { - context, - roles as rolesCore, - db as dbCore, -} from "@budibase/backend-core" -import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types" +import { context } from "@budibase/backend-core" +import { UserCtx } from "@budibase/types" import sdk from "../../sdk" -export async function syncUser(ctx: Ctx) { - let deleting = false, - user: User | any - const userId = ctx.params.id - - const previousUser = ctx.request.body?.previousUser - - try { - user = (await getRawGlobalUser(userId)) as User - } catch (err: any) { - if (err && err.status === 404) { - user = {} - deleting = true - } else { - throw err - } - } - - let previousApps = previousUser - ? Object.keys(previousUser.roles).map(appId => appId) - : [] - - const roles = deleting ? {} : user.roles - // remove props which aren't useful to metadata - delete user.password - delete user.forceResetPassword - delete user.roles - // run through all production appIDs in the users roles - let prodAppIds - // if they are a builder then get all production app IDs - if ((user.builder && user.builder.global) || deleting) { - prodAppIds = await dbCore.getProdAppIDs() - } else { - prodAppIds = Object.entries(roles) - .filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC) - .map(([appId]) => appId) - } - for (let prodAppId of new Set([...prodAppIds, ...previousApps])) { - const roleId = roles[prodAppId] - const deleteFromApp = !roleId - const devAppId = dbCore.getDevelopmentAppID(prodAppId) - for (let appId of [prodAppId, devAppId]) { - if (!(await dbCore.dbExists(appId))) { - continue - } - await context.doInAppContext(appId, async () => { - const db = context.getAppDB() - const metadataId = generateUserMetadataID(userId) - let metadata - try { - metadata = await db.get(metadataId) - } catch (err) { - if (deleteFromApp) { - return - } - metadata = { - tableId: InternalTables.USER_METADATA, - } - } - - if (deleteFromApp) { - await db.remove(metadata) - return - } - - // assign the roleId for the metadata doc - if (roleId) { - metadata.roleId = roleId - } - let combined = sdk.users.combineMetadataAndUser(user, metadata) - // if its null then there was no updates required - if (combined) { - await db.put(combined) - } - }) - } - } - ctx.body = { - message: "User synced.", - } -} - -export async function fetchMetadata(ctx: BBContext) { +export async function fetchMetadata(ctx: UserCtx) { const global = await getGlobalUsers() const metadata = await sdk.users.rawUserMetadata() const users = [] @@ -111,7 +25,7 @@ export async function fetchMetadata(ctx: BBContext) { ctx.body = users } -export async function updateSelfMetadata(ctx: BBContext) { +export async function updateSelfMetadata(ctx: UserCtx) { // overwrite the ID with current users ctx.request.body._id = ctx.user?._id // make sure no stale rev @@ -121,7 +35,7 @@ export async function updateSelfMetadata(ctx: BBContext) { await updateMetadata(ctx) } -export async function updateMetadata(ctx: BBContext) { +export async function updateMetadata(ctx: UserCtx) { const db = context.getAppDB() const user = ctx.request.body // this isn't applicable to the user @@ -133,7 +47,7 @@ export async function updateMetadata(ctx: BBContext) { ctx.body = await db.put(metadata) } -export async function destroyMetadata(ctx: BBContext) { +export async function destroyMetadata(ctx: UserCtx) { const db = context.getAppDB() try { const dbUser = await db.get(ctx.params.id) @@ -146,11 +60,11 @@ export async function destroyMetadata(ctx: BBContext) { } } -export async function findMetadata(ctx: BBContext) { +export async function findMetadata(ctx: UserCtx) { ctx.body = await getFullUser(ctx, ctx.params.id) } -export async function setFlag(ctx: BBContext) { +export async function setFlag(ctx: UserCtx) { const userId = ctx.user?._id const { flag, value } = ctx.request.body if (!flag) { @@ -169,7 +83,7 @@ export async function setFlag(ctx: BBContext) { ctx.body = { message: "Flag set successfully" } } -export async function getFlags(ctx: BBContext) { +export async function getFlags(ctx: UserCtx) { const userId = ctx.user?._id const docId = generateUserFlagID(userId!) const db = context.getAppDB() diff --git a/packages/server/src/api/routes/tests/user.spec.js b/packages/server/src/api/routes/tests/user.spec.js index 0a2b02364a..e8ffd8df2b 100644 --- a/packages/server/src/api/routes/tests/user.spec.js +++ b/packages/server/src/api/routes/tests/user.spec.js @@ -205,41 +205,4 @@ describe("/users", () => { expect(res.body.message).toEqual("Flag set successfully") }) }) - - describe("syncUser", () => { - it("should sync the user", async () => { - let user = await config.createUser() - await config.createApp("New App") - let res = await request - .post(`/api/users/metadata/sync/${user._id}`) - .set(config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.message).toEqual("User synced.") - }) - - it("should sync the user when a previous user is specified", async () => { - const app1 = await config.createApp("App 1") - const app2 = await config.createApp("App 2") - - let user = await config.createUser({ - builder: false, - admin: true, - roles: { [app1.appId]: "ADMIN" }, - }) - let res = await request - .post(`/api/users/metadata/sync/${user._id}`) - .set(config.defaultHeaders()) - .send({ - previousUser: { - ...user, - roles: { ...user.roles, [app2.appId]: "BASIC" }, - }, - }) - .expect(200) - .expect("Content-Type", /json/) - - expect(res.body.message).toEqual("User synced.") - }) - }) }) diff --git a/packages/server/src/api/routes/user.ts b/packages/server/src/api/routes/user.ts index 14deb111e6..24f33140a6 100644 --- a/packages/server/src/api/routes/user.ts +++ b/packages/server/src/api/routes/user.ts @@ -32,11 +32,6 @@ router authorized(PermissionType.USER, PermissionLevel.WRITE), controller.destroyMetadata ) - .post( - "/api/users/metadata/sync/:id", - authorized(PermissionType.USER, PermissionLevel.WRITE), - controller.syncUser - ) .post( "/api/users/flags", authorized(PermissionType.USER, PermissionLevel.WRITE), diff --git a/packages/server/src/events/docUpdates/index.ts b/packages/server/src/events/docUpdates/index.ts new file mode 100644 index 0000000000..fa7a623108 --- /dev/null +++ b/packages/server/src/events/docUpdates/index.ts @@ -0,0 +1 @@ +export * from "./processors" diff --git a/packages/server/src/events/docUpdates/processors.ts b/packages/server/src/events/docUpdates/processors.ts new file mode 100644 index 0000000000..53036970e0 --- /dev/null +++ b/packages/server/src/events/docUpdates/processors.ts @@ -0,0 +1,14 @@ +import userGroupProcessor from "./syncUsers" +import { docUpdates } from "@budibase/backend-core" + +export type UpdateCallback = (docId: string) => void +let started = false + +export function init(updateCb?: UpdateCallback) { + if (started) { + return + } + const processors = [userGroupProcessor(updateCb)] + docUpdates.init(processors) + started = true +} diff --git a/packages/server/src/events/docUpdates/syncUsers.ts b/packages/server/src/events/docUpdates/syncUsers.ts new file mode 100644 index 0000000000..7957178168 --- /dev/null +++ b/packages/server/src/events/docUpdates/syncUsers.ts @@ -0,0 +1,35 @@ +import { constants, logging } from "@budibase/backend-core" +import { sdk as proSdk } from "@budibase/pro" +import { DocUpdateEvent, UserGroupSyncEvents } from "@budibase/types" +import { syncUsersToAllApps } from "../../sdk/app/applications/sync" +import { UpdateCallback } from "./processors" + +export default function process(updateCb?: UpdateCallback) { + const processor = async (update: DocUpdateEvent) => { + try { + const docId = update.id + const isGroup = docId.startsWith(constants.DocumentType.GROUP) + let userIds: string[] + if (isGroup) { + const group = await proSdk.groups.get(docId) + userIds = group.users?.map(user => user._id) || [] + } else { + userIds = [docId] + } + if (userIds.length > 0) { + await syncUsersToAllApps(userIds) + } + if (updateCb) { + updateCb(docId) + } + } catch (err: any) { + // if something not found - no changes to perform + if (err?.status === 404) { + return + } else { + logging.logAlert("Failed to perform user/group app sync", err) + } + } + } + return { events: UserGroupSyncEvents, processor } +} diff --git a/packages/server/src/events/index.ts b/packages/server/src/events/index.ts index fad7bcaa9a..23c3f3e512 100644 --- a/packages/server/src/events/index.ts +++ b/packages/server/src/events/index.ts @@ -2,4 +2,5 @@ import BudibaseEmitter from "./BudibaseEmitter" const emitter = new BudibaseEmitter() +export { init } from "./docUpdates" export default emitter diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 6fb3576ae6..d0a1d78428 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -1,6 +1,117 @@ import env from "../../../environment" -import { db as dbCore, context } from "@budibase/backend-core" +import { + db as dbCore, + context, + docUpdates, + constants, + logging, + roles, +} from "@budibase/backend-core" +import { User, ContextUser, UserGroup } from "@budibase/types" +import { sdk as proSdk } from "@budibase/pro" import sdk from "../../" +import { getGlobalUsers, processUser } from "../../../utilities/global" +import { generateUserMetadataID, InternalTables } from "../../../db/utils" + +type DeletedUser = { _id: string; deleted: boolean } + +async function syncUsersToApp( + appId: string, + users: (User | DeletedUser)[], + groups: UserGroup[] +) { + if (!(await dbCore.dbExists(appId))) { + return + } + await context.doInAppContext(appId, async () => { + const db = context.getAppDB() + for (let user of users) { + let ctxUser = user as ContextUser + let deletedUser = false + const metadataId = generateUserMetadataID(user._id!) + if ((user as DeletedUser).deleted) { + deletedUser = true + } + + // make sure role is correct + if (!deletedUser) { + ctxUser = await processUser(ctxUser, { appId, groups }) + } + let roleId = ctxUser.roleId + if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) { + roleId = undefined + } + + let metadata + try { + metadata = await db.get(metadataId) + } catch (err: any) { + if (err.status !== 404) { + throw err + } + // no metadata and user is to be deleted, can skip + // no role - user isn't in app anyway + if (!roleId) { + continue + } else if (!deletedUser) { + // doesn't exist yet, creating it + metadata = { + tableId: InternalTables.USER_METADATA, + } + } + } + + // the user doesn't exist, or doesn't have a role anymore + // get rid of their metadata + if (deletedUser || !roleId) { + await db.remove(metadata) + continue + } + + // assign the roleId for the metadata doc + if (roleId) { + metadata.roleId = roleId + } + + let combined = sdk.users.combineMetadataAndUser(ctxUser, metadata) + // if no combined returned, there are no updates to make + if (combined) { + await db.put(combined) + } + } + }) +} + +export async function syncUsersToAllApps(userIds: string[]) { + // list of users, if one has been deleted it will be undefined in array + const users = (await getGlobalUsers(userIds, { + noProcessing: true, + })) as User[] + const groups = await proSdk.groups.fetch() + const finalUsers: (User | DeletedUser)[] = [] + for (let userId of userIds) { + const user = users.find(user => user._id === userId) + if (!user) { + finalUsers.push({ _id: userId, deleted: true }) + } else { + finalUsers.push(user) + } + } + const devAppIds = await dbCore.getDevAppIDs() + let promises = [] + for (let devAppId of devAppIds) { + const prodAppId = dbCore.getProdAppID(devAppId) + for (let appId of [prodAppId, devAppId]) { + promises.push(syncUsersToApp(appId, finalUsers, groups)) + } + } + const resp = await Promise.allSettled(promises) + const failed = resp.filter(promise => promise.status === "rejected") + if (failed.length > 0) { + const reasons = failed.map(fail => (fail as PromiseRejectedResult).reason) + logging.logAlert("Failed to sync users to apps", reasons) + } +} export async function syncApp( appId: string, @@ -23,32 +134,28 @@ export async function syncApp( // specific case, want to make sure setup is skipped const prodDb = context.getProdAppDB({ skip_setup: true }) const exists = await prodDb.exists() - if (!exists) { - // the database doesn't exist. Don't replicate - return { - message: "App sync not required, app not deployed.", - } - } - const replication = new dbCore.Replication({ - source: prodAppId, - target: appId, - }) let error - try { - const replOpts = replication.appReplicateOpts() - if (opts?.automationOnly) { - replOpts.filter = (doc: any) => - doc._id.startsWith(dbCore.DocumentType.AUTOMATION) + if (exists) { + const replication = new dbCore.Replication({ + source: prodAppId, + target: appId, + }) + try { + const replOpts = replication.appReplicateOpts() + if (opts?.automationOnly) { + replOpts.filter = (doc: any) => + doc._id.startsWith(dbCore.DocumentType.AUTOMATION) + } + await replication.replicate(replOpts) + } catch (err) { + error = err + } finally { + await replication.close() } - await replication.replicate(replOpts) - } catch (err) { - error = err - } finally { - await replication.close() } - // sync the users + // sync the users - kept for safe keeping await sdk.users.syncGlobalUsers() if (error) { diff --git a/packages/server/src/sdk/app/applications/tests/sync.spec.ts b/packages/server/src/sdk/app/applications/tests/sync.spec.ts new file mode 100644 index 0000000000..8609e59a2f --- /dev/null +++ b/packages/server/src/sdk/app/applications/tests/sync.spec.ts @@ -0,0 +1,137 @@ +import TestConfiguration from "../../../../tests/utilities/TestConfiguration" +import { events, context, roles, constants } from "@budibase/backend-core" +import { init } from "../../../../events" +import { rawUserMetadata } from "../../../users/utils" +import EventEmitter from "events" +import { UserGroup, UserMetadata, UserRoles, User } from "@budibase/types" + +const config = new TestConfiguration() +let app, group: UserGroup, groupUser: User +const ROLE_ID = roles.BUILTIN_ROLE_IDS.BASIC + +const emitter = new EventEmitter() + +function updateCb(docId: string) { + const isGroup = docId.startsWith(constants.DocumentType.GROUP) + if (isGroup) { + emitter.emit("update-group") + } else { + emitter.emit("update-user") + } +} + +init(updateCb) + +function waitForUpdate(opts: { group?: boolean }) { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject() + }, 5000) + const event = opts?.group ? "update-group" : "update-user" + emitter.on(event, () => { + clearTimeout(timeout) + resolve() + }) + }) +} + +beforeAll(async () => { + app = await config.init("syncApp") +}) + +async function createUser(email: string, roles: UserRoles, builder?: boolean) { + const user = await config.createUser({ + email, + roles, + builder: builder || false, + admin: false, + }) + await context.doInContext(config.appId!, async () => { + await events.user.created(user) + }) + return user +} + +async function removeUserRole(user: User) { + const final = await config.globalUser({ + ...user, + id: user._id, + roles: {}, + builder: false, + admin: false, + }) + await context.doInContext(config.appId!, async () => { + await events.user.updated(final) + }) +} + +async function createGroupAndUser(email: string) { + groupUser = await config.createUser({ + email, + roles: {}, + builder: false, + admin: false, + }) + group = await config.createGroup() + await config.addUserToGroup(group._id!, groupUser._id!) +} + +async function removeUserFromGroup() { + await config.removeUserFromGroup(group._id!, groupUser._id!) + return context.doInContext(config.appId!, async () => { + await events.user.updated(groupUser) + }) +} + +async function getUserMetadata(): Promise { + return context.doInContext(config.appId!, async () => { + return await rawUserMetadata() + }) +} + +function buildRoles() { + return { [config.prodAppId!]: ROLE_ID } +} + +describe("app user/group sync", () => { + const groupEmail = "test2@test.com", + normalEmail = "test@test.com" + async function checkEmail( + email: string, + opts?: { group?: boolean; notFound?: boolean } + ) { + await waitForUpdate(opts || {}) + const metadata = await getUserMetadata() + const found = metadata.find(data => data.email === email) + if (opts?.notFound) { + expect(found).toBeUndefined() + } else { + expect(found).toBeDefined() + } + } + + it("should be able to sync a new user, add then remove", async () => { + const user = await createUser(normalEmail, buildRoles()) + await checkEmail(normalEmail) + await removeUserRole(user) + await checkEmail(normalEmail, { notFound: true }) + }) + + it("should be able to sync a group", async () => { + await createGroupAndUser(groupEmail) + await checkEmail(groupEmail, { group: true }) + }) + + it("should be able to remove user from group", async () => { + if (!group) { + await createGroupAndUser(groupEmail) + } + await removeUserFromGroup() + await checkEmail(groupEmail, { notFound: true }) + }) + + it("should be able to handle builder users", async () => { + await createUser("test3@test.com", {}, true) + await checkEmail("test3@test.com") + }) +}) diff --git a/packages/server/src/sdk/users/tests/utils.spec.ts b/packages/server/src/sdk/users/tests/utils.spec.ts index 11c2c53643..5c6777df59 100644 --- a/packages/server/src/sdk/users/tests/utils.spec.ts +++ b/packages/server/src/sdk/users/tests/utils.spec.ts @@ -121,38 +121,7 @@ describe("syncGlobalUsers", () => { await syncGlobalUsers() const metadata = await rawUserMetadata() - expect(metadata).toHaveLength(1) - }) - }) - }) - - it("app users are removed when app is removed from user group", async () => { - await config.doInTenant(async () => { - const group = await proSdk.groups.save(structures.userGroups.userGroup()) - const user1 = await config.createUser({ admin: false, builder: false }) - const user2 = await config.createUser({ admin: false, builder: false }) - await proSdk.groups.updateGroupApps(group.id, { - appsToAdd: [ - { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC }, - ], - }) - await proSdk.groups.addUsers(group.id, [user1._id, user2._id]) - - await config.doInContext(config.appId, async () => { - await syncGlobalUsers() - expect(await rawUserMetadata()).toHaveLength(3) - - await proSdk.groups.removeUsers(group.id, [user1._id]) - await syncGlobalUsers() - - const metadata = await rawUserMetadata() - expect(metadata).toHaveLength(2) - - expect(metadata).not.toContainEqual( - expect.objectContaining({ - _id: db.generateUserMetadataID(user1._id), - }) - ) + expect(metadata).toHaveLength(0) }) }) }) diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index 9b9ea04c56..7f7f0b4809 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -1,12 +1,13 @@ import { getGlobalUsers } from "../../utilities/global" import { context, roles as rolesCore } from "@budibase/backend-core" import { + getGlobalIDFromUserMetadataID, generateUserMetadataID, getUserMetadataParams, InternalTables, } from "../../db/utils" import { isEqual } from "lodash" -import { ContextUser, UserMetadata } from "@budibase/types" +import { ContextUser, UserMetadata, User } from "@budibase/types" export function combineMetadataAndUser( user: ContextUser, @@ -37,6 +38,10 @@ export function combineMetadataAndUser( if (found) { newDoc._rev = found._rev } + // clear fields that shouldn't be in metadata + delete newDoc.password + delete newDoc.forceResetPassword + delete newDoc.roles if (found == null || !isEqual(newDoc, found)) { return { ...found, @@ -60,10 +65,9 @@ export async function rawUserMetadata() { export async function syncGlobalUsers() { // sync user metadata const db = context.getAppDB() - const [users, metadata] = await Promise.all([ - getGlobalUsers(), - rawUserMetadata(), - ]) + const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()]) + const users = resp[0] as User[] + const metadata = resp[1] as UserMetadata[] const toWrite = [] for (let user of users) { const combined = combineMetadataAndUser(user, metadata) @@ -71,5 +75,19 @@ export async function syncGlobalUsers() { toWrite.push(combined) } } + let foundEmails: string[] = [] + for (let data of metadata) { + if (!data._id) { + continue + } + const alreadyExisting = data.email && foundEmails.indexOf(data.email) !== -1 + const globalId = getGlobalIDFromUserMetadataID(data._id) + if (!users.find(user => user._id === globalId) || alreadyExisting) { + toWrite.push({ ...data, _deleted: true }) + } + if (data.email) { + foundEmails.push(data.email) + } + } await db.bulkDocs(toWrite) } diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index 95d0715185..2fd59b7a8e 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -10,7 +10,7 @@ import fs from "fs" import { watch } from "./watch" import * as automations from "./automations" import * as fileSystem from "./utilities/fileSystem" -import eventEmitter from "./events" +import { default as eventEmitter, init as eventInit } from "./events" import * as migrations from "./migrations" import * as bullboard from "./automations/bullboard" import * as pro from "@budibase/pro" @@ -63,6 +63,7 @@ export async function startup(app?: any, server?: any) { eventEmitter.emitPort(env.PORT) fileSystem.init() await redis.init() + eventInit() // run migrations on startup if not done via http // not recommended in a clustered environment diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index ca48dd7f86..f5887e6558 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -49,6 +49,7 @@ import { SearchFilters, UserRoles, } from "@budibase/types" +import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/src/security/roles" type DefaultUserValues = { globalUserId: string @@ -306,6 +307,33 @@ class TestConfiguration { } } + async createGroup(roleId: string = BUILTIN_ROLE_IDS.BASIC) { + return context.doInTenant(this.tenantId!, async () => { + const baseGroup = structures.userGroups.userGroup() + baseGroup.roles = { + [this.prodAppId]: roleId, + } + const { id, rev } = await pro.sdk.groups.save(baseGroup) + return { + _id: id, + _rev: rev, + ...baseGroup, + } + }) + } + + async addUserToGroup(groupId: string, userId: string) { + return context.doInTenant(this.tenantId!, async () => { + await pro.sdk.groups.addUsers(groupId, [userId]) + }) + } + + async removeUserFromGroup(groupId: string, userId: string) { + return context.doInTenant(this.tenantId!, async () => { + await pro.sdk.groups.removeUsers(groupId, [userId]) + }) + } + async login({ roleId, userId, builder, prodApp = false }: any = {}) { const appId = prodApp ? this.prodAppId : this.appId return context.doInAppContext(appId, async () => { diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index a75fcc0b30..21e86a28b9 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -9,6 +9,7 @@ import { import env from "../environment" import { groups } from "@budibase/pro" import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types" +import { global } from "yargs" export function updateAppRole( user: ContextUser, @@ -16,7 +17,7 @@ export function updateAppRole( ) { appId = appId || context.getAppId() - if (!user || !user.roles) { + if (!user || (!user.roles && !user.userGroups)) { return user } // if in an multi-tenancy environment make sure roles are never updated @@ -27,7 +28,7 @@ export function updateAppRole( return user } // always use the deployed app - if (appId) { + if (appId && user.roles) { user.roleId = user.roles[dbCore.getProdAppID(appId)] } // if a role wasn't found then either set as admin (builder) or public (everyone else) @@ -60,7 +61,7 @@ async function checkGroupRoles( return user } -async function processUser( +export async function processUser( user: ContextUser, opts: { appId?: string; groups?: UserGroup[] } = {} ) { @@ -94,16 +95,15 @@ export async function getGlobalUser(userId: string) { return processUser(user, { appId }) } -export async function getGlobalUsers(users?: ContextUser[]) { +export async function getGlobalUsers( + userIds?: string[], + opts?: { noProcessing?: boolean } +) { const appId = context.getAppId() const db = tenancy.getGlobalDB() - const allGroups = await groups.fetch() let globalUsers - if (users) { - const globalIds = users.map(user => - getGlobalIDFromUserMetadataID(user._id!) - ) - globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map( + if (userIds) { + globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( row => row.doc ) } else { @@ -126,15 +126,20 @@ export async function getGlobalUsers(users?: ContextUser[]) { return globalUsers } - // pass in the groups, meaning we don't actually need to retrieve them for - // each user individually - return Promise.all( - globalUsers.map(user => processUser(user, { groups: allGroups })) - ) + if (opts?.noProcessing) { + return globalUsers + } else { + // pass in the groups, meaning we don't actually need to retrieve them for + // each user individually + const allGroups = await groups.fetch() + return Promise.all( + globalUsers.map(user => processUser(user, { groups: allGroups })) + ) + } } export async function getGlobalUsersFromMetadata(users: ContextUser[]) { - const globalUsers = await getGlobalUsers(users) + const globalUsers = await getGlobalUsers(users.map(user => user._id!)) return users.map(user => { const globalUser = globalUsers.find( globalUser => globalUser && user._id?.includes(globalUser._id) diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index 9eb0c26ebc..adff1822a4 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/shared-core", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Shared data utils", "main": "dist/cjs/src/index.js", "types": "dist/mjs/src/index.d.ts", @@ -20,7 +20,7 @@ "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\"" }, "dependencies": { - "@budibase/types": "2.5.6-alpha.1" + "@budibase/types": "2.5.6-alpha.2" }, "devDependencies": { "concurrently": "^7.6.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index f4def12c14..01840f3546 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index e36d039375..f7ede811a1 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase types", "main": "dist/cjs/index.js", "types": "dist/mjs/index.d.ts", diff --git a/packages/types/src/documents/app/user.ts b/packages/types/src/documents/app/user.ts index b5f31ca349..4defd4a414 100644 --- a/packages/types/src/documents/app/user.ts +++ b/packages/types/src/documents/app/user.ts @@ -2,4 +2,5 @@ import { Document } from "../document" export interface UserMetadata extends Document { roleId: string + email?: string } diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 0d59576435..c4990f869b 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -1,4 +1,5 @@ import { Hosting } from "../hosting" +import { Group, Identity } from "./identification" export enum Event { // USER @@ -186,6 +187,24 @@ export enum Event { AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded", } +export const UserGroupSyncEvents: Event[] = [ + Event.USER_CREATED, + Event.USER_UPDATED, + Event.USER_DELETED, + Event.USER_PERMISSION_ADMIN_ASSIGNED, + Event.USER_PERMISSION_ADMIN_REMOVED, + Event.USER_PERMISSION_BUILDER_ASSIGNED, + Event.USER_PERMISSION_BUILDER_REMOVED, + Event.USER_GROUP_CREATED, + Event.USER_GROUP_UPDATED, + Event.USER_GROUP_DELETED, + Event.USER_GROUP_USERS_ADDED, + Event.USER_GROUP_USERS_REMOVED, + Event.USER_GROUP_PERMISSIONS_EDITED, +] + +export const AsyncEvents: Event[] = [...UserGroupSyncEvents] + // all events that are not audited have been added to this record as undefined, this means // that Typescript can protect us against new events being added and auditing of those // events not being considered. This might be a little ugly, but provides a level of @@ -383,3 +402,21 @@ export interface BaseEvent { } export type TableExportFormat = "json" | "csv" + +export type DocUpdateEvent = { + id: string + tenantId: string + appId?: string +} + +export interface EventProcessor { + processEvent( + event: Event, + identity: Identity, + properties: any, + timestamp?: string | number + ): Promise + identify?(identity: Identity, timestamp?: string | number): Promise + identifyGroup?(group: Group, timestamp?: string | number): Promise + shutdown?(): void +} diff --git a/packages/types/src/sdk/events/identification.ts b/packages/types/src/sdk/events/identification.ts index 627254882e..7c7a2be8e0 100644 --- a/packages/types/src/sdk/events/identification.ts +++ b/packages/types/src/sdk/events/identification.ts @@ -46,6 +46,8 @@ export interface Identity { environment: string installationId?: string tenantId?: string + // usable - no unique format + realTenantId?: string hostInfo?: HostInfo } diff --git a/packages/worker/package.json b/packages/worker/package.json index 972437389b..f67a569935 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.5.6-alpha.1", + "version": "2.5.6-alpha.2", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -37,10 +37,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "2.5.6-alpha.1", - "@budibase/pro": "2.5.6-alpha.1", - "@budibase/string-templates": "2.5.6-alpha.1", - "@budibase/types": "2.5.6-alpha.1", + "@budibase/backend-core": "2.5.6-alpha.2", + "@budibase/pro": "2.5.6-alpha.2", + "@budibase/string-templates": "2.5.6-alpha.2", + "@budibase/types": "2.5.6-alpha.2", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index 9b53866f37..f87aa25ccc 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -1,5 +1,4 @@ import env from "../../environment" -import * as apps from "../../utilities/appService" import * as eventHelpers from "./events" import { accounts, @@ -30,7 +29,6 @@ import { PlatformUser, PlatformUserByEmail, RowResponse, - SearchUsersRequest, User, SaveUserOpts, Account, @@ -280,9 +278,6 @@ export const save = async ( await platform.users.addUser(tenantId, builtUser._id!, builtUser.email) await cache.user.invalidateUser(response.id) - // let server know to sync user - await apps.syncUserInApps(_id, dbUser) - await Promise.all(groupPromises) // finally returned the saved user from the db @@ -432,7 +427,6 @@ export const bulkCreate = async ( // instead of relying on looping tenant creation await platform.users.addUser(tenantId, user._id, user.email) await eventHelpers.handleSaveEvents(user, undefined) - await apps.syncUserInApps(user._id) } const saved = usersToBulkSave.map(user => { @@ -571,8 +565,6 @@ export const destroy = async (id: string) => { await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) await sessions.invalidateSessions(userId, { reason: "deletion" }) - // let server know to sync user - await apps.syncUserInApps(userId, dbUser) } const bulkDeleteProcessing = async (dbUser: User) => { @@ -581,8 +573,6 @@ const bulkDeleteProcessing = async (dbUser: User) => { await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) await sessions.invalidateSessions(userId, { reason: "bulk-deletion" }) - // let server know to sync user - await apps.syncUserInApps(userId, dbUser) } export const invite = async ( diff --git a/packages/worker/src/utilities/appService.ts b/packages/worker/src/utilities/appService.ts deleted file mode 100644 index 8f411d58fa..0000000000 --- a/packages/worker/src/utilities/appService.ts +++ /dev/null @@ -1,46 +0,0 @@ -import fetch from "node-fetch" -import { - constants, - tenancy, - logging, - env as coreEnv, -} from "@budibase/backend-core" -import { checkSlashesInUrl } from "../utilities" -import env from "../environment" -import { SyncUserRequest, User } from "@budibase/types" - -async function makeAppRequest(url: string, method: string, body: any) { - if (env.isTest()) { - return - } - const request: any = { headers: {} } - request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY - if (tenancy.isTenantIdSet()) { - request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId() - } - if (body) { - request.headers["Content-Type"] = "application/json" - request.body = JSON.stringify(body) - } - request.method = method - - // add x-budibase-correlation-id header - logging.correlation.setHeader(request.headers) - - return fetch(checkSlashesInUrl(env.APPS_URL + url), request) -} - -export async function syncUserInApps(userId: string, previousUser?: User) { - const body: SyncUserRequest = { - previousUser, - } - - const response = await makeAppRequest( - `/api/users/metadata/sync/${userId}`, - "POST", - body - ) - if (response && response.status !== 200) { - throw "Unable to sync user." - } -} diff --git a/qa-core/src/internal-api/tests/applications/publish.spec.ts b/qa-core/src/internal-api/tests/applications/publish.spec.ts index 3d5fa7c598..e614a0f2a4 100644 --- a/qa-core/src/internal-api/tests/applications/publish.spec.ts +++ b/qa-core/src/internal-api/tests/applications/publish.spec.ts @@ -36,7 +36,7 @@ describe("Internal API - Application creation, update, publish and delete", () = const [syncResponse, sync] = await config.api.apps.sync(app.appId!) expect(sync).toEqual({ - message: "App sync not required, app not deployed.", + message: "App sync completed successfully.", }) })