Merge branch 'develop' of github.com:Budibase/budibase into spreadsheet-integration

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "npmClient": "yarn",
   "useWorkspaces": true,
   "packages": ["packages/*"],

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -24,7 +24,7 @@
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.5.6-alpha.1",
+    "@budibase/types": "2.5.6-alpha.2",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/db/views.ts

@@ -42,7 +42,11 @@ async function removeDeprecated(db: Database, viewName: ViewName) {
   }
 }
 
-export async function createView(db: any, viewJs: string, viewName: string) {
+export async function createView(
+  db: any,
+  viewJs: string,
+  viewName: string
+): Promise<void> {
   let designDoc
   try {
     designDoc = (await db.get(DESIGN_DB)) as DesignDocument
@@ -57,7 +61,15 @@ export async function createView(db: any, viewJs: string, viewName: string) {
     ...designDoc.views,
     [viewName]: view,
   }
-  await db.put(designDoc)
+  try {
+    await db.put(designDoc)
+  } catch (err: any) {
+    if (err.status === 409) {
+      return await createView(db, viewJs, viewName)
+    } else {
+      throw err
+    }
+  }
 }
 
 export const createNewUserEmailView = async () => {
@@ -135,6 +147,10 @@ export const queryView = async <T>(
       await removeDeprecated(db, viewName)
       await createFunc()
       return queryView(viewName, params, db, createFunc, opts)
+    } else if (err.status === 409) {
+      // can happen when multiple queries occur at once, view couldn't be created
+      // other design docs being updated, re-run
+      return queryView(viewName, params, db, createFunc, opts)
     } else {
       throw err
     }

packages/backend-core/src/docUpdates/index.ts

@@ -0,0 +1,29 @@
+import { asyncEventQueue, init as initQueue } from "../events/asyncEvents"
+import {
+  ProcessorMap,
+  default as DocumentUpdateProcessor,
+} from "../events/processors/async/DocumentUpdateProcessor"
+
+let processingPromise: Promise<void>
+let documentProcessor: DocumentUpdateProcessor
+
+export function init(processors: ProcessorMap) {
+  if (!asyncEventQueue) {
+    initQueue()
+  }
+  if (!documentProcessor) {
+    documentProcessor = new DocumentUpdateProcessor(processors)
+  }
+  // if not processing in this instance, kick it off
+  if (!processingPromise) {
+    processingPromise = asyncEventQueue.process(async job => {
+      const { event, identity, properties, timestamp } = job.data
+      await documentProcessor.processEvent(
+        event,
+        identity,
+        properties,
+        timestamp
+      )
+    })
+  }
+}

packages/backend-core/src/events/asyncEvents/index.ts

@@ -0,0 +1,2 @@
+export * from "./queue"
+export * from "./publisher"

packages/backend-core/src/events/asyncEvents/publisher.ts

@@ -0,0 +1,12 @@
+import { AsyncEvents } from "@budibase/types"
+import { EventPayload, asyncEventQueue, init } from "./queue"
+
+export async function publishAsyncEvent(payload: EventPayload) {
+  if (!asyncEventQueue) {
+    init()
+  }
+  const { event, identity } = payload
+  if (AsyncEvents.indexOf(event) !== -1 && identity.tenantId) {
+    await asyncEventQueue.add(payload)
+  }
+}

packages/backend-core/src/events/asyncEvents/queue.ts

@@ -0,0 +1,22 @@
+import BullQueue from "bull"
+import { createQueue, JobQueue } from "../../queue"
+import { Event, Identity } from "@budibase/types"
+
+export interface EventPayload {
+  event: Event
+  identity: Identity
+  properties: any
+  timestamp?: string | number
+}
+
+export let asyncEventQueue: BullQueue.Queue
+
+export function init() {
+  asyncEventQueue = createQueue<EventPayload>(JobQueue.SYSTEM_EVENT_QUEUE)
+}
+
+export async function shutdown() {
+  if (asyncEventQueue) {
+    await asyncEventQueue.close()
+  }
+}

packages/backend-core/src/events/documentId.ts

@@ -0,0 +1,56 @@
+import {
+  Event,
+  UserCreatedEvent,
+  UserUpdatedEvent,
+  UserDeletedEvent,
+  UserPermissionAssignedEvent,
+  UserPermissionRemovedEvent,
+  GroupCreatedEvent,
+  GroupUpdatedEvent,
+  GroupDeletedEvent,
+  GroupUsersAddedEvent,
+  GroupUsersDeletedEvent,
+  GroupPermissionsEditedEvent,
+} from "@budibase/types"
+
+const getEventProperties: Record<
+  string,
+  (properties: any) => string | undefined
+> = {
+  [Event.USER_CREATED]: (properties: UserCreatedEvent) => properties.userId,
+  [Event.USER_UPDATED]: (properties: UserUpdatedEvent) => properties.userId,
+  [Event.USER_DELETED]: (properties: UserDeletedEvent) => properties.userId,
+  [Event.USER_PERMISSION_ADMIN_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => properties.userId,
+  [Event.USER_PERMISSION_ADMIN_REMOVED]: (
+    properties: UserPermissionRemovedEvent
+  ) => properties.userId,
+  [Event.USER_PERMISSION_BUILDER_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => properties.userId,
+  [Event.USER_PERMISSION_BUILDER_REMOVED]: (
+    properties: UserPermissionRemovedEvent
+  ) => properties.userId,
+  [Event.USER_GROUP_CREATED]: (properties: GroupCreatedEvent) =>
+    properties.groupId,
+  [Event.USER_GROUP_UPDATED]: (properties: GroupUpdatedEvent) =>
+    properties.groupId,
+  [Event.USER_GROUP_DELETED]: (properties: GroupDeletedEvent) =>
+    properties.groupId,
+  [Event.USER_GROUP_USERS_ADDED]: (properties: GroupUsersAddedEvent) =>
+    properties.groupId,
+  [Event.USER_GROUP_USERS_REMOVED]: (properties: GroupUsersDeletedEvent) =>
+    properties.groupId,
+  [Event.USER_GROUP_PERMISSIONS_EDITED]: (
+    properties: GroupPermissionsEditedEvent
+  ) => properties.groupId,
+}
+
+export function getDocumentId(event: Event, properties: any) {
+  const extractor = getEventProperties[event]
+  if (!extractor) {
+    throw new Error("Event does not have a method of document ID extraction")
+  }
+  return extractor(properties)
+}

packages/backend-core/src/events/events.ts

@@ -1,7 +1,8 @@
-import { Event, AuditedEventFriendlyName } from "@budibase/types"
+import { Event } from "@budibase/types"
 import { processors } from "./processors"
 import identification from "./identification"
 import * as backfill from "./backfill"
+import { publishAsyncEvent } from "./asyncEvents"
 
 export const publishEvent = async (
   event: Event,
@@ -14,6 +15,14 @@ export const publishEvent = async (
   const backfilling = await backfill.isBackfillingEvent(event)
   // no backfill - send the event and exit
   if (!backfilling) {
+    // send off async events if required
+    await publishAsyncEvent({
+      event,
+      identity,
+      properties,
+      timestamp,
+    })
+    // now handle the main sync event processing pipeline
     await processors.processEvent(event, identity, properties, timestamp)
     return
   }

packages/backend-core/src/events/identification.ts

@@ -65,6 +65,7 @@ const getCurrentIdentity = async (): Promise<Identity> => {
       hosting,
       installationId,
       tenantId,
+      realTenantId: context.getTenantId(),
       environment,
     }
   } else if (identityType === IdentityType.USER) {

packages/backend-core/src/events/index.ts

@@ -6,6 +6,8 @@ export * as backfillCache from "./backfill"
 
 import { processors } from "./processors"
 
+export function initAsyncEvents() {}
+
 export const shutdown = () => {
   processors.shutdown()
   console.log("Events shutdown")

packages/backend-core/src/events/processors/Processors.ts

@@ -25,7 +25,9 @@ export default class Processor implements EventProcessor {
     timestamp?: string | number
   ): Promise<void> {
     for (const eventProcessor of this.processors) {
-      await eventProcessor.identify(identity, timestamp)
+      if (eventProcessor.identify) {
+        await eventProcessor.identify(identity, timestamp)
+      }
     }
   }
 
@@ -34,13 +36,17 @@ export default class Processor implements EventProcessor {
     timestamp?: string | number
   ): Promise<void> {
     for (const eventProcessor of this.processors) {
-      await eventProcessor.identifyGroup(identity, timestamp)
+      if (eventProcessor.identifyGroup) {
+        await eventProcessor.identifyGroup(identity, timestamp)
+      }
     }
   }
 
   shutdown() {
     for (const eventProcessor of this.processors) {
-      eventProcessor.shutdown()
+      if (eventProcessor.shutdown) {
+        eventProcessor.shutdown()
+      }
     }
   }
 }

packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts

@@ -0,0 +1,43 @@
+import { EventProcessor } from "../types"
+import { Event, Identity, DocUpdateEvent } from "@budibase/types"
+import { doInTenant } from "../../../context"
+import { getDocumentId } from "../../documentId"
+import { shutdown } from "../../asyncEvents"
+
+export type Processor = (update: DocUpdateEvent) => Promise<void>
+export type ProcessorMap = { events: Event[]; processor: Processor }[]
+
+export default class DocumentUpdateProcessor implements EventProcessor {
+  processors: ProcessorMap = []
+
+  constructor(processors: ProcessorMap) {
+    this.processors = processors
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ) {
+    const tenantId = identity.realTenantId
+    const docId = getDocumentId(event, properties)
+    if (!tenantId || !docId) {
+      return
+    }
+    for (let { events, processor } of this.processors) {
+      if (events.includes(event)) {
+        await doInTenant(tenantId, async () => {
+          await processor({
+            id: docId,
+            tenantId,
+          })
+        })
+      }
+    }
+  }
+
+  shutdown() {
+    return shutdown()
+  }
+}

packages/backend-core/src/events/processors/types.ts

@@ -1,18 +1 @@
-import { Event, Identity, Group } from "@budibase/types"
-
-export enum EventProcessorType {
-  POSTHOG = "posthog",
-  LOGGING = "logging",
-}
-
-export interface EventProcessor {
-  processEvent(
-    event: Event,
-    identity: Identity,
-    properties: any,
-    timestamp?: string | number
-  ): Promise<void>
-  identify(identity: Identity, timestamp?: string | number): Promise<void>
-  identifyGroup(group: Group, timestamp?: string | number): Promise<void>
-  shutdown(): void
-}
+export { EventProcessor } from "@budibase/types"

packages/backend-core/src/index.ts

@@ -27,6 +27,7 @@ export * as errors from "./errors"
 export * as timers from "./timers"
 export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
+export * as docUpdates from "./docUpdates"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal

packages/backend-core/src/queue/constants.ts

@@ -2,4 +2,5 @@ export enum JobQueue {
   AUTOMATION = "automationQueue",
   APP_BACKUP = "appBackupQueue",
   AUDIT_LOG = "auditLogQueue",
+  SYSTEM_EVENT_QUEUE = "systemEventQueue",
 }

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,8 +38,8 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/shared-core": "2.5.6-alpha.1",
-    "@budibase/string-templates": "2.5.6-alpha.1",
+    "@budibase/shared-core": "2.5.6-alpha.2",
+    "@budibase/string-templates": "2.5.6-alpha.2",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,11 +58,11 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.5.6-alpha.1",
-    "@budibase/client": "2.5.6-alpha.1",
-    "@budibase/frontend-core": "2.5.6-alpha.1",
-    "@budibase/shared-core": "2.5.6-alpha.1",
-    "@budibase/string-templates": "2.5.6-alpha.1",
+    "@budibase/bbui": "2.5.6-alpha.2",
+    "@budibase/client": "2.5.6-alpha.2",
+    "@budibase/frontend-core": "2.5.6-alpha.2",
+    "@budibase/shared-core": "2.5.6-alpha.2",
+    "@budibase/string-templates": "2.5.6-alpha.2",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "dist/index.js",
   "bin": {
@@ -29,9 +29,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.5.6-alpha.1",
-    "@budibase/string-templates": "2.5.6-alpha.1",
-    "@budibase/types": "2.5.6-alpha.1",
+    "@budibase/backend-core": "2.5.6-alpha.2",
+    "@budibase/string-templates": "2.5.6-alpha.2",
+    "@budibase/types": "2.5.6-alpha.2",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,11 +19,11 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.5.6-alpha.1",
-    "@budibase/frontend-core": "2.5.6-alpha.1",
-    "@budibase/shared-core": "2.5.6-alpha.1",
-    "@budibase/string-templates": "2.5.6-alpha.1",
-    "@budibase/types": "2.5.6-alpha.1",
+    "@budibase/bbui": "2.5.6-alpha.2",
+    "@budibase/frontend-core": "2.5.6-alpha.2",
+    "@budibase/shared-core": "2.5.6-alpha.2",
+    "@budibase/string-templates": "2.5.6-alpha.2",
+    "@budibase/types": "2.5.6-alpha.2",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.5.6-alpha.1",
-    "@budibase/shared-core": "2.5.6-alpha.1",
+    "@budibase/bbui": "2.5.6-alpha.2",
+    "@budibase/shared-core": "2.5.6-alpha.2",
     "dayjs": "^1.11.7",
     "lodash": "^4.17.21",
     "socket.io-client": "^4.6.1",

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -45,12 +45,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.5.6-alpha.1",
-    "@budibase/client": "2.5.6-alpha.1",
-    "@budibase/pro": "2.5.6-alpha.1",
-    "@budibase/shared-core": "2.5.6-alpha.1",
-    "@budibase/string-templates": "2.5.6-alpha.1",
-    "@budibase/types": "2.5.6-alpha.1",
+    "@budibase/backend-core": "2.5.6-alpha.2",
+    "@budibase/client": "2.5.6-alpha.2",
+    "@budibase/pro": "2.5.6-alpha.2",
+    "@budibase/shared-core": "2.5.6-alpha.2",
+    "@budibase/string-templates": "2.5.6-alpha.2",
+    "@budibase/types": "2.5.6-alpha.2",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/row/internal.ts

@@ -30,7 +30,6 @@ import { finaliseRow, updateRelatedFormula } from "./staticFormula"
 import { csv, json, jsonWithSchema, Format } from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
-  Ctx,
   UserCtx,
   Database,
   LinkDocumentValue,
@@ -72,7 +71,7 @@ async function getView(db: Database, viewName: string) {
   return viewInfo
 }
 
-async function getRawTableData(ctx: Ctx, db: Database, tableId: string) {
+async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
   let rows
   if (tableId === InternalTables.USER_METADATA) {
     await userController.fetchMetadata(ctx)
@@ -188,7 +187,7 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function fetchView(ctx: Ctx) {
+export async function fetchView(ctx: UserCtx) {
   const viewName = decodeURIComponent(ctx.params.viewName)
 
   // if this is a table view being looked for just transfer to that
@@ -255,7 +254,7 @@ export async function fetchView(ctx: Ctx) {
   return rows
 }
 
-export async function fetch(ctx: Ctx) {
+export async function fetch(ctx: UserCtx) {
   const db = context.getAppDB()
 
   const tableId = ctx.params.tableId
@@ -264,7 +263,7 @@ export async function fetch(ctx: Ctx) {
   return outputProcessing(table, rows)
 }
 
-export async function find(ctx: Ctx) {
+export async function find(ctx: UserCtx) {
   const db = dbCore.getDB(ctx.appId)
   const table = await db.get(ctx.params.tableId)
   let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId)
@@ -272,7 +271,7 @@ export async function find(ctx: Ctx) {
   return row
 }
 
-export async function destroy(ctx: Ctx) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const { _id } = ctx.request.body
   let row = await db.get(_id)
@@ -308,7 +307,7 @@ export async function destroy(ctx: Ctx) {
   return { response, row }
 }
 
-export async function bulkDestroy(ctx: Ctx) {
+export async function bulkDestroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId
   const table = await db.get(tableId)
@@ -347,7 +346,7 @@ export async function bulkDestroy(ctx: Ctx) {
   return { response: { ok: true }, rows: processedRows }
 }
 
-export async function search(ctx: Ctx) {
+export async function search(ctx: UserCtx) {
   // Fetch the whole table when running in cypress, as search doesn't work
   if (!env.COUCH_DB_URL && env.isCypress()) {
     return { rows: await fetch(ctx) }
@@ -387,7 +386,7 @@ export async function search(ctx: Ctx) {
   return response
 }
 
-export async function exportRows(ctx: Ctx) {
+export async function exportRows(ctx: UserCtx) {
   const db = context.getAppDB()
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
@@ -439,7 +438,7 @@ export async function exportRows(ctx: Ctx) {
   }
 }
 
-export async function fetchEnrichedRow(ctx: Ctx) {
+export async function fetchEnrichedRow(ctx: UserCtx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId
   const rowId = ctx.params.rowId

packages/server/src/api/controllers/row/utils.ts

@@ -5,7 +5,7 @@ import { context } from "@budibase/backend-core"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Row, Table } from "@budibase/types"
 import { Format } from "../view/exporters"
-import { Ctx } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
 import sdk from "../../../sdk"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
@@ -26,7 +26,7 @@ export async function getDatasourceAndQuery(json: any) {
   return makeExternalQuery(datasource, json)
 }
 
-export async function findRow(ctx: Ctx, tableId: string, rowId: string) {
+export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   const db = context.getAppDB()
   let row
   // TODO remove special user case in future

packages/server/src/api/controllers/user.ts

@@ -1,98 +1,12 @@
 import { generateUserMetadataID, generateUserFlagID } from "../../db/utils"
 import { InternalTables } from "../../db/utils"
-import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global"
+import { getGlobalUsers } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
-import {
-  context,
-  roles as rolesCore,
-  db as dbCore,
-} from "@budibase/backend-core"
-import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types"
+import { context } from "@budibase/backend-core"
+import { UserCtx } from "@budibase/types"
 import sdk from "../../sdk"
 
-export async function syncUser(ctx: Ctx<SyncUserRequest>) {
-  let deleting = false,
-    user: User | any
-  const userId = ctx.params.id
-
-  const previousUser = ctx.request.body?.previousUser
-
-  try {
-    user = (await getRawGlobalUser(userId)) as User
-  } catch (err: any) {
-    if (err && err.status === 404) {
-      user = {}
-      deleting = true
-    } else {
-      throw err
-    }
-  }
-
-  let previousApps = previousUser
-    ? Object.keys(previousUser.roles).map(appId => appId)
-    : []
-
-  const roles = deleting ? {} : user.roles
-  // remove props which aren't useful to metadata
-  delete user.password
-  delete user.forceResetPassword
-  delete user.roles
-  // run through all production appIDs in the users roles
-  let prodAppIds
-  // if they are a builder then get all production app IDs
-  if ((user.builder && user.builder.global) || deleting) {
-    prodAppIds = await dbCore.getProdAppIDs()
-  } else {
-    prodAppIds = Object.entries(roles)
-      .filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC)
-      .map(([appId]) => appId)
-  }
-  for (let prodAppId of new Set([...prodAppIds, ...previousApps])) {
-    const roleId = roles[prodAppId]
-    const deleteFromApp = !roleId
-    const devAppId = dbCore.getDevelopmentAppID(prodAppId)
-    for (let appId of [prodAppId, devAppId]) {
-      if (!(await dbCore.dbExists(appId))) {
-        continue
-      }
-      await context.doInAppContext(appId, async () => {
-        const db = context.getAppDB()
-        const metadataId = generateUserMetadataID(userId)
-        let metadata
-        try {
-          metadata = await db.get(metadataId)
-        } catch (err) {
-          if (deleteFromApp) {
-            return
-          }
-          metadata = {
-            tableId: InternalTables.USER_METADATA,
-          }
-        }
-
-        if (deleteFromApp) {
-          await db.remove(metadata)
-          return
-        }
-
-        // assign the roleId for the metadata doc
-        if (roleId) {
-          metadata.roleId = roleId
-        }
-        let combined = sdk.users.combineMetadataAndUser(user, metadata)
-        // if its null then there was no updates required
-        if (combined) {
-          await db.put(combined)
-        }
-      })
-    }
-  }
-  ctx.body = {
-    message: "User synced.",
-  }
-}
-
-export async function fetchMetadata(ctx: BBContext) {
+export async function fetchMetadata(ctx: UserCtx) {
   const global = await getGlobalUsers()
   const metadata = await sdk.users.rawUserMetadata()
   const users = []
@@ -111,7 +25,7 @@ export async function fetchMetadata(ctx: BBContext) {
   ctx.body = users
 }
 
-export async function updateSelfMetadata(ctx: BBContext) {
+export async function updateSelfMetadata(ctx: UserCtx) {
   // overwrite the ID with current users
   ctx.request.body._id = ctx.user?._id
   // make sure no stale rev
@@ -121,7 +35,7 @@ export async function updateSelfMetadata(ctx: BBContext) {
   await updateMetadata(ctx)
 }
 
-export async function updateMetadata(ctx: BBContext) {
+export async function updateMetadata(ctx: UserCtx) {
   const db = context.getAppDB()
   const user = ctx.request.body
   // this isn't applicable to the user
@@ -133,7 +47,7 @@ export async function updateMetadata(ctx: BBContext) {
   ctx.body = await db.put(metadata)
 }
 
-export async function destroyMetadata(ctx: BBContext) {
+export async function destroyMetadata(ctx: UserCtx) {
   const db = context.getAppDB()
   try {
     const dbUser = await db.get(ctx.params.id)
@@ -146,11 +60,11 @@ export async function destroyMetadata(ctx: BBContext) {
   }
 }
 
-export async function findMetadata(ctx: BBContext) {
+export async function findMetadata(ctx: UserCtx) {
   ctx.body = await getFullUser(ctx, ctx.params.id)
 }
 
-export async function setFlag(ctx: BBContext) {
+export async function setFlag(ctx: UserCtx) {
   const userId = ctx.user?._id
   const { flag, value } = ctx.request.body
   if (!flag) {
@@ -169,7 +83,7 @@ export async function setFlag(ctx: BBContext) {
   ctx.body = { message: "Flag set successfully" }
 }
 
-export async function getFlags(ctx: BBContext) {
+export async function getFlags(ctx: UserCtx) {
   const userId = ctx.user?._id
   const docId = generateUserFlagID(userId!)
   const db = context.getAppDB()

packages/server/src/api/routes/tests/user.spec.js

@@ -205,41 +205,4 @@ describe("/users", () => {
       expect(res.body.message).toEqual("Flag set successfully")
     })
   })
-
-  describe("syncUser", () => {
-    it("should sync the user", async () => {
-      let user = await config.createUser()
-      await config.createApp("New App")
-      let res = await request
-        .post(`/api/users/metadata/sync/${user._id}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("User synced.")
-    })
-
-    it("should sync the user when a previous user is specified", async () => {
-      const app1 = await config.createApp("App 1")
-      const app2 = await config.createApp("App 2")
-
-      let user = await config.createUser({
-        builder: false,
-        admin: true,
-        roles: { [app1.appId]: "ADMIN" },
-      })
-      let res = await request
-        .post(`/api/users/metadata/sync/${user._id}`)
-        .set(config.defaultHeaders())
-        .send({
-          previousUser: {
-            ...user,
-            roles: { ...user.roles, [app2.appId]: "BASIC" },
-          },
-        })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      expect(res.body.message).toEqual("User synced.")
-    })
-  })
 })

packages/server/src/api/routes/user.ts

@@ -32,11 +32,6 @@ router
     authorized(PermissionType.USER, PermissionLevel.WRITE),
     controller.destroyMetadata
   )
-  .post(
-    "/api/users/metadata/sync/:id",
-    authorized(PermissionType.USER, PermissionLevel.WRITE),
-    controller.syncUser
-  )
   .post(
     "/api/users/flags",
     authorized(PermissionType.USER, PermissionLevel.WRITE),

packages/server/src/events/docUpdates/index.ts

@@ -0,0 +1 @@
+export * from "./processors"

packages/server/src/events/docUpdates/processors.ts

@@ -0,0 +1,14 @@
+import userGroupProcessor from "./syncUsers"
+import { docUpdates } from "@budibase/backend-core"
+
+export type UpdateCallback = (docId: string) => void
+let started = false
+
+export function init(updateCb?: UpdateCallback) {
+  if (started) {
+    return
+  }
+  const processors = [userGroupProcessor(updateCb)]
+  docUpdates.init(processors)
+  started = true
+}

packages/server/src/events/docUpdates/syncUsers.ts

@@ -0,0 +1,35 @@
+import { constants, logging } from "@budibase/backend-core"
+import { sdk as proSdk } from "@budibase/pro"
+import { DocUpdateEvent, UserGroupSyncEvents } from "@budibase/types"
+import { syncUsersToAllApps } from "../../sdk/app/applications/sync"
+import { UpdateCallback } from "./processors"
+
+export default function process(updateCb?: UpdateCallback) {
+  const processor = async (update: DocUpdateEvent) => {
+    try {
+      const docId = update.id
+      const isGroup = docId.startsWith(constants.DocumentType.GROUP)
+      let userIds: string[]
+      if (isGroup) {
+        const group = await proSdk.groups.get(docId)
+        userIds = group.users?.map(user => user._id) || []
+      } else {
+        userIds = [docId]
+      }
+      if (userIds.length > 0) {
+        await syncUsersToAllApps(userIds)
+      }
+      if (updateCb) {
+        updateCb(docId)
+      }
+    } catch (err: any) {
+      // if something not found - no changes to perform
+      if (err?.status === 404) {
+        return
+      } else {
+        logging.logAlert("Failed to perform user/group app sync", err)
+      }
+    }
+  }
+  return { events: UserGroupSyncEvents, processor }
+}

packages/server/src/events/index.ts

@@ -2,4 +2,5 @@ import BudibaseEmitter from "./BudibaseEmitter"
 
 const emitter = new BudibaseEmitter()
 
+export { init } from "./docUpdates"
 export default emitter

packages/server/src/sdk/app/applications/sync.ts

@@ -1,6 +1,117 @@
 import env from "../../../environment"
-import { db as dbCore, context } from "@budibase/backend-core"
+import {
+  db as dbCore,
+  context,
+  docUpdates,
+  constants,
+  logging,
+  roles,
+} from "@budibase/backend-core"
+import { User, ContextUser, UserGroup } from "@budibase/types"
+import { sdk as proSdk } from "@budibase/pro"
 import sdk from "../../"
+import { getGlobalUsers, processUser } from "../../../utilities/global"
+import { generateUserMetadataID, InternalTables } from "../../../db/utils"
+
+type DeletedUser = { _id: string; deleted: boolean }
+
+async function syncUsersToApp(
+  appId: string,
+  users: (User | DeletedUser)[],
+  groups: UserGroup[]
+) {
+  if (!(await dbCore.dbExists(appId))) {
+    return
+  }
+  await context.doInAppContext(appId, async () => {
+    const db = context.getAppDB()
+    for (let user of users) {
+      let ctxUser = user as ContextUser
+      let deletedUser = false
+      const metadataId = generateUserMetadataID(user._id!)
+      if ((user as DeletedUser).deleted) {
+        deletedUser = true
+      }
+
+      // make sure role is correct
+      if (!deletedUser) {
+        ctxUser = await processUser(ctxUser, { appId, groups })
+      }
+      let roleId = ctxUser.roleId
+      if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) {
+        roleId = undefined
+      }
+
+      let metadata
+      try {
+        metadata = await db.get(metadataId)
+      } catch (err: any) {
+        if (err.status !== 404) {
+          throw err
+        }
+        // no metadata and user is to be deleted, can skip
+        // no role - user isn't in app anyway
+        if (!roleId) {
+          continue
+        } else if (!deletedUser) {
+          // doesn't exist yet, creating it
+          metadata = {
+            tableId: InternalTables.USER_METADATA,
+          }
+        }
+      }
+
+      // the user doesn't exist, or doesn't have a role anymore
+      // get rid of their metadata
+      if (deletedUser || !roleId) {
+        await db.remove(metadata)
+        continue
+      }
+
+      // assign the roleId for the metadata doc
+      if (roleId) {
+        metadata.roleId = roleId
+      }
+
+      let combined = sdk.users.combineMetadataAndUser(ctxUser, metadata)
+      // if no combined returned, there are no updates to make
+      if (combined) {
+        await db.put(combined)
+      }
+    }
+  })
+}
+
+export async function syncUsersToAllApps(userIds: string[]) {
+  // list of users, if one has been deleted it will be undefined in array
+  const users = (await getGlobalUsers(userIds, {
+    noProcessing: true,
+  })) as User[]
+  const groups = await proSdk.groups.fetch()
+  const finalUsers: (User | DeletedUser)[] = []
+  for (let userId of userIds) {
+    const user = users.find(user => user._id === userId)
+    if (!user) {
+      finalUsers.push({ _id: userId, deleted: true })
+    } else {
+      finalUsers.push(user)
+    }
+  }
+  const devAppIds = await dbCore.getDevAppIDs()
+  let promises = []
+  for (let devAppId of devAppIds) {
+    const prodAppId = dbCore.getProdAppID(devAppId)
+    for (let appId of [prodAppId, devAppId]) {
+      promises.push(syncUsersToApp(appId, finalUsers, groups))
+    }
+  }
+  const resp = await Promise.allSettled(promises)
+  const failed = resp.filter(promise => promise.status === "rejected")
+  if (failed.length > 0) {
+    const reasons = failed.map(fail => (fail as PromiseRejectedResult).reason)
+    logging.logAlert("Failed to sync users to apps", reasons)
+  }
+}
 
 export async function syncApp(
   appId: string,
@@ -23,32 +134,28 @@ export async function syncApp(
   // specific case, want to make sure setup is skipped
   const prodDb = context.getProdAppDB({ skip_setup: true })
   const exists = await prodDb.exists()
-  if (!exists) {
-    // the database doesn't exist. Don't replicate
-    return {
-      message: "App sync not required, app not deployed.",
-    }
-  }
 
-  const replication = new dbCore.Replication({
-    source: prodAppId,
-    target: appId,
-  })
   let error
-  try {
-    const replOpts = replication.appReplicateOpts()
-    if (opts?.automationOnly) {
-      replOpts.filter = (doc: any) =>
-        doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
+  if (exists) {
+    const replication = new dbCore.Replication({
+      source: prodAppId,
+      target: appId,
+    })
+    try {
+      const replOpts = replication.appReplicateOpts()
+      if (opts?.automationOnly) {
+        replOpts.filter = (doc: any) =>
+          doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
+      }
+      await replication.replicate(replOpts)
+    } catch (err) {
+      error = err
+    } finally {
+      await replication.close()
     }
-    await replication.replicate(replOpts)
-  } catch (err) {
-    error = err
-  } finally {
-    await replication.close()
   }
 
-  // sync the users
+  // sync the users - kept for safe keeping
   await sdk.users.syncGlobalUsers()
 
   if (error) {

packages/server/src/sdk/app/applications/tests/sync.spec.ts

@@ -0,0 +1,137 @@
+import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+import { events, context, roles, constants } from "@budibase/backend-core"
+import { init } from "../../../../events"
+import { rawUserMetadata } from "../../../users/utils"
+import EventEmitter from "events"
+import { UserGroup, UserMetadata, UserRoles, User } from "@budibase/types"
+
+const config = new TestConfiguration()
+let app, group: UserGroup, groupUser: User
+const ROLE_ID = roles.BUILTIN_ROLE_IDS.BASIC
+
+const emitter = new EventEmitter()
+
+function updateCb(docId: string) {
+  const isGroup = docId.startsWith(constants.DocumentType.GROUP)
+  if (isGroup) {
+    emitter.emit("update-group")
+  } else {
+    emitter.emit("update-user")
+  }
+}
+
+init(updateCb)
+
+function waitForUpdate(opts: { group?: boolean }) {
+  return new Promise<void>((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject()
+    }, 5000)
+    const event = opts?.group ? "update-group" : "update-user"
+    emitter.on(event, () => {
+      clearTimeout(timeout)
+      resolve()
+    })
+  })
+}
+
+beforeAll(async () => {
+  app = await config.init("syncApp")
+})
+
+async function createUser(email: string, roles: UserRoles, builder?: boolean) {
+  const user = await config.createUser({
+    email,
+    roles,
+    builder: builder || false,
+    admin: false,
+  })
+  await context.doInContext(config.appId!, async () => {
+    await events.user.created(user)
+  })
+  return user
+}
+
+async function removeUserRole(user: User) {
+  const final = await config.globalUser({
+    ...user,
+    id: user._id,
+    roles: {},
+    builder: false,
+    admin: false,
+  })
+  await context.doInContext(config.appId!, async () => {
+    await events.user.updated(final)
+  })
+}
+
+async function createGroupAndUser(email: string) {
+  groupUser = await config.createUser({
+    email,
+    roles: {},
+    builder: false,
+    admin: false,
+  })
+  group = await config.createGroup()
+  await config.addUserToGroup(group._id!, groupUser._id!)
+}
+
+async function removeUserFromGroup() {
+  await config.removeUserFromGroup(group._id!, groupUser._id!)
+  return context.doInContext(config.appId!, async () => {
+    await events.user.updated(groupUser)
+  })
+}
+
+async function getUserMetadata(): Promise<UserMetadata[]> {
+  return context.doInContext(config.appId!, async () => {
+    return await rawUserMetadata()
+  })
+}
+
+function buildRoles() {
+  return { [config.prodAppId!]: ROLE_ID }
+}
+
+describe("app user/group sync", () => {
+  const groupEmail = "test2@test.com",
+    normalEmail = "test@test.com"
+  async function checkEmail(
+    email: string,
+    opts?: { group?: boolean; notFound?: boolean }
+  ) {
+    await waitForUpdate(opts || {})
+    const metadata = await getUserMetadata()
+    const found = metadata.find(data => data.email === email)
+    if (opts?.notFound) {
+      expect(found).toBeUndefined()
+    } else {
+      expect(found).toBeDefined()
+    }
+  }
+
+  it("should be able to sync a new user, add then remove", async () => {
+    const user = await createUser(normalEmail, buildRoles())
+    await checkEmail(normalEmail)
+    await removeUserRole(user)
+    await checkEmail(normalEmail, { notFound: true })
+  })
+
+  it("should be able to sync a group", async () => {
+    await createGroupAndUser(groupEmail)
+    await checkEmail(groupEmail, { group: true })
+  })
+
+  it("should be able to remove user from group", async () => {
+    if (!group) {
+      await createGroupAndUser(groupEmail)
+    }
+    await removeUserFromGroup()
+    await checkEmail(groupEmail, { notFound: true })
+  })
+
+  it("should be able to handle builder users", async () => {
+    await createUser("test3@test.com", {}, true)
+    await checkEmail("test3@test.com")
+  })
+})

packages/server/src/sdk/users/tests/utils.spec.ts

@@ -121,38 +121,7 @@ describe("syncGlobalUsers", () => {
         await syncGlobalUsers()
 
         const metadata = await rawUserMetadata()
-        expect(metadata).toHaveLength(1)
-      })
-    })
-  })
-
-  it("app users are removed when app is removed from user group", async () => {
-    await config.doInTenant(async () => {
-      const group = await proSdk.groups.save(structures.userGroups.userGroup())
-      const user1 = await config.createUser({ admin: false, builder: false })
-      const user2 = await config.createUser({ admin: false, builder: false })
-      await proSdk.groups.updateGroupApps(group.id, {
-        appsToAdd: [
-          { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC },
-        ],
-      })
-      await proSdk.groups.addUsers(group.id, [user1._id, user2._id])
-
-      await config.doInContext(config.appId, async () => {
-        await syncGlobalUsers()
-        expect(await rawUserMetadata()).toHaveLength(3)
-
-        await proSdk.groups.removeUsers(group.id, [user1._id])
-        await syncGlobalUsers()
-
-        const metadata = await rawUserMetadata()
-        expect(metadata).toHaveLength(2)
-
-        expect(metadata).not.toContainEqual(
-          expect.objectContaining({
-            _id: db.generateUserMetadataID(user1._id),
-          })
-        )
+        expect(metadata).toHaveLength(0)
       })
     })
   })

packages/server/src/sdk/users/utils.ts

@@ -1,12 +1,13 @@
 import { getGlobalUsers } from "../../utilities/global"
 import { context, roles as rolesCore } from "@budibase/backend-core"
 import {
+  getGlobalIDFromUserMetadataID,
   generateUserMetadataID,
   getUserMetadataParams,
   InternalTables,
 } from "../../db/utils"
 import { isEqual } from "lodash"
-import { ContextUser, UserMetadata } from "@budibase/types"
+import { ContextUser, UserMetadata, User } from "@budibase/types"
 
 export function combineMetadataAndUser(
   user: ContextUser,
@@ -37,6 +38,10 @@ export function combineMetadataAndUser(
   if (found) {
     newDoc._rev = found._rev
   }
+  // clear fields that shouldn't be in metadata
+  delete newDoc.password
+  delete newDoc.forceResetPassword
+  delete newDoc.roles
   if (found == null || !isEqual(newDoc, found)) {
     return {
       ...found,
@@ -60,10 +65,9 @@ export async function rawUserMetadata() {
 export async function syncGlobalUsers() {
   // sync user metadata
   const db = context.getAppDB()
-  const [users, metadata] = await Promise.all([
-    getGlobalUsers(),
-    rawUserMetadata(),
-  ])
+  const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()])
+  const users = resp[0] as User[]
+  const metadata = resp[1] as UserMetadata[]
   const toWrite = []
   for (let user of users) {
     const combined = combineMetadataAndUser(user, metadata)
@@ -71,5 +75,19 @@ export async function syncGlobalUsers() {
       toWrite.push(combined)
     }
   }
+  let foundEmails: string[] = []
+  for (let data of metadata) {
+    if (!data._id) {
+      continue
+    }
+    const alreadyExisting = data.email && foundEmails.indexOf(data.email) !== -1
+    const globalId = getGlobalIDFromUserMetadataID(data._id)
+    if (!users.find(user => user._id === globalId) || alreadyExisting) {
+      toWrite.push({ ...data, _deleted: true })
+    }
+    if (data.email) {
+      foundEmails.push(data.email)
+    }
+  }
   await db.bulkDocs(toWrite)
 }

packages/server/src/startup.ts

@@ -10,7 +10,7 @@ import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
 import * as fileSystem from "./utilities/fileSystem"
-import eventEmitter from "./events"
+import { default as eventEmitter, init as eventInit } from "./events"
 import * as migrations from "./migrations"
 import * as bullboard from "./automations/bullboard"
 import * as pro from "@budibase/pro"
@@ -63,6 +63,7 @@ export async function startup(app?: any, server?: any) {
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
+  eventInit()
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -49,6 +49,7 @@ import {
   SearchFilters,
   UserRoles,
 } from "@budibase/types"
+import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/src/security/roles"
 
 type DefaultUserValues = {
   globalUserId: string
@@ -306,6 +307,33 @@ class TestConfiguration {
     }
   }
 
+  async createGroup(roleId: string = BUILTIN_ROLE_IDS.BASIC) {
+    return context.doInTenant(this.tenantId!, async () => {
+      const baseGroup = structures.userGroups.userGroup()
+      baseGroup.roles = {
+        [this.prodAppId]: roleId,
+      }
+      const { id, rev } = await pro.sdk.groups.save(baseGroup)
+      return {
+        _id: id,
+        _rev: rev,
+        ...baseGroup,
+      }
+    })
+  }
+
+  async addUserToGroup(groupId: string, userId: string) {
+    return context.doInTenant(this.tenantId!, async () => {
+      await pro.sdk.groups.addUsers(groupId, [userId])
+    })
+  }
+
+  async removeUserFromGroup(groupId: string, userId: string) {
+    return context.doInTenant(this.tenantId!, async () => {
+      await pro.sdk.groups.removeUsers(groupId, [userId])
+    })
+  }
+
   async login({ roleId, userId, builder, prodApp = false }: any = {}) {
     const appId = prodApp ? this.prodAppId : this.appId
     return context.doInAppContext(appId, async () => {

packages/server/src/utilities/global.ts

@@ -9,6 +9,7 @@ import {
 import env from "../environment"
 import { groups } from "@budibase/pro"
 import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
+import { global } from "yargs"
 
 export function updateAppRole(
   user: ContextUser,
@@ -16,7 +17,7 @@ export function updateAppRole(
 ) {
   appId = appId || context.getAppId()
 
-  if (!user || !user.roles) {
+  if (!user || (!user.roles && !user.userGroups)) {
     return user
   }
   // if in an multi-tenancy environment make sure roles are never updated
@@ -27,7 +28,7 @@ export function updateAppRole(
     return user
   }
   // always use the deployed app
-  if (appId) {
+  if (appId && user.roles) {
     user.roleId = user.roles[dbCore.getProdAppID(appId)]
   }
   // if a role wasn't found then either set as admin (builder) or public (everyone else)
@@ -60,7 +61,7 @@ async function checkGroupRoles(
   return user
 }
 
-async function processUser(
+export async function processUser(
   user: ContextUser,
   opts: { appId?: string; groups?: UserGroup[] } = {}
 ) {
@@ -94,16 +95,15 @@ export async function getGlobalUser(userId: string) {
   return processUser(user, { appId })
 }
 
-export async function getGlobalUsers(users?: ContextUser[]) {
+export async function getGlobalUsers(
+  userIds?: string[],
+  opts?: { noProcessing?: boolean }
+) {
   const appId = context.getAppId()
   const db = tenancy.getGlobalDB()
-  const allGroups = await groups.fetch()
   let globalUsers
-  if (users) {
-    const globalIds = users.map(user =>
-      getGlobalIDFromUserMetadataID(user._id!)
-    )
-    globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
+  if (userIds) {
+    globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
       row => row.doc
     )
   } else {
@@ -126,15 +126,20 @@ export async function getGlobalUsers(users?: ContextUser[]) {
     return globalUsers
   }
 
-  // pass in the groups, meaning we don't actually need to retrieve them for
-  // each user individually
-  return Promise.all(
-    globalUsers.map(user => processUser(user, { groups: allGroups }))
-  )
+  if (opts?.noProcessing) {
+    return globalUsers
+  } else {
+    // pass in the groups, meaning we don't actually need to retrieve them for
+    // each user individually
+    const allGroups = await groups.fetch()
+    return Promise.all(
+      globalUsers.map(user => processUser(user, { groups: allGroups }))
+    )
+  }
 }
 
 export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
-  const globalUsers = await getGlobalUsers(users)
+  const globalUsers = await getGlobalUsers(users.map(user => user._id!))
   return users.map(user => {
     const globalUser = globalUsers.find(
       globalUser => globalUser && user._id?.includes(globalUser._id)

packages/shared-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/shared-core",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "description": "Shared data utils",
   "main": "dist/cjs/src/index.js",
   "types": "dist/mjs/src/index.d.ts",
@@ -20,7 +20,7 @@
     "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\""
   },
   "dependencies": {
-    "@budibase/types": "2.5.6-alpha.1"
+    "@budibase/types": "2.5.6-alpha.2"
   },
   "devDependencies": {
     "concurrently": "^7.6.0",

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "description": "Budibase types",
   "main": "dist/cjs/index.js",
   "types": "dist/mjs/index.d.ts",

packages/types/src/documents/app/user.ts

@@ -2,4 +2,5 @@ import { Document } from "../document"
 
 export interface UserMetadata extends Document {
   roleId: string
+  email?: string
 }

packages/types/src/sdk/events/event.ts

@@ -1,4 +1,5 @@
 import { Hosting } from "../hosting"
+import { Group, Identity } from "./identification"
 
 export enum Event {
   // USER
@@ -186,6 +187,24 @@ export enum Event {
   AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded",
 }
 
+export const UserGroupSyncEvents: Event[] = [
+  Event.USER_CREATED,
+  Event.USER_UPDATED,
+  Event.USER_DELETED,
+  Event.USER_PERMISSION_ADMIN_ASSIGNED,
+  Event.USER_PERMISSION_ADMIN_REMOVED,
+  Event.USER_PERMISSION_BUILDER_ASSIGNED,
+  Event.USER_PERMISSION_BUILDER_REMOVED,
+  Event.USER_GROUP_CREATED,
+  Event.USER_GROUP_UPDATED,
+  Event.USER_GROUP_DELETED,
+  Event.USER_GROUP_USERS_ADDED,
+  Event.USER_GROUP_USERS_REMOVED,
+  Event.USER_GROUP_PERMISSIONS_EDITED,
+]
+
+export const AsyncEvents: Event[] = [...UserGroupSyncEvents]
+
 // all events that are not audited have been added to this record as undefined, this means
 // that Typescript can protect us against new events being added and auditing of those
 // events not being considered. This might be a little ugly, but provides a level of
@@ -383,3 +402,21 @@ export interface BaseEvent {
 }
 
 export type TableExportFormat = "json" | "csv"
+
+export type DocUpdateEvent = {
+  id: string
+  tenantId: string
+  appId?: string
+}
+
+export interface EventProcessor {
+  processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void>
+  identify?(identity: Identity, timestamp?: string | number): Promise<void>
+  identifyGroup?(group: Group, timestamp?: string | number): Promise<void>
+  shutdown?(): void
+}

packages/types/src/sdk/events/identification.ts

@@ -46,6 +46,8 @@ export interface Identity {
   environment: string
   installationId?: string
   tenantId?: string
+  // usable - no unique format
+  realTenantId?: string
   hostInfo?: HostInfo
 }
 

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.5.6-alpha.1",
+  "version": "2.5.6-alpha.2",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -37,10 +37,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.5.6-alpha.1",
-    "@budibase/pro": "2.5.6-alpha.1",
-    "@budibase/string-templates": "2.5.6-alpha.1",
-    "@budibase/types": "2.5.6-alpha.1",
+    "@budibase/backend-core": "2.5.6-alpha.2",
+    "@budibase/pro": "2.5.6-alpha.2",
+    "@budibase/string-templates": "2.5.6-alpha.2",
+    "@budibase/types": "2.5.6-alpha.2",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",

packages/worker/src/sdk/users/users.ts

@@ -1,5 +1,4 @@
 import env from "../../environment"
-import * as apps from "../../utilities/appService"
 import * as eventHelpers from "./events"
 import {
   accounts,
@@ -30,7 +29,6 @@ import {
   PlatformUser,
   PlatformUserByEmail,
   RowResponse,
-  SearchUsersRequest,
   User,
   SaveUserOpts,
   Account,
@@ -280,9 +278,6 @@ export const save = async (
     await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
     await cache.user.invalidateUser(response.id)
 
-    // let server know to sync user
-    await apps.syncUserInApps(_id, dbUser)
-
     await Promise.all(groupPromises)
 
     // finally returned the saved user from the db
@@ -432,7 +427,6 @@ export const bulkCreate = async (
     // instead of relying on looping tenant creation
     await platform.users.addUser(tenantId, user._id, user.email)
     await eventHelpers.handleSaveEvents(user, undefined)
-    await apps.syncUserInApps(user._id)
   }
 
   const saved = usersToBulkSave.map(user => {
@@ -571,8 +565,6 @@ export const destroy = async (id: string) => {
   await eventHelpers.handleDeleteEvents(dbUser)
   await cache.user.invalidateUser(userId)
   await sessions.invalidateSessions(userId, { reason: "deletion" })
-  // let server know to sync user
-  await apps.syncUserInApps(userId, dbUser)
 }
 
 const bulkDeleteProcessing = async (dbUser: User) => {
@@ -581,8 +573,6 @@ const bulkDeleteProcessing = async (dbUser: User) => {
   await eventHelpers.handleDeleteEvents(dbUser)
   await cache.user.invalidateUser(userId)
   await sessions.invalidateSessions(userId, { reason: "bulk-deletion" })
-  // let server know to sync user
-  await apps.syncUserInApps(userId, dbUser)
 }
 
 export const invite = async (

packages/worker/src/utilities/appService.ts

@@ -1,46 +0,0 @@
-import fetch from "node-fetch"
-import {
-  constants,
-  tenancy,
-  logging,
-  env as coreEnv,
-} from "@budibase/backend-core"
-import { checkSlashesInUrl } from "../utilities"
-import env from "../environment"
-import { SyncUserRequest, User } from "@budibase/types"
-
-async function makeAppRequest(url: string, method: string, body: any) {
-  if (env.isTest()) {
-    return
-  }
-  const request: any = { headers: {} }
-  request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
-  if (tenancy.isTenantIdSet()) {
-    request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
-  }
-  if (body) {
-    request.headers["Content-Type"] = "application/json"
-    request.body = JSON.stringify(body)
-  }
-  request.method = method
-
-  // add x-budibase-correlation-id header
-  logging.correlation.setHeader(request.headers)
-
-  return fetch(checkSlashesInUrl(env.APPS_URL + url), request)
-}
-
-export async function syncUserInApps(userId: string, previousUser?: User) {
-  const body: SyncUserRequest = {
-    previousUser,
-  }
-
-  const response = await makeAppRequest(
-    `/api/users/metadata/sync/${userId}`,
-    "POST",
-    body
-  )
-  if (response && response.status !== 200) {
-    throw "Unable to sync user."
-  }
-}

qa-core/src/internal-api/tests/applications/publish.spec.ts

@@ -36,7 +36,7 @@ describe("Internal API - Application creation, update, publish and delete", () =
 
     const [syncResponse, sync] = await config.api.apps.sync(app.appId!)
     expect(sync).toEqual({
-      message: "App sync not required, app not deployed.",
+      message: "App sync completed successfully.",
     })
   })