Main body of PR comments.

packages/backend-core/src/context/mainContext.ts

@@ -239,7 +239,7 @@ export function getGlobalDB(): Database {
 
 export function getAuditLogsDB(): Database {
   if (!getTenantId()) {
-    throw new Error("Audit log DB not found")
+    throw new Error("No tenant ID found - cannot open audit log DB")
   }
   return getDB(getAuditLogDBName())
 }

packages/backend-core/src/db/utils.ts

@@ -366,7 +366,7 @@ export async function getAllApps({
   }
 }
 
-export async function getAppsById(appIds: string[]) {
+export async function getAppsByIDs(appIds: string[]) {
   const settled = await Promise.allSettled(
     appIds.map(appId => getAppMetadata(appId))
   )

packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts

@@ -47,6 +47,8 @@ export default class PosthogProcessor implements EventProcessor {
       return
     }
 
+    properties = this.clearPIIProperties(properties)
+
     properties.version = pkg.version
     properties.service = env.SERVICE
     properties.environment = identity.environment
@@ -79,6 +81,13 @@ export default class PosthogProcessor implements EventProcessor {
     this.posthog.capture(payload)
   }
 
+  clearPIIProperties(properties: any) {
+    if (properties.email) {
+      delete properties.email
+    }
+    return properties
+  }
+
   async identify(identity: Identity, timestamp?: string | number) {
     const payload: any = { distinctId: identity.id, properties: identity }
     if (timestamp) {

packages/backend-core/src/events/publishers/auditLog.ts

@@ -1,26 +1,26 @@
 import {
   Event,
   AuditLogSearchParams,
-  AuditLogFilterEvent,
+  AuditLogFilteredEvent,
-  AuditLogDownloadEvent,
+  AuditLogDownloadedEvent,
 } from "@budibase/types"
 import { publishEvent } from "../events"
 
 async function filtered(search: AuditLogSearchParams) {
-  const properties: AuditLogFilterEvent = {
+  const properties: AuditLogFilteredEvent = {
     filters: search,
   }
-  await publishEvent(Event.AUDIT_LOG_FILTER, properties)
+  await publishEvent(Event.AUDIT_LOGS_FILTERED, properties)
 }
 
-async function download(search: AuditLogSearchParams) {
+async function downloaded(search: AuditLogSearchParams) {
-  const properties: AuditLogDownloadEvent = {
+  const properties: AuditLogDownloadedEvent = {
     filters: search,
   }
-  await publishEvent(Event.AUDIT_LOG_DOWNLOAD, properties)
+  await publishEvent(Event.AUDIT_LOGS_DOWNLOADED, properties)
 }
 
 export default {
   filtered,
-  download,
+  downloaded,
 }

packages/backend-core/src/middleware/querystringToBody.ts

@@ -7,6 +7,12 @@ import { Ctx } from "@budibase/types"
  */
 export default function (ctx: Ctx, next: any) {
   const queryString = ctx.request.query?.query as string | undefined
+  if (ctx.request.method.toLowerCase() !== "get") {
+    ctx.throw(
+      500,
+      "Query to download middleware can only be used for get requests."
+    )
+  }
   if (!queryString) {
     return next()
   }

packages/backend-core/src/users.ts

@@ -12,7 +12,7 @@ import * as context from "./context"
 
 type GetOpts = { cleanup?: boolean }
 
-function cleanupUsers(users: User | User[]) {
+function removeUserPassword(users: User | User[]) {
   if (Array.isArray(users)) {
     return users.map(user => {
       if (user) {
@@ -39,7 +39,7 @@ export const bulkGetGlobalUsersById = async (
     })
   ).rows.map(row => row.doc) as User[]
   if (opts?.cleanup) {
-    users = cleanupUsers(users) as User[]
+    users = removeUserPassword(users) as User[]
   }
   return users
 }
@@ -53,7 +53,7 @@ export async function getById(id: string, opts?: GetOpts): Promise<User> {
   const db = context.getGlobalDB()
   let user = await db.get(id)
   if (opts?.cleanup) {
-    user = cleanupUsers(user)
+    user = removeUserPassword(user)
   }
   return user
 }
@@ -61,7 +61,6 @@ export async function getById(id: string, opts?: GetOpts): Promise<User> {
 /**
  * Given an email address this will use a view to search through
  * all the users to find one with this email address.
- * @param {string} email the email to lookup the user by.
  */
 export const getGlobalUserByEmail = async (
   email: String,
@@ -83,7 +82,7 @@ export const getGlobalUserByEmail = async (
 
   let user = response as User
   if (opts?.cleanup) {
-    user = cleanupUsers(user) as User
+    user = removeUserPassword(user) as User
   }
 
   return user
@@ -107,7 +106,7 @@ export const searchGlobalUsersByApp = async (
   }
   let users: User[] = Array.isArray(response) ? response : [response]
   if (getOpts?.cleanup) {
-    users = cleanupUsers(users) as User[]
+    users = removeUserPassword(users) as User[]
   }
   return users
 }
@@ -143,7 +142,7 @@ export const searchGlobalUsersByEmail = async (
   }
   let users: User[] = Array.isArray(response) ? response : [response]
   if (getOpts?.cleanup) {
-    users = cleanupUsers(users) as User[]
+    users = removeUserPassword(users) as User[]
   }
   return users
 }

packages/backend-core/src/utils/utils.ts

@@ -214,36 +214,6 @@ export async function getBuildersCount() {
   return builders.length
 }
 
-/**
- * Logs a user out from budibase. Re-used across account portal and builder.
- */
-export async function platformLogout(opts: PlatformLogoutOpts) {
-  const ctx = opts.ctx
-  const email = ctx.user?.email!
-  const userId = opts.userId
-  const keepActiveSession = opts.keepActiveSession
-
-  if (!ctx) throw new Error("Koa context must be supplied to logout.")
-
-  const currentSession = getCookie(ctx, Cookie.Auth)
-  let sessions = await getSessionsForUser(userId)
-
-  if (keepActiveSession) {
-    sessions = sessions.filter(
-      session => session.sessionId !== currentSession.sessionId
-    )
-  } else {
-    // clear cookies
-    clearCookie(ctx, Cookie.Auth)
-    clearCookie(ctx, Cookie.CurrentApp)
-  }
-
-  const sessionIds = sessions.map(({ sessionId }) => sessionId)
-  await invalidateSessions(userId, { sessionIds, reason: "logout" })
-  await events.auth.logout(email)
-  await userCache.invalidateUser(userId)
-}
-
 export function timeout(timeMs: number) {
   return new Promise(resolve => setTimeout(resolve, timeMs))
 }

packages/server/src/app.ts

@@ -28,17 +28,12 @@ import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
 import { events, logging, middleware } from "@budibase/backend-core"
-import { sdk } from "@budibase/pro"
 import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")
 const destroyable = require("server-destroy")
 const { userAgent } = require("koa-useragent")
 
-// configure events to use the pro audit log write
-// can't integrate directly into backend-core due to cyclic issues
-events.configure(sdk.auditLogs.write)
-
 const app = new Koa()
 
 let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10")

packages/server/src/startup.ts

@@ -5,7 +5,7 @@ import {
   generateApiKey,
   getChecklist,
 } from "./utilities/workerRequests"
-import { installation, tenancy, logging } from "@budibase/backend-core"
+import { installation, tenancy, logging, events } from "@budibase/backend-core"
 import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
@@ -17,6 +17,7 @@ import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
 const pino = require("koa-pino-logger")
+import { sdk as proSdk } from "@budibase/pro"
 
 let STARTUP_RAN = false
 
@@ -124,6 +125,9 @@ export async function startup(app?: any, server?: any) {
   // get the references to the queue promises, don't await as
   // they will never end, unless the processing stops
   let queuePromises = []
+  // configure events to use the pro audit log write
+  // can't integrate directly into backend-core due to cyclic issues
+  queuePromises.push(events.configure(proSdk.auditLogs.write))
   queuePromises.push(automations.init())
   queuePromises.push(initPro())
   if (app) {

packages/types/src/sdk/auditLogs.ts

@@ -11,7 +11,7 @@ export type AuditLogFn = (
   event: Event,
   metadata: any,
   opts: AuditWriteOpts
-) => Promise<any>
+) => Promise<void>
 
 export type AuditLogQueueEvent = {
   event: Event

packages/types/src/sdk/events/auditLog.ts

@@ -1,10 +1,10 @@
 import { BaseEvent } from "./event"
 import { AuditLogSearchParams } from "../../api"
 
-export interface AuditLogFilterEvent extends BaseEvent {
+export interface AuditLogFilteredEvent extends BaseEvent {
   filters: AuditLogSearchParams
 }
 
-export interface AuditLogDownloadEvent extends BaseEvent {
+export interface AuditLogDownloadedEvent extends BaseEvent {
   filters: AuditLogSearchParams
 }

packages/types/src/sdk/events/event.ts

@@ -182,8 +182,8 @@ export enum Event {
   ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable:upgrade_panel_opened",
 
   // AUDIT LOG
-  AUDIT_LOG_FILTER = "audit_log:filter",
+  AUDIT_LOGS_FILTERED = "audit_log:filtered",
-  AUDIT_LOG_DOWNLOAD = "audit_log:download",
+  AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded",
 }
 
 // all events that are not audited have been added to this record as undefined, this means
@@ -362,8 +362,8 @@ export const AuditedEventFriendlyName: Record<Event, string | undefined> = {
   [Event.INSTALLATION_FIRST_STARTUP]: undefined,
 
   // AUDIT LOG - NOT AUDITED
-  [Event.AUDIT_LOG_FILTER]: undefined,
+  [Event.AUDIT_LOGS_FILTERED]: undefined,
-  [Event.AUDIT_LOG_DOWNLOAD]: undefined,
+  [Event.AUDIT_LOGS_DOWNLOADED]: undefined,
 }
 
 // properties added at the final stage of the event pipeline

packages/worker/src/api/routes/global/tests/auditLogs.spec.ts

@@ -23,87 +23,89 @@ describe("/api/global/auditlogs", () => {
     await config.afterAll()
   })
 
-  it("should be able to fire some events (create audit logs)", async () => {
+  describe("POST /api/global/auditlogs/search", () => {
-    await context.doInTenant(config.tenantId, async () => {
+    it("should be able to fire some events (create audit logs)", async () => {
-      const userId = config.user!._id!
+      await context.doInTenant(config.tenantId, async () => {
-      const identity = {
+        const userId = config.user!._id!
-        ...BASE_IDENTITY,
+        const identity = {
-        _id: userId,
+          ...BASE_IDENTITY,
-        tenantId: config.tenantId,
+          _id: userId,
-      }
+          tenantId: config.tenantId,
-      await context.doInIdentityContext(identity, async () => {
-        for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) {
-          await events.user.created(structures.users.user())
         }
-        await context.doInAppContext(APP_ID, async () => {
+        await context.doInIdentityContext(identity, async () => {
-          await events.app.created(structures.apps.app(APP_ID))
+          for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) {
+            await events.user.created(structures.users.user())
+          }
+          await context.doInAppContext(APP_ID, async () => {
+            await events.app.created(structures.apps.app(APP_ID))
+          })
+          // fetch the user created events
+          const response = await config.api.auditLogs.search({
+            events: [Event.USER_CREATED],
+          })
+          expect(response.data).toBeDefined()
+          // there will be an initial event which comes from the default user creation
+          expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1)
         })
-        // fetch the user created events
-        const response = await config.api.auditLogs.search({
-          events: [Event.USER_CREATED],
-        })
-        expect(response.data).toBeDefined()
-        // there will be an initial event which comes from the default user creation
-        expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1)
       })
     })
-  })
 
-  it("should be able to search by event", async () => {
+    it("should be able to search by event", async () => {
-    const response = await config.api.auditLogs.search({
+      const response = await config.api.auditLogs.search({
-      events: [Event.USER_CREATED],
+        events: [Event.USER_CREATED],
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.event).toBe(Event.USER_CREATED)
+      }
     })
-    expect(response.data.length).toBeGreaterThan(0)
-    for (let log of response.data) {
-      expect(log.event).toBe(Event.USER_CREATED)
-    }
-  })
 
-  it("should be able to search by time range (frozen)", async () => {
+    it("should be able to search by time range (frozen)", async () => {
-    // this is frozen, only need to add 1 and minus 1
+      // this is frozen, only need to add 1 and minus 1
-    const now = new Date()
+      const now = new Date()
-    const start = new Date()
+      const start = new Date()
-    start.setSeconds(now.getSeconds() - 1)
+      start.setSeconds(now.getSeconds() - 1)
-    const end = new Date()
+      const end = new Date()
-    end.setSeconds(now.getSeconds() + 1)
+      end.setSeconds(now.getSeconds() + 1)
-    const response = await config.api.auditLogs.search({
+      const response = await config.api.auditLogs.search({
-      startDate: start.toISOString(),
+        startDate: start.toISOString(),
-      endDate: end.toISOString(),
+        endDate: end.toISOString(),
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.timestamp).toBe(now.toISOString())
+      }
     })
-    expect(response.data.length).toBeGreaterThan(0)
-    for (let log of response.data) {
-      expect(log.timestamp).toBe(now.toISOString())
-    }
-  })
 
-  it("should be able to search by user ID", async () => {
+    it("should be able to search by user ID", async () => {
-    const userId = config.user!._id!
+      const userId = config.user!._id!
-    const response = await config.api.auditLogs.search({
+      const response = await config.api.auditLogs.search({
-      userIds: [userId],
+        userIds: [userId],
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.user._id).toBe(userId)
+      }
     })
-    expect(response.data.length).toBeGreaterThan(0)
-    for (let log of response.data) {
-      expect(log.user._id).toBe(userId)
-    }
-  })
 
-  it("should be able to search by app ID", async () => {
+    it("should be able to search by app ID", async () => {
-    const response = await config.api.auditLogs.search({
+      const response = await config.api.auditLogs.search({
-      appIds: [APP_ID],
+        appIds: [APP_ID],
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.app?._id).toBe(APP_ID)
+      }
     })
-    expect(response.data.length).toBeGreaterThan(0)
-    for (let log of response.data) {
-      expect(log.app?._id).toBe(APP_ID)
-    }
-  })
 
-  it("should be able to search by full string", async () => {
+    it("should be able to search by full string", async () => {
-    const response = await config.api.auditLogs.search({
+      const response = await config.api.auditLogs.search({
-      fullSearch: "User",
+        fullSearch: "User",
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.name.includes("User")).toBe(true)
+      }
     })
-    expect(response.data.length).toBeGreaterThan(0)
-    for (let log of response.data) {
-      expect(log.name.includes("User")).toBe(true)
-    }
   })
 })