diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 14b7547ea9..96ddd61268 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -239,7 +239,7 @@ export function getGlobalDB(): Database { export function getAuditLogsDB(): Database { if (!getTenantId()) { - throw new Error("Audit log DB not found") + throw new Error("No tenant ID found - cannot open audit log DB") } return getDB(getAuditLogDBName()) } diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index d9dbb809d1..8ad7714973 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -366,7 +366,7 @@ export async function getAllApps({ } } -export async function getAppsById(appIds: string[]) { +export async function getAppsByIDs(appIds: string[]) { const settled = await Promise.allSettled( appIds.map(appId => getAppMetadata(appId)) ) diff --git a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 593e5ff082..280669b992 100644 --- a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -47,6 +47,8 @@ export default class PosthogProcessor implements EventProcessor { return } + properties = this.clearPIIProperties(properties) + properties.version = pkg.version properties.service = env.SERVICE properties.environment = identity.environment @@ -79,6 +81,13 @@ export default class PosthogProcessor implements EventProcessor { this.posthog.capture(payload) } + clearPIIProperties(properties: any) { + if (properties.email) { + delete properties.email + } + return properties + } + async identify(identity: Identity, timestamp?: string | number) { const payload: any = { distinctId: identity.id, properties: identity } if (timestamp) { diff --git a/packages/backend-core/src/events/publishers/auditLog.ts b/packages/backend-core/src/events/publishers/auditLog.ts index b72c135799..7cfb76147a 100644 --- a/packages/backend-core/src/events/publishers/auditLog.ts +++ b/packages/backend-core/src/events/publishers/auditLog.ts @@ -1,26 +1,26 @@ import { Event, AuditLogSearchParams, - AuditLogFilterEvent, - AuditLogDownloadEvent, + AuditLogFilteredEvent, + AuditLogDownloadedEvent, } from "@budibase/types" import { publishEvent } from "../events" async function filtered(search: AuditLogSearchParams) { - const properties: AuditLogFilterEvent = { + const properties: AuditLogFilteredEvent = { filters: search, } - await publishEvent(Event.AUDIT_LOG_FILTER, properties) + await publishEvent(Event.AUDIT_LOGS_FILTERED, properties) } -async function download(search: AuditLogSearchParams) { - const properties: AuditLogDownloadEvent = { +async function downloaded(search: AuditLogSearchParams) { + const properties: AuditLogDownloadedEvent = { filters: search, } - await publishEvent(Event.AUDIT_LOG_DOWNLOAD, properties) + await publishEvent(Event.AUDIT_LOGS_DOWNLOADED, properties) } export default { filtered, - download, + downloaded, } diff --git a/packages/backend-core/src/middleware/downloadBody.ts b/packages/backend-core/src/middleware/querystringToBody.ts similarity index 79% rename from packages/backend-core/src/middleware/downloadBody.ts rename to packages/backend-core/src/middleware/querystringToBody.ts index a7050e08c5..b6f109231a 100644 --- a/packages/backend-core/src/middleware/downloadBody.ts +++ b/packages/backend-core/src/middleware/querystringToBody.ts @@ -7,6 +7,12 @@ import { Ctx } from "@budibase/types" */ export default function (ctx: Ctx, next: any) { const queryString = ctx.request.query?.query as string | undefined + if (ctx.request.method.toLowerCase() !== "get") { + ctx.throw( + 500, + "Query to download middleware can only be used for get requests." + ) + } if (!queryString) { return next() } diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 5100066ff9..9ef2c5c31f 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -12,7 +12,7 @@ import * as context from "./context" type GetOpts = { cleanup?: boolean } -function cleanupUsers(users: User | User[]) { +function removeUserPassword(users: User | User[]) { if (Array.isArray(users)) { return users.map(user => { if (user) { @@ -39,7 +39,7 @@ export const bulkGetGlobalUsersById = async ( }) ).rows.map(row => row.doc) as User[] if (opts?.cleanup) { - users = cleanupUsers(users) as User[] + users = removeUserPassword(users) as User[] } return users } @@ -53,7 +53,7 @@ export async function getById(id: string, opts?: GetOpts): Promise { const db = context.getGlobalDB() let user = await db.get(id) if (opts?.cleanup) { - user = cleanupUsers(user) + user = removeUserPassword(user) } return user } @@ -61,7 +61,6 @@ export async function getById(id: string, opts?: GetOpts): Promise { /** * Given an email address this will use a view to search through * all the users to find one with this email address. - * @param {string} email the email to lookup the user by. */ export const getGlobalUserByEmail = async ( email: String, @@ -83,7 +82,7 @@ export const getGlobalUserByEmail = async ( let user = response as User if (opts?.cleanup) { - user = cleanupUsers(user) as User + user = removeUserPassword(user) as User } return user @@ -107,7 +106,7 @@ export const searchGlobalUsersByApp = async ( } let users: User[] = Array.isArray(response) ? response : [response] if (getOpts?.cleanup) { - users = cleanupUsers(users) as User[] + users = removeUserPassword(users) as User[] } return users } @@ -143,7 +142,7 @@ export const searchGlobalUsersByEmail = async ( } let users: User[] = Array.isArray(response) ? response : [response] if (getOpts?.cleanup) { - users = cleanupUsers(users) as User[] + users = removeUserPassword(users) as User[] } return users } diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 1458c94e77..3731e134ad 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -214,36 +214,6 @@ export async function getBuildersCount() { return builders.length } -/** - * Logs a user out from budibase. Re-used across account portal and builder. - */ -export async function platformLogout(opts: PlatformLogoutOpts) { - const ctx = opts.ctx - const email = ctx.user?.email! - const userId = opts.userId - const keepActiveSession = opts.keepActiveSession - - if (!ctx) throw new Error("Koa context must be supplied to logout.") - - const currentSession = getCookie(ctx, Cookie.Auth) - let sessions = await getSessionsForUser(userId) - - if (keepActiveSession) { - sessions = sessions.filter( - session => session.sessionId !== currentSession.sessionId - ) - } else { - // clear cookies - clearCookie(ctx, Cookie.Auth) - clearCookie(ctx, Cookie.CurrentApp) - } - - const sessionIds = sessions.map(({ sessionId }) => sessionId) - await invalidateSessions(userId, { sessionIds, reason: "logout" }) - await events.auth.logout(email) - await userCache.invalidateUser(userId) -} - export function timeout(timeMs: number) { return new Promise(resolve => setTimeout(resolve, timeMs)) } diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 86b2783533..00f2aca7fc 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -28,17 +28,12 @@ import * as automations from "./automations" import { Thread } from "./threads" import * as redis from "./utilities/redis" import { events, logging, middleware } from "@budibase/backend-core" -import { sdk } from "@budibase/pro" import { initialise as initialiseWebsockets } from "./websocket" import { startup } from "./startup" const Sentry = require("@sentry/node") const destroyable = require("server-destroy") const { userAgent } = require("koa-useragent") -// configure events to use the pro audit log write -// can't integrate directly into backend-core due to cyclic issues -events.configure(sdk.auditLogs.write) - const app = new Koa() let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10") diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index f70694226d..1729c7ba20 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -5,7 +5,7 @@ import { generateApiKey, getChecklist, } from "./utilities/workerRequests" -import { installation, tenancy, logging } from "@budibase/backend-core" +import { installation, tenancy, logging, events } from "@budibase/backend-core" import fs from "fs" import { watch } from "./watch" import * as automations from "./automations" @@ -17,6 +17,7 @@ import * as pro from "@budibase/pro" import * as api from "./api" import sdk from "./sdk" const pino = require("koa-pino-logger") +import { sdk as proSdk } from "@budibase/pro" let STARTUP_RAN = false @@ -124,6 +125,9 @@ export async function startup(app?: any, server?: any) { // get the references to the queue promises, don't await as // they will never end, unless the processing stops let queuePromises = [] + // configure events to use the pro audit log write + // can't integrate directly into backend-core due to cyclic issues + queuePromises.push(events.configure(proSdk.auditLogs.write)) queuePromises.push(automations.init()) queuePromises.push(initPro()) if (app) { diff --git a/packages/types/src/sdk/auditLogs.ts b/packages/types/src/sdk/auditLogs.ts index e5d0c77d90..9a5d6e7556 100644 --- a/packages/types/src/sdk/auditLogs.ts +++ b/packages/types/src/sdk/auditLogs.ts @@ -11,7 +11,7 @@ export type AuditLogFn = ( event: Event, metadata: any, opts: AuditWriteOpts -) => Promise +) => Promise export type AuditLogQueueEvent = { event: Event diff --git a/packages/types/src/sdk/events/auditLog.ts b/packages/types/src/sdk/events/auditLog.ts index c5e3208536..5f3edfb826 100644 --- a/packages/types/src/sdk/events/auditLog.ts +++ b/packages/types/src/sdk/events/auditLog.ts @@ -1,10 +1,10 @@ import { BaseEvent } from "./event" import { AuditLogSearchParams } from "../../api" -export interface AuditLogFilterEvent extends BaseEvent { +export interface AuditLogFilteredEvent extends BaseEvent { filters: AuditLogSearchParams } -export interface AuditLogDownloadEvent extends BaseEvent { +export interface AuditLogDownloadedEvent extends BaseEvent { filters: AuditLogSearchParams } diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 119ae4ad7a..e2c85c970c 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -182,8 +182,8 @@ export enum Event { ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable:upgrade_panel_opened", // AUDIT LOG - AUDIT_LOG_FILTER = "audit_log:filter", - AUDIT_LOG_DOWNLOAD = "audit_log:download", + AUDIT_LOGS_FILTERED = "audit_log:filtered", + AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded", } // all events that are not audited have been added to this record as undefined, this means @@ -362,8 +362,8 @@ export const AuditedEventFriendlyName: Record = { [Event.INSTALLATION_FIRST_STARTUP]: undefined, // AUDIT LOG - NOT AUDITED - [Event.AUDIT_LOG_FILTER]: undefined, - [Event.AUDIT_LOG_DOWNLOAD]: undefined, + [Event.AUDIT_LOGS_FILTERED]: undefined, + [Event.AUDIT_LOGS_DOWNLOADED]: undefined, } // properties added at the final stage of the event pipeline diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index 536c510129..19e3cd64b4 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -23,87 +23,89 @@ describe("/api/global/auditlogs", () => { await config.afterAll() }) - it("should be able to fire some events (create audit logs)", async () => { - await context.doInTenant(config.tenantId, async () => { - const userId = config.user!._id! - const identity = { - ...BASE_IDENTITY, - _id: userId, - tenantId: config.tenantId, - } - await context.doInIdentityContext(identity, async () => { - for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) { - await events.user.created(structures.users.user()) + describe("POST /api/global/auditlogs/search", () => { + it("should be able to fire some events (create audit logs)", async () => { + await context.doInTenant(config.tenantId, async () => { + const userId = config.user!._id! + const identity = { + ...BASE_IDENTITY, + _id: userId, + tenantId: config.tenantId, } - await context.doInAppContext(APP_ID, async () => { - await events.app.created(structures.apps.app(APP_ID)) + await context.doInIdentityContext(identity, async () => { + for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) { + await events.user.created(structures.users.user()) + } + await context.doInAppContext(APP_ID, async () => { + await events.app.created(structures.apps.app(APP_ID)) + }) + // fetch the user created events + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data).toBeDefined() + // there will be an initial event which comes from the default user creation + expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1) }) - // fetch the user created events - const response = await config.api.auditLogs.search({ - events: [Event.USER_CREATED], - }) - expect(response.data).toBeDefined() - // there will be an initial event which comes from the default user creation - expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1) }) }) - }) - it("should be able to search by event", async () => { - const response = await config.api.auditLogs.search({ - events: [Event.USER_CREATED], + it("should be able to search by event", async () => { + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.event).toBe(Event.USER_CREATED) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.event).toBe(Event.USER_CREATED) - } - }) - it("should be able to search by time range (frozen)", async () => { - // this is frozen, only need to add 1 and minus 1 - const now = new Date() - const start = new Date() - start.setSeconds(now.getSeconds() - 1) - const end = new Date() - end.setSeconds(now.getSeconds() + 1) - const response = await config.api.auditLogs.search({ - startDate: start.toISOString(), - endDate: end.toISOString(), + it("should be able to search by time range (frozen)", async () => { + // this is frozen, only need to add 1 and minus 1 + const now = new Date() + const start = new Date() + start.setSeconds(now.getSeconds() - 1) + const end = new Date() + end.setSeconds(now.getSeconds() + 1) + const response = await config.api.auditLogs.search({ + startDate: start.toISOString(), + endDate: end.toISOString(), + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.timestamp).toBe(now.toISOString()) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.timestamp).toBe(now.toISOString()) - } - }) - it("should be able to search by user ID", async () => { - const userId = config.user!._id! - const response = await config.api.auditLogs.search({ - userIds: [userId], + it("should be able to search by user ID", async () => { + const userId = config.user!._id! + const response = await config.api.auditLogs.search({ + userIds: [userId], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.user._id).toBe(userId) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.user._id).toBe(userId) - } - }) - it("should be able to search by app ID", async () => { - const response = await config.api.auditLogs.search({ - appIds: [APP_ID], + it("should be able to search by app ID", async () => { + const response = await config.api.auditLogs.search({ + appIds: [APP_ID], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.app?._id).toBe(APP_ID) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.app?._id).toBe(APP_ID) - } - }) - it("should be able to search by full string", async () => { - const response = await config.api.auditLogs.search({ - fullSearch: "User", + it("should be able to search by full string", async () => { + const response = await config.api.auditLogs.search({ + fullSearch: "User", + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.name.includes("User")).toBe(true) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.name.includes("User")).toBe(true) - } }) })