diff --git a/packages/backend-core/context.js b/packages/backend-core/context.js index 0d86c530a7..aaa0f56f92 100644 --- a/packages/backend-core/context.js +++ b/packages/backend-core/context.js @@ -5,10 +5,11 @@ const { getAppId, updateAppId, doInAppContext, - doInUserContext, doInTenant, } = require("./src/context") +const identity = require("./src/context/identity") + module.exports = { getAppDB, getDevAppDB, @@ -16,6 +17,6 @@ module.exports = { getAppId, updateAppId, doInAppContext, - doInUserContext, doInTenant, + identity, } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 3f8b0bef81..a717fd1211 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -32,6 +32,7 @@ "pouchdb-find": "^7.2.2", "pouchdb-replication-stream": "^1.2.9", "sanitize-s3-objectkey": "^0.0.1", + "semver": "^7.0.0", "tar-fs": "^2.1.1", "uuid": "^8.3.2", "zlib": "^1.0.5" @@ -52,6 +53,7 @@ "@types/node-fetch": "^2.6.1", "@types/tar-fs": "^2.0.1", "@types/uuid": "^8.3.4", + "@types/semver": "^7.0.0", "ioredis-mock": "^5.5.5", "jest": "^27.0.3", "koa": "2.7.0", diff --git a/packages/backend-core/src/context/identity.ts b/packages/backend-core/src/context/identity.ts new file mode 100644 index 0000000000..37e1ecf40a --- /dev/null +++ b/packages/backend-core/src/context/identity.ts @@ -0,0 +1,50 @@ +import { + IdentityContext, + IdentityType, + User, + UserContext, + isCloudAccount, + Account, + AccountUserContext, +} from "@budibase/types" +import * as context from "." + +export const getIdentity = (): IdentityContext | undefined => { + return context.getIdentity() +} + +export const doInIdentityContext = (identity: IdentityContext, task: any) => { + return context.doInIdentityContext(identity, task) +} + +export const doInUserContext = (user: User, task: any) => { + const userContext: UserContext = { + ...user, + _id: user._id as string, + type: IdentityType.USER, + } + return doInIdentityContext(userContext, task) +} + +export const doInAccountContext = (account: Account, task: any) => { + const _id = getAccountUserId(account) + const tenantId = account.tenantId + const accountContext: AccountUserContext = { + _id, + type: IdentityType.USER, + tenantId, + account, + } + return doInIdentityContext(accountContext, task) +} + +export const getAccountUserId = (account: Account) => { + let userId: string + if (isCloudAccount(account)) { + userId = account.budibaseUserId + } else { + // use account id as user id for self hosting + userId = account.accountId + } + return userId +} diff --git a/packages/backend-core/src/context/index.js b/packages/backend-core/src/context/index.js index 7c7c5c45e6..59dc0cda79 100644 --- a/packages/backend-core/src/context/index.js +++ b/packages/backend-core/src/context/index.js @@ -15,7 +15,7 @@ const ContextKeys = { TENANT_ID: "tenantId", GLOBAL_DB: "globalDb", APP_ID: "appId", - USER: "user", + IDENTITY: "identity", // whatever the request app DB was CURRENT_DB: "currentDb", // get the prod app DB from the request @@ -138,19 +138,19 @@ exports.doInAppContext = (appId, task) => { throw new Error("appId is required") } - const user = exports.getUser() + const identity = exports.getIdentity() // the internal function is so that we can re-use an existing // context - don't want to close DB on a parent context - async function internal(opts = { existing: false, user: undefined }) { + async function internal(opts = { existing: false }) { // set the app tenant id if (!opts.existing) { setAppTenantId(appId) } // set the app ID cls.setOnContext(ContextKeys.APP_ID, appId) - // preserve the user - exports.setUser(user) + // preserve the identity + exports.setIdentity(identity) try { // invoke the task return await task() @@ -175,17 +175,17 @@ exports.doInAppContext = (appId, task) => { } } -exports.doInUserContext = (user, task) => { - if (!user) { - throw new Error("user is required") +exports.doInIdentityContext = (identity, task) => { + if (!identity) { + throw new Error("identity is required") } async function internal(opts = { existing: false }) { if (!opts.existing) { - cls.setOnContext(ContextKeys.USER, user) - // set the tenant so that doInTenant will preserve user - if (user.tenantId) { - exports.updateTenantId(user.tenantId) + cls.setOnContext(ContextKeys.IDENTITY, identity) + // set the tenant so that doInTenant will preserve identity + if (identity.tenantId) { + exports.updateTenantId(identity.tenantId) } } @@ -195,16 +195,16 @@ exports.doInUserContext = (user, task) => { } finally { const using = cls.getFromContext(ContextKeys.IN_USE) if (!using || using <= 1) { - exports.setUser(null) + exports.setIdentity(null) } else { cls.setOnContext(using - 1) } } } - const existing = cls.getFromContext(ContextKeys.USER) + const existing = cls.getFromContext(ContextKeys.IDENTITY) const using = cls.getFromContext(ContextKeys.IN_USE) - if (using && existing && existing._id === user._id) { + if (using && existing && existing._id === identity._id) { cls.setOnContext(ContextKeys.IN_USE, using + 1) return internal({ existing: true }) } else { @@ -215,15 +215,15 @@ exports.doInUserContext = (user, task) => { } } -exports.setUser = user => { - cls.setOnContext(ContextKeys.USER, user) +exports.setIdentity = identity => { + cls.setOnContext(ContextKeys.IDENTITY, identity) } -exports.getUser = () => { +exports.getIdentity = () => { try { - return cls.getFromContext(ContextKeys.USER) + return cls.getFromContext(ContextKeys.IDENTITY) } catch (e) { - // do nothing - user is not in context + // do nothing - identity is not in context } } diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 417fcd7866..9cb1d24423 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -52,6 +52,7 @@ const env: any = { process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads", USE_COUCH: process.env.USE_COUCH || true, DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, + SERVICE: process.env.SERVICE || "budibase", _set(key: any, value: any) { process.env[key] = value module.exports[key] = value diff --git a/packages/backend-core/src/events/analytics.ts b/packages/backend-core/src/events/analytics.ts index e2f219f554..58c49714f7 100644 --- a/packages/backend-core/src/events/analytics.ts +++ b/packages/backend-core/src/events/analytics.ts @@ -3,6 +3,7 @@ import * as tenancy from "../tenancy" import * as dbUtils from "../db/utils" import { Configs } from "../constants" +// TODO: cache in redis export const enabled = async () => { // cloud - always use the environment variable if (!env.SELF_HOSTED) { diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 8eabfa2618..6157b28916 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -1,56 +1,239 @@ import * as context from "../context" +import * as identityCtx from "../context/identity" import env from "../environment" import { Hosting, User, - SessionUser, Identity, IdentityType, Account, - BudibaseIdentity, isCloudAccount, isSSOAccount, - TenantIdentity, + TenantGroup, SettingsConfig, CloudAccount, UserIdentity, - InstallationIdentity, - Installation, - isInstallation, + InstallationGroup, + isSelfHostAccount, + UserContext, + Group, } from "@budibase/types" import { processors } from "./processors" import * as dbUtils from "../db/utils" import { Configs } from "../constants" import * as hashing from "../hashing" +import * as installation from "../installation" const pkg = require("../../package.json") +/** + * An identity can be: + * - account user (Self host) + * - budibase user + * - tenant + * - installation + */ export const getCurrentIdentity = async (): Promise => { - const user: SessionUser | undefined = context.getUser() + let identityContext = identityCtx.getIdentity() - const tenantId = await getGlobalTenantId(context.getTenantId()) - let id: string - let type: IdentityType + let identityType - if (user) { - id = user._id - type = IdentityType.USER + if (!identityContext) { + identityType = IdentityType.TENANT } else { - id = tenantId - type = IdentityType.TENANT + identityType = identityContext.type } - if (user && isInstallation(user)) { - type = IdentityType.INSTALLATION - } + if (identityType === IdentityType.INSTALLATION) { + const installationId = await getInstallationId() + return { + id: formatDistinctId(installationId, identityType), + type: identityType, + installationId, + } + } else if (identityType === IdentityType.TENANT) { + const installationId = await getInstallationId() + const tenantId = await getCurrentTenantId() - return { - id, - tenantId, - type, + return { + id: formatDistinctId(tenantId, identityType), + type: identityType, + installationId, + tenantId, + } + } else if (identityType === IdentityType.USER) { + const userContext = identityContext as UserContext + const tenantId = await getCurrentTenantId() + let installationId: string | undefined + + // self host account users won't have installation + if (!userContext.account || !isSelfHostAccount(userContext.account)) { + installationId = await getInstallationId() + } + + return { + id: userContext._id, + type: identityType, + installationId, + tenantId, + } + } else { + throw new Error("Unknown identity type") } } +export const identifyInstallationGroup = async ( + installId: string, + timestamp?: string | number +): Promise => { + const id = installId + const type = IdentityType.INSTALLATION + const hosting = getHostingFromEnv() + const version = pkg.version + + const group: InstallationGroup = { + id, + type, + hosting, + version, + } + + await identifyGroup(group, timestamp) + // need to create a normal identity for the group to be able to query it globally + // match the posthog syntax to link this identity to the empty auto generated one + await identify({ ...group, id: `$${type}_${id}` }, timestamp) +} + +export const identifyTenantGroup = async ( + tenantId: string, + account: Account | undefined, + timestamp?: string | number +): Promise => { + const id = await getGlobalTenantId(tenantId) + const type = IdentityType.TENANT + + let hosting: Hosting + let profession: string | undefined + let companySize: string | undefined + + if (account) { + profession = account.profession + companySize = account.size + hosting = account.hosting + } else { + hosting = getHostingFromEnv() + } + + const group: TenantGroup = { + id, + type, + hosting, + profession, + companySize, + } + + await identifyGroup(group, timestamp) + // need to create a normal identity for the group to be able to query it globally + // match the posthog syntax to link this identity to the auto generated one + await identify({ ...group, id: `$${type}_${id}` }, timestamp) +} + +export const identifyUser = async ( + user: User, + account: CloudAccount | undefined, + timestamp?: string | number +) => { + const id = user._id as string + const tenantId = await getGlobalTenantId(user.tenantId) + const type = IdentityType.USER + let builder = user.builder?.global || false + let admin = user.admin?.global || false + let providerType = user.providerType + const accountHolder = account?.budibaseUserId === user._id || false + const verified = + account && account?.budibaseUserId === user._id ? account.verified : false + const installationId = await getInstallationId() + + const identity: UserIdentity = { + id, + type, + installationId, + tenantId, + verified, + accountHolder, + providerType, + builder, + admin, + } + + await identify(identity, timestamp) +} + +export const identifyAccount = async (account: Account) => { + let id = account.accountId + const tenantId = account.tenantId + let type = IdentityType.USER + let providerType = isSSOAccount(account) ? account.providerType : undefined + const verified = account.verified + const accountHolder = true + + if (isCloudAccount(account)) { + if (account.budibaseUserId) { + // use the budibase user as the id if set + id = account.budibaseUserId + } + } + + const identity: UserIdentity = { + id, + type, + tenantId, + providerType, + verified, + accountHolder, + } + + await identify(identity) +} + +export const identify = async ( + identity: Identity, + timestamp?: string | number +) => { + await processors.identify(identity, timestamp) +} + +export const identifyGroup = async ( + group: Group, + timestamp?: string | number +) => { + await processors.identifyGroup(group, timestamp) +} + +const getHostingFromEnv = () => { + return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD +} + +export const getCurrentTenantId = () => getGlobalTenantId(context.getTenantId()) + +export const getInstallationId = async () => { + if (isAccountPortal()) { + return "account-portal" + } + const install = await installation.getInstall() + return install.installId +} + +const getGlobalTenantId = async (tenantId: string): Promise => { + if (env.SELF_HOSTED) { + return getGlobalId(tenantId) + } else { + // tenant id's in the cloud are already unique + return tenantId + } +} + +// TODO: cache in redis const getGlobalId = async (tenantId: string): Promise => { const db = context.getGlobalDB() const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, { @@ -68,134 +251,14 @@ const getGlobalId = async (tenantId: string): Promise => { } } -const getGlobalTenantId = async (tenantId: string): Promise => { - if (env.SELF_HOSTED) { - return getGlobalId(tenantId) +const isAccountPortal = () => { + return env.SERVICE === "account-portal" +} + +const formatDistinctId = (id: string, type: IdentityType) => { + if (type === IdentityType.INSTALLATION || type === IdentityType.TENANT) { + return `$${type}_${id}` } else { - // tenant id's in the cloud are already unique - return tenantId + return id } } - -const getHostingFromEnv = () => { - return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD -} - -export const identifyInstallation = async ( - install: Installation, - timestamp: string | number -) => { - const id = install.installId - // the default tenant id, so we can match installations to other events - const tenantId = await getGlobalTenantId(context.getTenantId()) - const version: string = pkg.version as string - const type = IdentityType.INSTALLATION - const hosting = getHostingFromEnv() - - const identity: InstallationIdentity = { - id, - tenantId, - type, - version, - hosting, - } - await identify(identity, timestamp) -} - -export const identifyTenant = async ( - tenantId: string, - account: CloudAccount | undefined, - timestamp?: string | number -) => { - const globalTenantId = await getGlobalTenantId(tenantId) - const id = globalTenantId - const hosting = getHostingFromEnv() - const type = IdentityType.TENANT - const profession = account?.profession - const companySize = account?.size - - const identity: TenantIdentity = { - id, - tenantId: globalTenantId, - hosting, - type, - profession, - companySize, - } - await identify(identity, timestamp) -} - -export const identifyUser = async ( - user: User, - account: CloudAccount | undefined, - timestamp?: string | number -) => { - const id = user._id as string - const tenantId = user.tenantId - const hosting = env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD - const type = IdentityType.USER - let builder = user.builder?.global - let admin = user.admin?.global - let providerType = user.providerType - const accountHolder = account?.budibaseUserId === user._id - const verified = - account && account?.budibaseUserId === user._id ? account.verified : false - const profession = account?.profession - const companySize = account?.size - - const identity: BudibaseIdentity = { - id, - tenantId, - hosting, - type, - builder, - admin, - providerType, - accountHolder, - verified, - profession, - companySize, - } - - await identify(identity, timestamp) -} - -export const identifyAccount = async (account: Account) => { - let id = account.accountId - const tenantId = account.tenantId - const hosting = account.hosting - let type = IdentityType.USER - let providerType = isSSOAccount(account) ? account.providerType : undefined - const verified = account.verified - const profession = account.profession - const companySize = account.size - const accountHolder = true - - if (isCloudAccount(account)) { - if (account.budibaseUserId) { - // use the budibase user as the id if set - id = account.budibaseUserId - } - } - - const identity: UserIdentity = { - id, - tenantId, - hosting, - type, - providerType, - verified, - profession, - companySize, - accountHolder, - } - - await identify(identity) -} - -export const identify = async ( - identity: Identity, - timestamp?: string | number -) => { - await processors.identify(identity, timestamp) -} diff --git a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts index eaa5ae1135..30928abc8d 100644 --- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts +++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts @@ -1,9 +1,15 @@ -import { Event, Identity } from "@budibase/types" +import { Event, Identity, Group, IdentityType } from "@budibase/types" import { EventProcessor } from "./types" import env from "../../environment" import * as analytics from "../analytics" import PosthogProcessor from "./PosthogProcessor" +/** + * Events that are always captured. + */ +const EVENT_WHITELIST = [Event.VERSION_UPGRADED, Event.VERSION_DOWNGRADED] +const IDENTITY_WHITELIST = [IdentityType.INSTALLATION, IdentityType.TENANT] + export default class AnalyticsProcessor implements EventProcessor { posthog: PosthogProcessor | undefined @@ -19,7 +25,7 @@ export default class AnalyticsProcessor implements EventProcessor { properties: any, timestamp?: string | number ): Promise { - if (!(await analytics.enabled())) { + if (!EVENT_WHITELIST.includes(event) && !(await analytics.enabled())) { return } if (this.posthog) { @@ -28,7 +34,11 @@ export default class AnalyticsProcessor implements EventProcessor { } async identify(identity: Identity, timestamp?: string | number) { - if (!(await analytics.enabled())) { + // Group indentifications (tenant and installation) always on + if ( + !IDENTITY_WHITELIST.includes(identity.type) && + !(await analytics.enabled()) + ) { return } if (this.posthog) { @@ -36,6 +46,13 @@ export default class AnalyticsProcessor implements EventProcessor { } } + async identifyGroup(group: Group, timestamp?: string | number) { + // Group indentifications (tenant and installation) always on + if (this.posthog) { + this.posthog.identifyGroup(group, timestamp) + } + } + shutdown() { if (this.posthog) { this.posthog.shutdown() diff --git a/packages/backend-core/src/events/processors/LoggingProcessor.ts b/packages/backend-core/src/events/processors/LoggingProcessor.ts index 47b91b7dea..a517fba09c 100644 --- a/packages/backend-core/src/events/processors/LoggingProcessor.ts +++ b/packages/backend-core/src/events/processors/LoggingProcessor.ts @@ -1,7 +1,17 @@ -import { Event, Identity } from "@budibase/types" +import { Event, Identity, Group } from "@budibase/types" import { EventProcessor } from "./types" import env from "../../environment" +const getTimestampString = (timestamp?: string | number) => { + let timestampString = "" + if (timestamp) { + timestampString = `[timestamp=${new Date(timestamp).toISOString()}]` + } + return timestampString +} + +const skipLogging = env.SELF_HOSTED && !env.isDev() + export default class LoggingProcessor implements EventProcessor { async processEvent( event: Event, @@ -9,34 +19,35 @@ export default class LoggingProcessor implements EventProcessor { properties: any, timestamp?: string ): Promise { - if (env.SELF_HOSTED && !env.isDev()) { + if (skipLogging) { return } - let timestampString = "" - if (timestamp) { - timestampString = `[timestamp=${new Date(timestamp).toISOString()}]` - } - + let timestampString = getTimestampString(timestamp) console.log( `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} ` ) } async identify(identity: Identity, timestamp?: string | number) { - if (env.SELF_HOSTED && !env.isDev()) { + if (skipLogging) { return } - - let timestampString = "" - if (timestamp) { - timestampString = `[timestamp=${new Date(timestamp).toISOString()}]` - } - + let timestampString = getTimestampString(timestamp) console.log( `[audit] [${JSON.stringify(identity)}] ${timestampString} identified` ) } + async identifyGroup(group: Group, timestamp?: string | number) { + if (skipLogging) { + return + } + let timestampString = getTimestampString(timestamp) + console.log( + `[audit] [${JSON.stringify(group)}] ${timestampString} group identified` + ) + } + shutdown(): void { // no-op } diff --git a/packages/backend-core/src/events/processors/PosthogProcessor.ts b/packages/backend-core/src/events/processors/PosthogProcessor.ts index 54b5133dfb..4c1b4b1195 100644 --- a/packages/backend-core/src/events/processors/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/PosthogProcessor.ts @@ -1,6 +1,8 @@ import PostHog from "posthog-node" -import { Event, Identity } from "@budibase/types" +import { Event, Identity, Group } from "@budibase/types" import { EventProcessor } from "./types" +import env from "../../environment" +const pkg = require("../../../package.json") export default class PosthogProcessor implements EventProcessor { posthog: PostHog @@ -18,10 +20,24 @@ export default class PosthogProcessor implements EventProcessor { properties: any, timestamp?: string | number ): Promise { + properties.version = pkg.version + properties.service = env.SERVICE const payload: any = { distinctId: identity.id, event, properties } if (timestamp) { payload.timestamp = new Date(timestamp) } + + // add groups to the event + if (identity.installationId || identity.tenantId) { + payload.groups = {} + if (identity.installationId) { + payload.groups.installation = identity.installationId + } + if (identity.tenantId) { + payload.groups.tenant = identity.tenantId + } + } + this.posthog.capture(payload) } @@ -33,6 +49,20 @@ export default class PosthogProcessor implements EventProcessor { this.posthog.identify(payload) } + async identifyGroup(group: Group, timestamp?: string | number) { + const payload: any = { + distinctId: group.id, + groupType: group.type, + groupKey: group.id, + properties: group, + } + + if (timestamp) { + payload.timestamp = new Date(timestamp) + } + this.posthog.groupIdentify(payload) + } + shutdown() { this.posthog.shutdown() } diff --git a/packages/backend-core/src/events/processors/Processors.ts b/packages/backend-core/src/events/processors/Processors.ts index 4263b7e06b..4baedd909f 100644 --- a/packages/backend-core/src/events/processors/Processors.ts +++ b/packages/backend-core/src/events/processors/Processors.ts @@ -1,4 +1,4 @@ -import { Event, Identity } from "@budibase/types" +import { Event, Identity, Group } from "@budibase/types" import { EventProcessor } from "./types" export default class Processor implements EventProcessor { @@ -29,6 +29,15 @@ export default class Processor implements EventProcessor { } } + async identifyGroup( + identity: Group, + timestamp?: string | number + ): Promise { + for (const eventProcessor of this.processors) { + await eventProcessor.identifyGroup(identity, timestamp) + } + } + shutdown() { for (const eventProcessor of this.processors) { eventProcessor.shutdown() diff --git a/packages/backend-core/src/events/processors/types.ts b/packages/backend-core/src/events/processors/types.ts index b8574cbd3d..f4066fe248 100644 --- a/packages/backend-core/src/events/processors/types.ts +++ b/packages/backend-core/src/events/processors/types.ts @@ -1,4 +1,4 @@ -import { Event, Identity } from "@budibase/types" +import { Event, Identity, Group } from "@budibase/types" export enum EventProcessorType { POSTHOG = "posthog", @@ -13,5 +13,6 @@ export interface EventProcessor { timestamp?: string | number ): Promise identify(identity: Identity, timestamp?: string | number): Promise + identifyGroup(group: Group, timestamp?: string | number): Promise shutdown(): void } diff --git a/packages/backend-core/src/events/publishers/layout.ts b/packages/backend-core/src/events/publishers/layout.ts index 7a334b059f..82e9f613ca 100644 --- a/packages/backend-core/src/events/publishers/layout.ts +++ b/packages/backend-core/src/events/publishers/layout.ts @@ -13,9 +13,9 @@ export async function created(layout: Layout, timestamp?: string) { await publishEvent(Event.LAYOUT_CREATED, properties, timestamp) } -export async function deleted(layout: Layout) { +export async function deleted(layoutId: string) { const properties: LayoutDeletedEvent = { - layoutId: layout._id as string, + layoutId, } await publishEvent(Event.LAYOUT_DELETED, properties) } diff --git a/packages/backend-core/src/events/publishers/serve.ts b/packages/backend-core/src/events/publishers/serve.ts index 62aaf4fb4c..8505df2987 100644 --- a/packages/backend-core/src/events/publishers/serve.ts +++ b/packages/backend-core/src/events/publishers/serve.ts @@ -9,17 +9,23 @@ import { /* eslint-disable */ -export async function servedBuilder(version: number) { +export async function servedBuilder() { const properties: BuilderServedEvent = {} await publishEvent(Event.SERVED_BUILDER, properties) } export async function servedApp(app: App) { - const properties: AppServedEvent = {} + const properties: AppServedEvent = { + appId: app.appId, + appVersion: app.version, + } await publishEvent(Event.SERVED_APP, properties) } export async function servedAppPreview(app: App) { - const properties: AppPreviewServedEvent = {} + const properties: AppPreviewServedEvent = { + appId: app.appId, + appVersion: app.version, + } await publishEvent(Event.SERVED_APP_PREVIEW, properties) } diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index c54ee1394e..b3687cc171 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -4,6 +4,7 @@ import * as events from "./events" import * as migrations from "./migrations" import * as users from "./users" import * as accounts from "./cloud/accounts" +import * as installation from "./installation" import env from "./environment" import tenancy from "./tenancy" import featureFlags from "./featureFlags" @@ -46,4 +47,5 @@ export = { events, sessions, deprovisioning, + installation, } diff --git a/packages/server/src/installation.ts b/packages/backend-core/src/installation.ts similarity index 71% rename from packages/server/src/installation.ts rename to packages/backend-core/src/installation.ts index 3832d6f474..25e4a311ed 100644 --- a/packages/server/src/installation.ts +++ b/packages/backend-core/src/installation.ts @@ -1,29 +1,27 @@ -import { - db as dbUtils, - events, - utils, - context, - tenancy, -} from "@budibase/backend-core" -import { Installation } from "@budibase/types" +import * as hashing from "./hashing" +import * as events from "./events" +import { StaticDatabases } from "./db/constants" +import { doWithDB } from "./db" +import { Installation, IdentityType } from "@budibase/types" +import * as context from "./context" import semver from "semver" const pkg = require("../package.json") export const getInstall = async (): Promise => { - return dbUtils.doWithDB( - dbUtils.StaticDatabases.PLATFORM_INFO.name, + return doWithDB( + StaticDatabases.PLATFORM_INFO.name, async (platformDb: any) => { let install: Installation try { install = await platformDb.get( - dbUtils.StaticDatabases.PLATFORM_INFO.docs.install + StaticDatabases.PLATFORM_INFO.docs.install ) } catch (e: any) { if (e.status === 404) { install = { - _id: dbUtils.StaticDatabases.PLATFORM_INFO.docs.install, - installId: utils.newid(), + _id: StaticDatabases.PLATFORM_INFO.docs.install, + installId: hashing.newid(), version: pkg.version, } const resp = await platformDb.put(install) @@ -40,8 +38,8 @@ export const getInstall = async (): Promise => { const updateVersion = async (version: string): Promise => { try { - await dbUtils.doWithDB( - dbUtils.StaticDatabases.PLATFORM_INFO.name, + await doWithDB( + StaticDatabases.PLATFORM_INFO.name, async (platformDb: any) => { const install = await getInstall() install.version = version @@ -73,11 +71,10 @@ export const checkInstallVersion = async (): Promise => { const success = await updateVersion(newVersion) if (success) { - await context.doInUserContext( + await context.doInIdentityContext( { _id: install.installId, - isInstall: true, - tenantId: tenancy.DEFAULT_TENANT_ID, + type: IdentityType.INSTALLATION, }, async () => { if (isUpgrade) { @@ -87,6 +84,7 @@ export const checkInstallVersion = async (): Promise => { } } ) + await events.identification.identifyInstallationGroup(install.installId) } } } diff --git a/packages/backend-core/src/middleware/authenticated.js b/packages/backend-core/src/middleware/authenticated.js index 3fef6eadac..66e0d6c755 100644 --- a/packages/backend-core/src/middleware/authenticated.js +++ b/packages/backend-core/src/middleware/authenticated.js @@ -7,7 +7,7 @@ const env = require("../environment") const { SEPARATOR, ViewNames, queryGlobalView } = require("../../db") const { getGlobalDB, doInTenant } = require("../tenancy") const { decrypt } = require("../security/encryption") -const context = require("../context") +const identity = require("../context/identity") function finalise( ctx, @@ -135,7 +135,7 @@ module.exports = ( finalise(ctx, { authenticated, user, internal, version, publicEndpoint }) if (user && user.email) { - return context.doInUserContext(user, next) + return identity.doInUserContext(user, next) } else { return next() } @@ -147,7 +147,7 @@ module.exports = ( // allow configuring for public access if ((opts && opts.publicAllowed) || publicEndpoint) { finalise(ctx, { authenticated: false, version, publicEndpoint }) - return context.doInUserContext({ _id: "public_user" }, next) + return next() } else { ctx.throw(err.status || 403, err) } diff --git a/packages/backend-core/src/tests/utilities/mocks/events.js b/packages/backend-core/src/tests/utilities/mocks/events.js index 86bfda2dbe..746deb3428 100644 --- a/packages/backend-core/src/tests/utilities/mocks/events.js +++ b/packages/backend-core/src/tests/utilities/mocks/events.js @@ -56,9 +56,11 @@ jest.mock("../../../events", () => { nameUpdated: jest.fn(), logoUpdated: jest.fn(), platformURLUpdated: jest.fn(), - versionChecked: jest.fn(), analyticsOptOut: jest.fn(), }, + version: { + checked: jest.fn(), + }, query: { created: jest.fn(), updated: jest.fn(), diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 9669545887..638fc47117 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -971,6 +971,11 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== +"@types/semver@^7.0.0": + version "7.3.9" + resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc" + integrity sha512-L/TMpyURfBkf+o/526Zb6kd/tchUP3iBDEPjqjb+U2MAJhVRxxrmr2fwpe08E7QsV7YLcpq0tUaQ9O9x97ZIxQ== + "@types/serve-static@*": version "1.13.10" resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.10.tgz#f5e0ce8797d2d7cc5ebeda48a52c96c4fa47a8d9" @@ -5012,7 +5017,7 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -semver@7.x, semver@^7.3.4: +semver@7.x, semver@^7.0.0, semver@^7.3.4: version "7.3.7" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== diff --git a/packages/server/package.json b/packages/server/package.json index 9e336d0280..36ea8391ee 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -129,7 +129,6 @@ "pouchdb-find": "^7.2.2", "pouchdb-replication-stream": "1.2.9", "redis": "4", - "semver": "^7.0.0", "server-destroy": "1.0.1", "svelte": "^3.38.2", "swagger-parser": "^10.0.3", @@ -160,7 +159,6 @@ "@types/node": "^15.12.4", "@types/oracledb": "^5.2.1", "@types/redis": "^4.0.11", - "@types/semver": "^7.0.0", "@typescript-eslint/parser": "5.12.0", "apidoc": "^0.50.2", "babel-jest": "^27.0.2", diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.js index 03ac140eb2..dd2bf45b21 100644 --- a/packages/server/src/api/controllers/dev.js +++ b/packages/server/src/api/controllers/dev.js @@ -131,5 +131,5 @@ exports.getBudibaseVersion = async ctx => { ctx.body = { version, } - await events.version.versionChecked(version) + await events.version.checked(version) } diff --git a/packages/server/src/api/controllers/layout.js b/packages/server/src/api/controllers/layout.js index 044d4a1acb..c7d3216cfc 100644 --- a/packages/server/src/api/controllers/layout.js +++ b/packages/server/src/api/controllers/layout.js @@ -20,7 +20,7 @@ exports.save = async function (ctx) { layout._id = layout._id || generateLayoutID() const response = await db.put(layout) - await events.layout.created() + await events.layout.created(layout) layout._rev = response.rev ctx.body = layout @@ -48,7 +48,7 @@ exports.destroy = async function (ctx) { } await db.remove(layoutId, layoutRev) - await events.layout.deleted() + await events.layout.deleted(layoutId) ctx.body = { message: "Layout deleted successfully" } ctx.status = 200 } diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index 07ad691faa..d21e38f879 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -19,7 +19,6 @@ const { getAppDB, getAppId } = require("@budibase/backend-core/context") const AWS = require("aws-sdk") const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1" const { events } = require("@budibase/backend-core") -const version = require("../../../../package.json").version async function prepareUpload({ s3Key, bucket, metadata, file }) { const response = await upload({ @@ -43,7 +42,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) { exports.serveBuilder = async function (ctx) { let builderPath = resolve(TOP_LEVEL_PATH, "builder") await send(ctx, ctx.file, { root: builderPath }) - await events.serve.servedBuilder(version) + await events.serve.servedBuilder() } exports.uploadFile = async function (ctx) { diff --git a/packages/server/src/api/routes/tests/dev.spec.js b/packages/server/src/api/routes/tests/dev.spec.js index 0138ce2a6c..8dee3fd418 100644 --- a/packages/server/src/api/routes/tests/dev.spec.js +++ b/packages/server/src/api/routes/tests/dev.spec.js @@ -33,8 +33,8 @@ describe("/dev", () => { .expect(200) expect(res.body.version).toBe(version) - expect(events.org.versionChecked).toBeCalledTimes(1) - expect(events.org.versionChecked).toBeCalledWith(version) + expect(events.version.checked).toBeCalledTimes(1) + expect(events.version.checked).toBeCalledWith(version) }) }) }) \ No newline at end of file diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 127d0f8853..63dc3f1a6f 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -16,8 +16,7 @@ const fileSystem = require("./utilities/fileSystem") const bullboard = require("./automations/bullboard") import redis from "./utilities/redis" import * as migrations from "./migrations" -import { events } from "@budibase/backend-core" -import * as installation from "./installation" +import { events, installation } from "@budibase/backend-core" const app = new Koa() diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts index 99d08c46ed..a03ae67f41 100644 --- a/packages/server/src/migrations/functions/backfill/global.ts +++ b/packages/server/src/migrations/functions/backfill/global.ts @@ -2,9 +2,8 @@ import * as users from "./global/users" import * as rows from "./global/rows" import * as configs from "./global/configs" import { tenancy, events, migrations, accounts } from "@budibase/backend-core" -import { CloudAccount, Installation } from "@budibase/types" +import { CloudAccount } from "@budibase/types" import env from "../../../environment" -import * as installation from "../../../installation" /** * Date: @@ -22,7 +21,7 @@ export const run = async (db: any) => { account = await accounts.getAccountByTenantId(tenantId) } - await events.identification.identifyTenant( + await events.identification.identifyTenantGroup( tenantId, account, installTimestamp diff --git a/packages/server/src/migrations/functions/backfill/installation.ts b/packages/server/src/migrations/functions/backfill/installation.ts index bf3b413fa7..b807d650e8 100644 --- a/packages/server/src/migrations/functions/backfill/installation.ts +++ b/packages/server/src/migrations/functions/backfill/installation.ts @@ -1,6 +1,5 @@ -import { events, tenancy } from "@budibase/backend-core" +import { events, tenancy, installation } from "@budibase/backend-core" import { Installation } from "@budibase/types" -import * as installation from "../../../installation" import * as global from "./global" /** @@ -17,6 +16,9 @@ export const run = async () => { const db = tenancy.getGlobalDB() const installTimestamp = (await global.getInstallTimestamp(db)) as number const install: Installation = await installation.getInstall() - await events.identification.identifyInstallation(install, installTimestamp) + await events.identification.identifyInstallationGroup( + install.installId, + installTimestamp + ) }) } diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 7236ec8d84..4548fd2bd0 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -2754,11 +2754,6 @@ "@types/tough-cookie" "*" form-data "^2.5.0" -"@types/semver@^7.0.0": - version "7.3.9" - resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc" - integrity sha512-L/TMpyURfBkf+o/526Zb6kd/tchUP3iBDEPjqjb+U2MAJhVRxxrmr2fwpe08E7QsV7YLcpq0tUaQ9O9x97ZIxQ== - "@types/serve-static@*": version "1.13.10" resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.10.tgz#f5e0ce8797d2d7cc5ebeda48a52c96c4fa47a8d9" @@ -11650,13 +11645,6 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== -semver@^7.0.0: - version "7.3.7" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" - integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== - dependencies: - lru-cache "^6.0.0" - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" diff --git a/packages/types/src/core/context.ts b/packages/types/src/core/context.ts new file mode 100644 index 0000000000..bf433d5e5f --- /dev/null +++ b/packages/types/src/core/context.ts @@ -0,0 +1,19 @@ +import { User, Account } from "../documents" +import { IdentityType } from "./identification" + +export interface BaseContext { + _id: string + type: IdentityType +} + +export interface AccountUserContext extends BaseContext { + tenantId: string + account: Account +} + +export interface UserContext extends BaseContext, User { + _id: string + account?: Account +} + +export type IdentityContext = BaseContext | AccountUserContext | UserContext diff --git a/packages/types/src/core/identification.ts b/packages/types/src/core/identification.ts new file mode 100644 index 0000000000..b7ce0fbaf4 --- /dev/null +++ b/packages/types/src/core/identification.ts @@ -0,0 +1,49 @@ +import { Hosting } from "." + +// GROUPS + +export enum GroupType { + TENANT = "tenant", + INSTALLATION = "installation", +} + +export interface Group { + id: string + type: IdentityType +} + +export interface TenantGroup extends Group { + // account level information is associated with the tenant group + // as we don't have this at the user level + profession?: string // only available in cloud + companySize?: string // only available in cloud + hosting: Hosting // need hosting at the tenant level for cloud self host accounts +} + +export interface InstallationGroup extends Group { + version: string + hosting: Hosting +} + +// IDENTITIES + +export enum IdentityType { + USER = "user", + TENANT = "tenant", + INSTALLATION = "installation", +} + +export interface Identity { + id: string + type: IdentityType + installationId?: string + tenantId?: string +} + +export interface UserIdentity extends Identity { + verified: boolean + accountHolder: boolean + providerType?: string + builder?: boolean + admin?: boolean +} diff --git a/packages/types/src/core/index.ts b/packages/types/src/core/index.ts index 23d772467a..bed86d4f9f 100644 --- a/packages/types/src/core/index.ts +++ b/packages/types/src/core/index.ts @@ -1,2 +1,3 @@ export * from "./hosting" -export * from "./sessions" +export * from "./context" +export * from "./identification" diff --git a/packages/types/src/core/sessions.ts b/packages/types/src/core/sessions.ts deleted file mode 100644 index 56db6e4f54..0000000000 --- a/packages/types/src/core/sessions.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { User, Account } from "../documents" -import { Hosting } from "./hosting" - -/** - * Account portal user session. Used for self hosted accounts only. - */ -export interface AccountUserSession { - _id: string - email: string - tenantId: string - accountPortalAccess: boolean - account: Account -} - -/** - * Budibase user session. - */ -export interface BudibaseUserSession extends User { - _id: string // overwrite potentially undefined - account?: Account - accountPortalAccess?: boolean -} - -export const isAccountSession = ( - user: AccountUserSession | BudibaseUserSession -): user is AccountUserSession => { - return user.account?.hosting === Hosting.SELF -} - -export const isUserSession = ( - user: AccountUserSession | BudibaseUserSession -): user is BudibaseUserSession => { - return !user.account || user.account?.hosting === Hosting.CLOUD -} - -// not technically a session, but used to identify the installation -export interface InstallationSession { - _id: string - isInstallation: boolean -} - -export const isInstallation = (user: any): user is InstallationSession => { - return !!user.isInstallation -} - -export type SessionUser = - | AccountUserSession - | BudibaseUserSession - | InstallationSession diff --git a/packages/types/src/events/auth.ts b/packages/types/src/events/auth.ts index c0e046a2ca..c7e6cbb4ef 100644 --- a/packages/types/src/events/auth.ts +++ b/packages/types/src/events/auth.ts @@ -1,4 +1,4 @@ -export type LoginSource = "local" | "google" | "oidc" +export type LoginSource = "local" | "google" | "oidc" | "google-internal" export type SSOType = "oidc" | "google" export interface LoginEvent { diff --git a/packages/types/src/events/event.ts b/packages/types/src/events/event.ts index 910a30f925..d5528ea114 100644 --- a/packages/types/src/events/event.ts +++ b/packages/types/src/events/event.ts @@ -8,7 +8,7 @@ export enum Event { USER_PERMISSION_ADMIN_ASSIGNED = "user:admin:assigned", USER_PERMISSION_ADMIN_REMOVED = "user:admin:removed", USER_PERMISSION_BUILDER_ASSIGNED = "user:builder:assigned", - USER_PERMISSION_BUILDER_REMOVED = "userbuilder:removed", + USER_PERMISSION_BUILDER_REMOVED = "user:builder:removed", // USER / INVITE USER_INVITED = "user:invited", diff --git a/packages/types/src/events/identification.ts b/packages/types/src/events/identification.ts deleted file mode 100644 index 29d0aa333e..0000000000 --- a/packages/types/src/events/identification.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Hosting } from "../core" - -export enum IdentityType { - USER = "user", - TENANT = "tenant", - INSTALLATION = "installation", -} - -export interface Identity { - id: string - tenantId: string - type: IdentityType -} - -export interface InstallationIdentity extends Identity { - version: string - hosting: Hosting -} - -export interface TenantIdentity extends Identity { - hosting: Hosting - profession?: string - companySize?: string -} - -export interface UserIdentity extends TenantIdentity { - hosting: Hosting - type: IdentityType - verified: boolean - accountHolder: boolean - providerType?: string -} - -export interface BudibaseIdentity extends UserIdentity { - builder?: boolean - admin?: boolean -} diff --git a/packages/types/src/events/index.ts b/packages/types/src/events/index.ts index c55711205c..44f6f297a6 100644 --- a/packages/types/src/events/index.ts +++ b/packages/types/src/events/index.ts @@ -15,5 +15,4 @@ export * from "./serve" export * from "./table" export * from "./user" export * from "./view" -export * from "./identification" export * from "./account" diff --git a/packages/types/src/events/serve.ts b/packages/types/src/events/serve.ts index f3fd21dd15..8ac9ae2c6b 100644 --- a/packages/types/src/events/serve.ts +++ b/packages/types/src/events/serve.ts @@ -1,5 +1,11 @@ export interface BuilderServedEvent {} -export interface AppServedEvent {} +export interface AppServedEvent { + appId: string + appVersion: string +} -export interface AppPreviewServedEvent {} +export interface AppPreviewServedEvent { + appId: string + appVersion: string +} diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index 71d0afbb52..35d705c2fd 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -70,9 +70,9 @@ async function authInternal(ctx: any, user: any, err = null, info = null) { export const authenticate = async (ctx: any, next: any) => { return passport.authenticate( "local", - async (err: any, user: any, info: any) => { + async (err: any, user: User, info: any) => { await authInternal(ctx, user, err, info) - await context.doInUserContext(user, async () => { + await context.identity.doInUserContext(user, async () => { await events.auth.login("local") }) ctx.status = 200 @@ -213,10 +213,10 @@ export const googleAuth = async (ctx: any, next: any) => { return passport.authenticate( strategy, { successRedirect: "/", failureRedirect: "/error" }, - async (err: any, user: any, info: any) => { + async (err: any, user: User, info: any) => { await authInternal(ctx, user, err, info) - await context.doInUserContext(user, async () => { - await events.auth.login("google") + await context.identity.doInUserContext(user, async () => { + await events.auth.login("google-internal") }) ctx.redirect("/") } @@ -261,7 +261,7 @@ export const oidcAuth = async (ctx: any, next: any) => { { successRedirect: "/", failureRedirect: "/error" }, async (err: any, user: any, info: any) => { await authInternal(ctx, user, err, info) - await context.doInUserContext(user, async () => { + await context.identity.doInUserContext(user, async () => { await events.auth.login("oidc") }) ctx.redirect("/") diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 42b98fd791..5a86ace16b 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -69,7 +69,7 @@ export const adminUser = async (ctx: any) => { if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { account = await accounts.getAccountByTenantId(tenantId) } - await events.identification.identifyTenant(tenantId, account) + await events.identification.identifyTenantGroup(tenantId, account) } catch (err: any) { ctx.throw(err.status || 400, err) } diff --git a/scripts/link-dependencies.sh b/scripts/link-dependencies.sh index 8c4294fa08..cf9c536b1d 100755 --- a/scripts/link-dependencies.sh +++ b/scripts/link-dependencies.sh @@ -8,6 +8,12 @@ cd packages/string-templates yarn link cd - +echo "Linking types" +cd packages/types +yarn link +cd - + + if [ -d "../budibase-pro" ]; then cd ../budibase-pro yarn bootstrap @@ -38,6 +44,9 @@ if [ -d "../account-portal" ]; then echo "Linking string-templates to account-portal" yarn link "@budibase/string-templates" + echo "Linking types to account-portal" + yarn link "@budibase/types" + if [ -d "../../../budibase-pro" ]; then echo "Linking pro to account-portal" yarn link "@budibase/pro"