From 500e697974a7d9c5308f0a37ada7c16e2245d081 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 10 Aug 2022 11:01:54 +0100 Subject: [PATCH 1/2] Trigger served events on ping endpoint --- .../src/events/emit/BBEventEmitter.ts | 27 ---------- .../backend-core/src/events/emit/index.ts | 1 - packages/backend-core/src/events/events.ts | 40 -------------- .../builder/src/pages/builder/_layout.svelte | 7 +++ .../client/src/components/ClientApp.svelte | 2 + packages/client/src/stores/builder.js | 8 +++ packages/frontend-core/src/api/analytics.js | 6 +++ .../server/src/api/controllers/analytics.ts | 27 ++++++++++ .../src/api/controllers/static/index.ts | 12 +---- packages/server/src/api/routes/analytics.js | 1 + .../src/api/routes/tests/analytics.spec.js | 54 +++++++++++++++++++ .../src/api/routes/tests/static.spec.js | 10 ---- .../src/tests/utilities/TestConfiguration.js | 27 ++++------ packages/types/src/api/index.ts | 1 + packages/types/src/api/web/.keep | 0 packages/types/src/api/web/analytics.ts | 8 +++ packages/types/src/api/web/index.ts | 1 + packages/types/src/index.ts | 1 + 18 files changed, 127 insertions(+), 106 deletions(-) delete mode 100644 packages/backend-core/src/events/emit/BBEventEmitter.ts delete mode 100644 packages/backend-core/src/events/emit/index.ts create mode 100644 packages/server/src/api/routes/tests/analytics.spec.js create mode 100644 packages/types/src/api/index.ts delete mode 100644 packages/types/src/api/web/.keep create mode 100644 packages/types/src/api/web/analytics.ts create mode 100644 packages/types/src/api/web/index.ts diff --git a/packages/backend-core/src/events/emit/BBEventEmitter.ts b/packages/backend-core/src/events/emit/BBEventEmitter.ts deleted file mode 100644 index 41668858e1..0000000000 --- a/packages/backend-core/src/events/emit/BBEventEmitter.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { EventEmitter } from "events" -import * as context from "../../context" -import { Identity, Event } from "@budibase/types" - -export interface EmittedEvent { - tenantId: string - identity: Identity - appId: string | undefined - properties: any -} - -class BBEventEmitter extends EventEmitter { - emitEvent(event: Event, properties: any, identity: Identity) { - const tenantId = context.getTenantId() - const appId = context.getAppId() - - const emittedEvent: EmittedEvent = { - tenantId, - identity, - appId, - properties, - } - this.emit(event, emittedEvent) - } -} - -export const emitter = new BBEventEmitter() diff --git a/packages/backend-core/src/events/emit/index.ts b/packages/backend-core/src/events/emit/index.ts deleted file mode 100644 index 60e47f7a13..0000000000 --- a/packages/backend-core/src/events/emit/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./BBEventEmitter" diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index 63fedf4754..cda90d12c9 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -2,41 +2,6 @@ import { Event } from "@budibase/types" import { processors } from "./processors" import * as identification from "./identification" import * as backfill from "./backfill" -import { emitter, EmittedEvent } from "./emit" -import * as context from "../context" -import * as logging from "../logging" - -const USE_EMITTER: any[] = [ - Event.SERVED_BUILDER, - Event.SERVED_APP, - Event.SERVED_APP_PREVIEW, -] - -for (let event of USE_EMITTER) { - emitter.on(event, async (props: EmittedEvent) => { - try { - await context.doInTenant(props.tenantId, async () => { - if (props.appId) { - await context.doInAppContext(props.appId, async () => { - await processors.processEvent( - event as Event, - props.identity, - props.properties - ) - }) - } else { - await processors.processEvent( - event as Event, - props.identity, - props.properties - ) - } - }) - } catch (e) { - logging.logAlert(`Unable to process async event ${event}`, e) - } - }) -} export const publishEvent = async ( event: Event, @@ -46,11 +11,6 @@ export const publishEvent = async ( // in future this should use async events via a distributed queue. const identity = await identification.getCurrentIdentity() - if (USE_EMITTER.includes(event)) { - emitter.emitEvent(event, properties, identity) - return - } - const backfilling = await backfill.isBackfillingEvent(event) // no backfill - send the event and exit if (!backfilling) { diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte index cb760cd165..2e8ea2ef0a 100644 --- a/packages/builder/src/pages/builder/_layout.svelte +++ b/packages/builder/src/pages/builder/_layout.svelte @@ -3,6 +3,7 @@ import { admin, auth } from "stores/portal" import { onMount } from "svelte" import { CookieUtils, Constants } from "@budibase/frontend-core" + import { API } from "api" let loaded = false @@ -53,6 +54,9 @@ await auth.setOrganisation(urlTenantId) } } + async function analyticsPing() { + await API.analyticsPing({ source: "builder" }) + } onMount(async () => { try { @@ -73,6 +77,9 @@ // being logged in } loaded = true + + // lastly + await analyticsPing() }) $: { diff --git a/packages/client/src/components/ClientApp.svelte b/packages/client/src/components/ClientApp.svelte index 64b1712b89..884f6e8ea1 100644 --- a/packages/client/src/components/ClientApp.svelte +++ b/packages/client/src/components/ClientApp.svelte @@ -83,6 +83,8 @@ dataLoaded = true if (get(builderStore).inBuilder) { builderStore.actions.notifyLoaded() + } else { + builderStore.actions.analyticsPing({ source: "app" }) } }) diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index 9d85eee022..be748f0d81 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -1,4 +1,5 @@ import { writable, get } from "svelte/store" +import { API } from "api" import { devToolsStore } from "./devTools.js" const dispatchEvent = (type, data = {}) => { @@ -48,6 +49,13 @@ const createBuilderStore = () => { notifyLoaded: () => { dispatchEvent("preview-loaded") }, + analyticsPing: async () => { + try { + await API.analyticsPing({ source: "app" }) + } catch (error) { + // Do nothing + } + }, moveComponent: (componentId, destinationComponentId, mode) => { dispatchEvent("move-component", { componentId, diff --git a/packages/frontend-core/src/api/analytics.js b/packages/frontend-core/src/api/analytics.js index 5aa35469ae..0f7a19323a 100644 --- a/packages/frontend-core/src/api/analytics.js +++ b/packages/frontend-core/src/api/analytics.js @@ -7,4 +7,10 @@ export const buildAnalyticsEndpoints = API => ({ url: "/api/bbtel", }) }, + analyticsPing: async ({ source }) => { + return await API.post({ + url: "/api/bbtel/ping", + body: { source }, + }) + }, }) diff --git a/packages/server/src/api/controllers/analytics.ts b/packages/server/src/api/controllers/analytics.ts index efb9115e54..d763658cca 100644 --- a/packages/server/src/api/controllers/analytics.ts +++ b/packages/server/src/api/controllers/analytics.ts @@ -1,4 +1,7 @@ import { events } from "@budibase/backend-core" +import { AnalyticsPingRequest, PingSource } from "@budibase/types" +import { DocumentTypes, isDevAppID } from "../../db/utils" +import { context } from "@budibase/backend-core" export const isEnabled = async (ctx: any) => { const enabled = await events.analytics.enabled() @@ -6,3 +9,27 @@ export const isEnabled = async (ctx: any) => { enabled, } } + +export const ping = async (ctx: any) => { + const body = ctx.request.body as AnalyticsPingRequest + switch (body.source) { + case PingSource.APP: { + const db = context.getAppDB({ skip_setup: true }) + const appInfo = await db.get(DocumentTypes.APP_METADATA) + let appId = context.getAppId() + + if (isDevAppID(appId)) { + await events.serve.servedAppPreview(appInfo) + } else { + await events.serve.servedApp(appInfo) + } + break + } + case PingSource.BUILDER: { + await events.serve.servedBuilder() + break + } + } + + ctx.status = 200 +} diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index b0e36b8943..235bcd234c 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -14,11 +14,10 @@ const env = require("../../../environment") const { clientLibraryPath } = require("../../../utilities") const { upload } = require("../../../utilities/fileSystem") const { attachmentsRelativeURL } = require("../../../utilities") -const { DocumentTypes, isDevAppID } = require("../../../db/utils") +const { DocumentTypes } = require("../../../db/utils") const { getAppDB, getAppId } = require("@budibase/backend-core/context") const { setCookie, clearCookie } = require("@budibase/backend-core/utils") const AWS = require("aws-sdk") -import { events } from "@budibase/backend-core" const fs = require("fs") const { @@ -75,9 +74,6 @@ export const toggleBetaUiFeature = async function (ctx: any) { export const serveBuilder = async function (ctx: any) { const builderPath = resolve(TOP_LEVEL_PATH, "builder") await send(ctx, ctx.file, { root: builderPath }) - if (ctx.file === "index.html") { - await events.serve.servedBuilder() - } } export const uploadFile = async function (ctx: any) { @@ -126,12 +122,6 @@ export const serveApp = async function (ctx: any) { // just return the app info for jest to assert on ctx.body = appInfo } - - if (isDevAppID(appInfo.appId)) { - await events.serve.servedAppPreview(appInfo) - } else { - await events.serve.servedApp(appInfo) - } } export const serveClientLibrary = async function (ctx: any) { diff --git a/packages/server/src/api/routes/analytics.js b/packages/server/src/api/routes/analytics.js index 1f58ea04e7..d13ace12d1 100644 --- a/packages/server/src/api/routes/analytics.js +++ b/packages/server/src/api/routes/analytics.js @@ -4,5 +4,6 @@ const controller = require("../controllers/analytics") const router = Router() router.get("/api/bbtel", controller.isEnabled) +router.post("/api/bbtel/ping", controller.ping) module.exports = router diff --git a/packages/server/src/api/routes/tests/analytics.spec.js b/packages/server/src/api/routes/tests/analytics.spec.js new file mode 100644 index 0000000000..652d1e7ead --- /dev/null +++ b/packages/server/src/api/routes/tests/analytics.spec.js @@ -0,0 +1,54 @@ +const setup = require("./utilities") +const { events, constants, db } = require("@budibase/backend-core") + +describe("/static", () => { + let request = setup.getRequest() + let config = setup.getConfig() + let app + + afterAll(setup.afterAll) + + beforeEach(async () => { + app = await config.init() + jest.clearAllMocks() + }) + + describe("/ping", () => { + it("should ping from builder", async () => { + await request + .post("/api/bbtel/ping") + .send({source: "builder"}) + .set(config.defaultHeaders()) + .expect(200) + + expect(events.serve.servedBuilder).toBeCalledTimes(1) + }) + + it("should ping from app preview", async () => { + await request + .post("/api/bbtel/ping") + .send({source: "app"}) + .set(config.defaultHeaders()) + .expect(200) + + expect(events.serve.servedAppPreview).toBeCalledTimes(1) + expect(events.serve.servedAppPreview).toBeCalledWith(config.getApp()) + expect(events.serve.servedApp).not.toBeCalled() + }) + + it("should ping from app", async () => { + const headers = config.defaultHeaders() + headers[constants.Headers.APP_ID] = config.prodAppId + + await request + .post("/api/bbtel/ping") + .send({source: "app"}) + .set(headers) + .expect(200) + + expect(events.serve.servedApp).toBeCalledTimes(1) + expect(events.serve.servedApp).toBeCalledWith(config.getProdApp()) + expect(events.serve.servedAppPreview).not.toBeCalled() + }) + }) +}) diff --git a/packages/server/src/api/routes/tests/static.spec.js b/packages/server/src/api/routes/tests/static.spec.js index f0a1e6621c..37176f5cf5 100644 --- a/packages/server/src/api/routes/tests/static.spec.js +++ b/packages/server/src/api/routes/tests/static.spec.js @@ -36,7 +36,6 @@ describe("/static", () => { .expect(200) expect(res.text).toContain("Budibase") - expect(events.serve.servedBuilder).toBeCalledTimes(1) }) }) @@ -56,9 +55,6 @@ describe("/static", () => { .expect(200) expect(res.body.appId).toBe(config.prodAppId) - expect(events.serve.servedApp).toBeCalledTimes(1) - expect(events.serve.servedApp).toBeCalledWith(res.body) - expect(events.serve.servedAppPreview).not.toBeCalled() }) it("should serve the app by url", async () => { @@ -71,9 +67,6 @@ describe("/static", () => { .expect(200) expect(res.body.appId).toBe(config.prodAppId) - expect(events.serve.servedApp).toBeCalledTimes(1) - expect(events.serve.servedApp).toBeCalledWith(res.body) - expect(events.serve.servedAppPreview).not.toBeCalled() }) it("should serve the app preview by id", async () => { @@ -83,9 +76,6 @@ describe("/static", () => { .expect(200) expect(res.body.appId).toBe(config.appId) - expect(events.serve.servedAppPreview).toBeCalledTimes(1) - expect(events.serve.servedAppPreview).toBeCalledWith(res.body) - expect(events.serve.servedApp).not.toBeCalled() }) }) diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index da108b6978..cf0bd6f5b6 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -25,6 +25,7 @@ const newid = require("../../db/newid") const context = require("@budibase/backend-core/context") const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db") const { encrypt } = require("@budibase/backend-core/encryption") +const { DocumentTypes } = require("../../db/utils") const GLOBAL_USER_ID = "us_uuid1" const EMAIL = "babs@babs.com" @@ -53,6 +54,10 @@ class TestConfiguration { return this.app } + getProdApp() { + return this.prodApp + } + getAppId() { return this.appId } @@ -106,19 +111,11 @@ class TestConfiguration { // UTILS - async _req(body, params, controlFunc, opts = { prodApp: false }) { + async _req(body, params, controlFunc) { // create a fake request ctx const request = {} - - // set the app id - let appId - if (opts.prodApp) { - appId = this.prodAppId - } else { - appId = this.appId - } + const appId = this.appId request.appId = appId - // fake cookies, we don't need them request.cookies = { set: () => {}, get: () => {} } request.config = { jwtSecret: env.JWT_SECRET } @@ -344,14 +341,10 @@ class TestConfiguration { await this._req(null, null, controllers.deploy.deployApp) const prodAppId = this.getAppId().replace("_dev", "") this.prodAppId = prodAppId + return context.doInAppContext(prodAppId, async () => { - const appPackage = await this._req( - null, - { appId: prodAppId }, - controllers.app.fetchAppPackage, - { prodApp: true } - ) - return appPackage.application + const db = context.getProdAppDB() + return await db.get(DocumentTypes.APP_METADATA) }) } diff --git a/packages/types/src/api/index.ts b/packages/types/src/api/index.ts new file mode 100644 index 0000000000..36a400ee16 --- /dev/null +++ b/packages/types/src/api/index.ts @@ -0,0 +1 @@ +export * from "./web" diff --git a/packages/types/src/api/web/.keep b/packages/types/src/api/web/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/types/src/api/web/analytics.ts b/packages/types/src/api/web/analytics.ts new file mode 100644 index 0000000000..886803a809 --- /dev/null +++ b/packages/types/src/api/web/analytics.ts @@ -0,0 +1,8 @@ +export enum PingSource { + BUILDER = "builder", + APP = "app", +} + +export interface AnalyticsPingRequest { + source: PingSource +} diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts new file mode 100644 index 0000000000..b2258fe18e --- /dev/null +++ b/packages/types/src/api/web/index.ts @@ -0,0 +1 @@ +export * from "./analytics" diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index b7453c7349..4a645c5266 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -2,3 +2,4 @@ export * from "./documents" export * from "./sdk/events" export * from "./sdk/licensing" export * from "./sdk" +export * from "./api" From 54df92a8c31d43a90f91074b1385e19f881c1f12 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 10 Aug 2022 11:29:11 +0100 Subject: [PATCH 2/2] Add timezone to served events --- .../backend-core/src/events/publishers/serve.ts | 12 ++++++++---- packages/frontend-core/src/api/analytics.js | 3 ++- packages/server/src/api/controllers/analytics.ts | 6 +++--- .../server/src/api/routes/tests/analytics.spec.js | 15 ++++++++++----- packages/types/src/api/web/analytics.ts | 1 + packages/types/src/sdk/events/serve.ts | 6 +++++- 6 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/events/publishers/serve.ts b/packages/backend-core/src/events/publishers/serve.ts index 13afede029..128e0b9b11 100644 --- a/packages/backend-core/src/events/publishers/serve.ts +++ b/packages/backend-core/src/events/publishers/serve.ts @@ -7,22 +7,26 @@ import { AppServedEvent, } from "@budibase/types" -export async function servedBuilder() { - const properties: BuilderServedEvent = {} +export async function servedBuilder(timezone: string) { + const properties: BuilderServedEvent = { + timezone, + } await publishEvent(Event.SERVED_BUILDER, properties) } -export async function servedApp(app: App) { +export async function servedApp(app: App, timezone: string) { const properties: AppServedEvent = { appVersion: app.version, + timezone, } await publishEvent(Event.SERVED_APP, properties) } -export async function servedAppPreview(app: App) { +export async function servedAppPreview(app: App, timezone: string) { const properties: AppPreviewServedEvent = { appId: app.appId, appVersion: app.version, + timezone, } await publishEvent(Event.SERVED_APP_PREVIEW, properties) } diff --git a/packages/frontend-core/src/api/analytics.js b/packages/frontend-core/src/api/analytics.js index 0f7a19323a..653943c6ab 100644 --- a/packages/frontend-core/src/api/analytics.js +++ b/packages/frontend-core/src/api/analytics.js @@ -8,9 +8,10 @@ export const buildAnalyticsEndpoints = API => ({ }) }, analyticsPing: async ({ source }) => { + const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone return await API.post({ url: "/api/bbtel/ping", - body: { source }, + body: { source, timezone }, }) }, }) diff --git a/packages/server/src/api/controllers/analytics.ts b/packages/server/src/api/controllers/analytics.ts index d763658cca..44a99bbbb6 100644 --- a/packages/server/src/api/controllers/analytics.ts +++ b/packages/server/src/api/controllers/analytics.ts @@ -19,14 +19,14 @@ export const ping = async (ctx: any) => { let appId = context.getAppId() if (isDevAppID(appId)) { - await events.serve.servedAppPreview(appInfo) + await events.serve.servedAppPreview(appInfo, body.timezone) } else { - await events.serve.servedApp(appInfo) + await events.serve.servedApp(appInfo, body.timezone) } break } case PingSource.BUILDER: { - await events.serve.servedBuilder() + await events.serve.servedBuilder(body.timezone) break } } diff --git a/packages/server/src/api/routes/tests/analytics.spec.js b/packages/server/src/api/routes/tests/analytics.spec.js index 652d1e7ead..73d5810d7f 100644 --- a/packages/server/src/api/routes/tests/analytics.spec.js +++ b/packages/server/src/api/routes/tests/analytics.spec.js @@ -6,6 +6,8 @@ describe("/static", () => { let config = setup.getConfig() let app + const timezone = "Europe/London" + afterAll(setup.afterAll) beforeEach(async () => { @@ -17,22 +19,25 @@ describe("/static", () => { it("should ping from builder", async () => { await request .post("/api/bbtel/ping") - .send({source: "builder"}) + .send({source: "builder", timezone}) .set(config.defaultHeaders()) .expect(200) expect(events.serve.servedBuilder).toBeCalledTimes(1) + expect(events.serve.servedBuilder).toBeCalledWith(timezone) + expect(events.serve.servedApp).not.toBeCalled() + expect(events.serve.servedAppPreview).not.toBeCalled() }) it("should ping from app preview", async () => { await request .post("/api/bbtel/ping") - .send({source: "app"}) + .send({source: "app", timezone}) .set(config.defaultHeaders()) .expect(200) expect(events.serve.servedAppPreview).toBeCalledTimes(1) - expect(events.serve.servedAppPreview).toBeCalledWith(config.getApp()) + expect(events.serve.servedAppPreview).toBeCalledWith(config.getApp(), timezone) expect(events.serve.servedApp).not.toBeCalled() }) @@ -42,12 +47,12 @@ describe("/static", () => { await request .post("/api/bbtel/ping") - .send({source: "app"}) + .send({source: "app", timezone}) .set(headers) .expect(200) expect(events.serve.servedApp).toBeCalledTimes(1) - expect(events.serve.servedApp).toBeCalledWith(config.getProdApp()) + expect(events.serve.servedApp).toBeCalledWith(config.getProdApp(), timezone) expect(events.serve.servedAppPreview).not.toBeCalled() }) }) diff --git a/packages/types/src/api/web/analytics.ts b/packages/types/src/api/web/analytics.ts index 886803a809..018c85ded2 100644 --- a/packages/types/src/api/web/analytics.ts +++ b/packages/types/src/api/web/analytics.ts @@ -5,4 +5,5 @@ export enum PingSource { export interface AnalyticsPingRequest { source: PingSource + timezone: string } diff --git a/packages/types/src/sdk/events/serve.ts b/packages/types/src/sdk/events/serve.ts index fe5b723942..cb03c47e05 100644 --- a/packages/types/src/sdk/events/serve.ts +++ b/packages/types/src/sdk/events/serve.ts @@ -1,11 +1,15 @@ import { BaseEvent } from "./event" -export interface BuilderServedEvent extends BaseEvent {} +export interface BuilderServedEvent extends BaseEvent { + timezone: string +} export interface AppServedEvent extends BaseEvent { appVersion: string + timezone: string } export interface AppPreviewServedEvent extends BaseEvent { appVersion: string + timezone: string }