From 113a23ba3d1440d30e433b5c36c0d9e9ecd379b6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Oct 2022 16:48:32 +0100 Subject: [PATCH] Adding statistics output for backups. --- .../src/events/publishers/backup.ts | 4 +- packages/server/src/sdk/app/backups/backup.ts | 22 ++++-- .../server/src/sdk/app/statistics/index.ts | 77 +++++++++++++++++++ packages/types/src/documents/app/backup.ts | 26 +++---- 4 files changed, 105 insertions(+), 24 deletions(-) create mode 100644 packages/server/src/sdk/app/statistics/index.ts diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts index 00b4f8db69..bd346cad64 100644 --- a/packages/backend-core/src/events/publishers/backup.ts +++ b/packages/backend-core/src/events/publishers/backup.ts @@ -4,8 +4,8 @@ import { publishEvent } from "../events" export async function appBackupRestored(backup: AppBackup) { const properties: AppBackupRevertEvent = { appId: backup.appId, - backupName: backup.name, - backupCreatedAt: backup.createdAt, + backupName: backup.name!, + backupCreatedAt: backup.timestamp, } await publishEvent(Event.APP_BACKUP_RESTORED, properties) diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts index 0f2a1aa760..6f7906d1cc 100644 --- a/packages/server/src/sdk/app/backups/backup.ts +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -3,6 +3,7 @@ import { objectStore, tenancy, db as dbCore } from "@budibase/backend-core" import { AppBackupQueueData } from "@budibase/types" import { exportApp } from "./exports" import { importApp } from "./imports" +import { calculateBackupStats } from "../statistics" import { Job } from "bull" import fs from "fs" import env from "../../../environment" @@ -52,26 +53,35 @@ async function exportProcessor(job: Job) { name = data.export!.name || `${trigger} - backup` const tenantId = tenancy.getTenantIDFromAppID(appId) await tenancy.doInTenant(tenantId, async () => { - const createdAt = new Date().toISOString() - const tarPath = await exportApp(appId, { tar: true }) - let filename = `${appId}/backup-${createdAt}.tar.gz` + const devAppId = dbCore.getDevAppID(appId), + prodAppId = dbCore.getProdAppID(appId) + const timestamp = new Date().toISOString() + const tarPath = await exportApp(devAppId, { tar: true }) + const contents = await calculateBackupStats(devAppId) + let filename = `${prodAppId}/backup-${timestamp}.tar.gz` // add the tenant to the bucket path if backing up within a multi-tenant environment if (env.MULTI_TENANCY) { filename = `${tenantId}/${filename}` } const bucket = objectStore.ObjectStoreBuckets.BACKUPS const metadata = { - appId, - createdAt, + appId: prodAppId, + timestamp, trigger, name, + contents, } await objectStore.upload({ path: tarPath, type: "application/gzip", bucket, filename, - metadata, + metadata: { + name, + trigger, + timestamp, + appId: prodAppId, + }, }) await backups.storeAppBackupMetadata(filename, metadata) // clear up the tarball after uploading it diff --git a/packages/server/src/sdk/app/statistics/index.ts b/packages/server/src/sdk/app/statistics/index.ts new file mode 100644 index 0000000000..3f03158264 --- /dev/null +++ b/packages/server/src/sdk/app/statistics/index.ts @@ -0,0 +1,77 @@ +import { context, db as dbCore } from "@budibase/backend-core" +import { + getDatasourceParams, + getTableParams, + getAutomationParams, + getScreenParams, +} from "../../../db/utils" + +async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { + if (db) { + return cb(db) + } else { + const devAppId = dbCore.getDevAppID(appId) + return context.doInAppContext(devAppId, () => { + const db = context.getAppDB() + return cb(db) + }) + } +} + +export async function calculateDatasourceCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const datasourceList = await db.allDocs(getDatasourceParams()) + const tableList = await db.allDocs(getTableParams()) + return datasourceList.rows.length + tableList.rows.length + }, + db + ) +} + +export async function calculateAutomationCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const automationList = await db.allDocs(getAutomationParams()) + return automationList.rows.length + }, + db + ) +} + +export async function calculateScreenCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const screenList = await db.allDocs(getScreenParams()) + return screenList.rows.length + }, + db + ) +} + +export async function calculateBackupStats(appId: string) { + return runInContext(appId, async (db: PouchDB.Database) => { + const promises = [] + promises.push(calculateDatasourceCount(appId, db)) + promises.push(calculateAutomationCount(appId, db)) + promises.push(calculateScreenCount(appId, db)) + const responses = await Promise.all(promises) + return { + datasources: responses[0], + automations: responses[1], + screens: responses[2], + } + }) +} diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index 28e927c772..a62b603a96 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -11,21 +11,23 @@ export enum AppBackupEventType { IMPORT = "import", } -export interface AppBackup extends Document { - trigger: AppBackupTrigger - name: string - createdAt: string - createdBy?: string - filename: string +export interface AppBackupMetadata { appId: string - userId?: string - contents?: { + trigger: AppBackupTrigger + name?: string + createdBy?: string + timestamp: string + contents: { datasources: string[] screens: string[] automations: string[] } } +export interface AppBackup extends Document, AppBackupMetadata { + filename: string +} + export type AppBackupFetchOpts = { trigger?: AppBackupTrigger limit?: number @@ -47,11 +49,3 @@ export interface AppBackupQueueData { backupId: string } } - -export interface AppBackupMetadata { - appId: string - trigger: AppBackupTrigger - name?: string - createdBy?: string - createdAt: string -}