Merge pull request #11185 from Budibase/budi-7113-self-host-console-log-export

Self host console log export

hosting/.env

@@ -28,3 +28,4 @@ BB_ADMIN_USER_PASSWORD=
 
 # A path that is watched for plugin bundles. Any bundles found are imported automatically/
 PLUGINS_DIR=
+ROLLING_LOG_MAX_SIZE=

packages/backend-core/package.json

@@ -51,6 +51,7 @@
     "pouchdb": "7.3.0",
     "pouchdb-find": "7.2.2",
     "redlock": "4.2.0",
+    "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "7.3.7",
     "tar-fs": "2.1.1",

packages/backend-core/src/environment.ts

@@ -47,7 +47,10 @@ function httpLogging() {
   return process.env.HTTP_LOGGING
 }
 
-function findVersion() {
+function getPackageJsonFields(): {
+  VERSION: string
+  SERVICE_NAME: string
+} {
   function findFileInAncestors(
     fileName: string,
     currentDir: string
@@ -69,10 +72,14 @@ function findVersion() {
   try {
     const packageJsonFile = findFileInAncestors("package.json", process.cwd())
     const content = readFileSync(packageJsonFile!, "utf-8")
-    return JSON.parse(content).version
+    const parsedContent = JSON.parse(content)
+    return {
+      VERSION: parsedContent.version,
+      SERVICE_NAME: parsedContent.name,
+    }
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return undefined
+    return { VERSION: "", SERVICE_NAME: "" }
   }
 }
 
@@ -154,13 +161,14 @@ const environment = {
   ENABLE_SSO_MAINTENANCE_MODE: selfHosted
     ? process.env.ENABLE_SSO_MAINTENANCE_MODE
     : false,
-  VERSION: findVersion(),
+  ...getPackageJsonFields(),
   DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore
     environment[key] = value
   },
+  ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
 }
 
 // clean up any environment variable edge cases

packages/backend-core/src/logging/index.ts

@@ -1,6 +1,7 @@
 export * as correlation from "./correlation/correlation"
 export { logger } from "./pino/logger"
 export * from "./alerts"
+export * as system from "./system"
 
 // turn off or on context logging i.e. tenantId, appId etc
 export let LOG_CONTEXT = true

packages/backend-core/src/logging/pino/logger.ts

@@ -1,10 +1,15 @@
-import env from "../../environment"
 import pino, { LoggerOptions } from "pino"
+import pinoPretty from "pino-pretty"
+
+import { IdentityType } from "@budibase/types"
+
+import env from "../../environment"
 import * as context from "../../context"
 import * as correlation from "../correlation"
-import { IdentityType } from "@budibase/types"
 import { LOG_CONTEXT } from "../index"
 
+import { localFileDestination } from "../system"
+
 // LOGGER
 
 let pinoInstance: pino.Logger | undefined
@@ -16,22 +21,27 @@ if (!env.DISABLE_PINO_LOGGER) {
         return { level: label.toUpperCase() }
       },
       bindings: () => {
-        return {}
+        return {
+          service: env.SERVICE_NAME,
+        }
       },
     },
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   }
 
+  const destinations: pino.DestinationStream[] = []
+
   if (env.isDev()) {
-    pinoOptions.transport = {
-      target: "pino-pretty",
-      options: {
-        singleLine: true,
-      },
-    }
+    destinations.push(pinoPretty({ singleLine: true }))
   }
 
-  pinoInstance = pino(pinoOptions)
+  if (env.SELF_HOSTED) {
+    destinations.push(localFileDestination())
+  }
+
+  pinoInstance = destinations.length
+    ? pino(pinoOptions, pino.multistream(destinations))
+    : pino(pinoOptions)
 
   // CONSOLE OVERRIDES
 

packages/backend-core/src/logging/system.ts

@@ -0,0 +1,81 @@
+import fs from "fs"
+import path from "path"
+import * as rfs from "rotating-file-stream"
+
+import env from "../environment"
+import { budibaseTempDir } from "../objectStore"
+
+const logsFileName = `budibase.log`
+const budibaseLogsHistoryFileName = "budibase-logs-history.txt"
+
+const logsPath = path.join(budibaseTempDir(), "systemlogs")
+
+function getFullPath(fileName: string) {
+  return path.join(logsPath, fileName)
+}
+
+export function getSingleFileMaxSizeInfo(totalMaxSize: string) {
+  const regex = /(\d+)([A-Za-z])/
+  const match = totalMaxSize?.match(regex)
+  if (!match) {
+    console.warn(`totalMaxSize does not have a valid value`, {
+      totalMaxSize,
+    })
+    return undefined
+  }
+
+  const size = +match[1]
+  const unit = match[2]
+  if (size === 1) {
+    switch (unit) {
+      case "B":
+        return { size: `${size}B`, totalHistoryFiles: 1 }
+      case "K":
+        return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 }
+      case "M":
+        return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 }
+      case "G":
+        return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 }
+      default:
+        return undefined
+    }
+  }
+
+  if (size % 2 === 0) {
+    return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 }
+  }
+
+  return { size: `1${unit}`, totalHistoryFiles: size - 1 }
+}
+
+export function localFileDestination() {
+  const fileInfo = getSingleFileMaxSizeInfo(env.ROLLING_LOG_MAX_SIZE)
+  const outFile = rfs.createStream(logsFileName, {
+    // As we have a rolling size, we want to half the max size
+    size: fileInfo?.size,
+    path: logsPath,
+    maxFiles: fileInfo?.totalHistoryFiles || 1,
+    immutable: true,
+    history: budibaseLogsHistoryFileName,
+    initialRotation: false,
+  })
+
+  return outFile
+}
+
+export function getLogReadStream() {
+  const streams = []
+  const historyFile = getFullPath(budibaseLogsHistoryFileName)
+  if (fs.existsSync(historyFile)) {
+    const fileContent = fs.readFileSync(historyFile, "utf-8")
+    const historyFiles = fileContent.split("\n")
+    for (const historyFile of historyFiles.filter(x => x)) {
+      streams.push(fs.readFileSync(historyFile))
+    }
+  }
+
+  streams.push(fs.readFileSync(getFullPath(logsFileName)))
+
+  const combinedContent = Buffer.concat(streams)
+  return combinedContent
+}

packages/backend-core/src/logging/tests/system.spec.ts

@@ -0,0 +1,61 @@
+import { getSingleFileMaxSizeInfo } from "../system"
+
+describe("system", () => {
+  describe("getSingleFileMaxSizeInfo", () => {
+    it.each([
+      ["100B", "50B"],
+      ["200K", "100K"],
+      ["20M", "10M"],
+      ["4G", "2G"],
+    ])(
+      "Halving even number (%s) returns halved size and 1 history file (%s)",
+      (totalValue, expectedMaxSize) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles: 1,
+        })
+      }
+    )
+
+    it.each([
+      ["5B", "1B", 4],
+      ["17K", "1K", 16],
+      ["21M", "1M", 20],
+      ["3G", "1G", 2],
+    ])(
+      "Halving an odd number (%s) returns as many files as size (-1) (%s)",
+      (totalValue, expectedMaxSize, totalHistoryFiles) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles,
+        })
+      }
+    )
+
+    it.each([
+      ["1B", "1B"],
+      ["1K", "500B"],
+      ["1M", "500K"],
+      ["1G", "500M"],
+    ])(
+      "Halving '%s' returns halved unit (%s)",
+      (totalValue, expectedMaxSize) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles: 1,
+        })
+      }
+    )
+
+    it.each([[undefined], [""], ["50"], ["wrongvalue"]])(
+      "Halving wrongly formatted value ('%s') returns undefined",
+      totalValue => {
+        const result = getSingleFileMaxSizeInfo(totalValue!)
+        expect(result).toBeUndefined()
+      }
+    )
+  })
+})

packages/builder/src/pages/builder/portal/account/systemLogs/index.svelte

@@ -0,0 +1,40 @@
+<script>
+  import { Layout, Body, Button } from "@budibase/bbui"
+  import { downloadStream } from "@budibase/frontend-core"
+  import Spinner from "components/common/Spinner.svelte"
+
+  import { API } from "api"
+
+  let loading = false
+
+  async function download() {
+    loading = true
+    try {
+      await downloadStream(await API.getSystemLogs())
+    } finally {
+      loading = false
+    }
+  }
+</script>
+
+<Layout noPadding>
+  <Body>Download your latest logs to share with the Budibase team</Body>
+  <div class="download-button">
+    <Button cta on:click={download} disabled={loading}>
+      <div class="button-content">
+        {#if loading}
+          <Spinner size="10" />
+        {/if}
+        Download system logs
+      </div>
+    </Button>
+  </div>
+</Layout>
+
+<style>
+  .button-content {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-m);
+  }
+</style>

packages/builder/src/stores/portal/menu.js

@@ -85,6 +85,13 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         title: "Audit Logs",
         href: "/builder/portal/account/auditLogs",
       })
+
+      if (!$admin.cloud) {
+        accountSubPages.push({
+          title: "System Logs",
+          href: "/builder/portal/account/systemLogs",
+        })
+      }
     }
     if ($admin.cloud && $auth?.user?.accountPortalAccess) {
       accountSubPages.push({

packages/frontend-core/src/api/index.js

@@ -30,6 +30,7 @@ import { buildBackupsEndpoints } from "./backups"
 import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
 import { buildEventEndpoints } from "./events"
 import { buildAuditLogsEndpoints } from "./auditLogs"
+import { buildLogsEndpoints } from "./logs"
 
 /**
  * Random identifier to uniquely identify a session in a tab. This is
@@ -277,5 +278,6 @@ export const createAPIClient = config => {
     ...buildEnvironmentVariableEndpoints(API),
     ...buildEventEndpoints(API),
     ...buildAuditLogsEndpoints(API),
+    ...buildLogsEndpoints(API),
   }
 }

packages/frontend-core/src/api/logs.js

@@ -0,0 +1,14 @@
+export const buildLogsEndpoints = API => ({
+  /**
+   * Gets a stream for the system logs.
+   */
+  getSystemLogs: async () => {
+    return await API.get({
+      url: "/api/system/logs",
+      json: false,
+      parseResponse: async response => {
+        return response
+      },
+    })
+  },
+})

packages/frontend-core/src/utils/download.js

@@ -11,3 +11,26 @@ export function downloadText(filename, text) {
 
   URL.revokeObjectURL(url)
 }
+
+export async function downloadStream(streamResponse) {
+  const blob = await streamResponse.blob()
+
+  const contentDisposition = streamResponse.headers.get("Content-Disposition")
+
+  const matches = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/.exec(
+    contentDisposition
+  )
+
+  const filename = matches[1].replace(/['"]/g, "")
+
+  const resBlob = new Blob([blob])
+
+  const blobUrl = URL.createObjectURL(resBlob)
+
+  const link = document.createElement("a")
+  link.href = blobUrl
+  link.download = filename
+  link.click()
+
+  URL.revokeObjectURL(blobUrl)
+}

packages/frontend-core/src/utils/index.js

@@ -5,4 +5,4 @@ export * as RoleUtils from "./roles"
 export * as Utils from "./utils"
 export { memo, derivedMemo } from "./memo"
 export { createWebsocket } from "./websocket"
-export { downloadText } from "./download"
+export * from "./download"

packages/worker/src/api/controllers/system/logs.ts

@@ -0,0 +1,13 @@
+import { UserCtx } from "@budibase/types"
+import { installation, logging } from "@budibase/backend-core"
+
+export async function getLogs(ctx: UserCtx) {
+  const logReadStream = logging.system.getLogReadStream()
+
+  const { installId } = await installation.getInstall()
+
+  const fileName = `${installId}-${Date.now()}.log`
+
+  ctx.set("content-disposition", `attachment; filename=${fileName}`)
+  ctx.body = logReadStream
+}

packages/worker/src/api/routes/index.ts

@@ -16,6 +16,9 @@ import licenseRoutes from "./global/license"
 import migrationRoutes from "./system/migrations"
 import accountRoutes from "./system/accounts"
 import restoreRoutes from "./system/restore"
+import systemLogRoutes from "./system/logs"
+
+import env from "../../environment"
 
 export const routes: Router[] = [
   configRoutes,
@@ -38,3 +41,7 @@ export const routes: Router[] = [
   eventRoutes,
   pro.scim,
 ]
+
+if (env.SELF_HOSTED) {
+  routes.push(systemLogRoutes)
+}

packages/worker/src/api/routes/system/logs.ts

@@ -0,0 +1,9 @@
+import Router from "@koa/router"
+import { middleware } from "@budibase/backend-core"
+import * as controller from "../../controllers/system/logs"
+
+const router: Router = new Router()
+
+router.get("/api/system/logs", middleware.adminOnly, controller.getLogs)
+
+export default router

yarn.lock

@@ -23197,6 +23197,11 @@ rollup@^3.18.0:
   optionalDependencies:
     fsevents "~2.3.2"
 
+rotating-file-stream@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/rotating-file-stream/-/rotating-file-stream-3.1.0.tgz#6cf50e1671de82a396de6d31d39a6f2445f45fba"
+  integrity sha512-TkMF6cP1/QDcon9D71mjxHoflNuznNOrY5JJQfuxkKklZRmoow/lWBLNxXVjb6KcjAU8BDCV145buLgOx9Px1Q==
+
 rrweb-cssom@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz#ed298055b97cbddcdeb278f904857629dec5e0e1"