diff --git a/hosting/.env b/hosting/.env index c2b6d55eef..8a0756c0e3 100644 --- a/hosting/.env +++ b/hosting/.env @@ -28,3 +28,4 @@ BB_ADMIN_USER_PASSWORD= # A path that is watched for plugin bundles. Any bundles found are imported automatically/ PLUGINS_DIR= +ROLLING_LOG_MAX_SIZE= \ No newline at end of file diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 4a1ed5c373..7f3c064c92 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -51,6 +51,7 @@ "pouchdb": "7.3.0", "pouchdb-find": "7.2.2", "redlock": "4.2.0", + "rotating-file-stream": "3.1.0", "sanitize-s3-objectkey": "0.0.1", "semver": "7.3.7", "tar-fs": "2.1.1", diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index eab8cd4c45..5076b7569b 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -47,7 +47,10 @@ function httpLogging() { return process.env.HTTP_LOGGING } -function findVersion() { +function getPackageJsonFields(): { + VERSION: string + SERVICE_NAME: string +} { function findFileInAncestors( fileName: string, currentDir: string @@ -69,10 +72,14 @@ function findVersion() { try { const packageJsonFile = findFileInAncestors("package.json", process.cwd()) const content = readFileSync(packageJsonFile!, "utf-8") - return JSON.parse(content).version + const parsedContent = JSON.parse(content) + return { + VERSION: parsedContent.version, + SERVICE_NAME: parsedContent.name, + } } catch { // throwing an error here is confusing/causes backend-core to be hard to import - return undefined + return { VERSION: "", SERVICE_NAME: "" } } } @@ -154,13 +161,14 @@ const environment = { ENABLE_SSO_MAINTENANCE_MODE: selfHosted ? process.env.ENABLE_SSO_MAINTENANCE_MODE : false, - VERSION: findVersion(), + ...getPackageJsonFields(), DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER, _set(key: any, value: any) { process.env[key] = value // @ts-ignore environment[key] = value }, + ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M", } // clean up any environment variable edge cases diff --git a/packages/backend-core/src/logging/index.ts b/packages/backend-core/src/logging/index.ts index b87062c478..f7e1c4fa41 100644 --- a/packages/backend-core/src/logging/index.ts +++ b/packages/backend-core/src/logging/index.ts @@ -1,6 +1,7 @@ export * as correlation from "./correlation/correlation" export { logger } from "./pino/logger" export * from "./alerts" +export * as system from "./system" // turn off or on context logging i.e. tenantId, appId etc export let LOG_CONTEXT = true diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index c96bc83e04..0b130068f3 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -1,10 +1,15 @@ -import env from "../../environment" import pino, { LoggerOptions } from "pino" +import pinoPretty from "pino-pretty" + +import { IdentityType } from "@budibase/types" + +import env from "../../environment" import * as context from "../../context" import * as correlation from "../correlation" -import { IdentityType } from "@budibase/types" import { LOG_CONTEXT } from "../index" +import { localFileDestination } from "../system" + // LOGGER let pinoInstance: pino.Logger | undefined @@ -16,22 +21,27 @@ if (!env.DISABLE_PINO_LOGGER) { return { level: label.toUpperCase() } }, bindings: () => { - return {} + return { + service: env.SERVICE_NAME, + } }, }, timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`, } + const destinations: pino.DestinationStream[] = [] + if (env.isDev()) { - pinoOptions.transport = { - target: "pino-pretty", - options: { - singleLine: true, - }, - } + destinations.push(pinoPretty({ singleLine: true })) } - pinoInstance = pino(pinoOptions) + if (env.SELF_HOSTED) { + destinations.push(localFileDestination()) + } + + pinoInstance = destinations.length + ? pino(pinoOptions, pino.multistream(destinations)) + : pino(pinoOptions) // CONSOLE OVERRIDES diff --git a/packages/backend-core/src/logging/system.ts b/packages/backend-core/src/logging/system.ts new file mode 100644 index 0000000000..d918c6efd6 --- /dev/null +++ b/packages/backend-core/src/logging/system.ts @@ -0,0 +1,81 @@ +import fs from "fs" +import path from "path" +import * as rfs from "rotating-file-stream" + +import env from "../environment" +import { budibaseTempDir } from "../objectStore" + +const logsFileName = `budibase.log` +const budibaseLogsHistoryFileName = "budibase-logs-history.txt" + +const logsPath = path.join(budibaseTempDir(), "systemlogs") + +function getFullPath(fileName: string) { + return path.join(logsPath, fileName) +} + +export function getSingleFileMaxSizeInfo(totalMaxSize: string) { + const regex = /(\d+)([A-Za-z])/ + const match = totalMaxSize?.match(regex) + if (!match) { + console.warn(`totalMaxSize does not have a valid value`, { + totalMaxSize, + }) + return undefined + } + + const size = +match[1] + const unit = match[2] + if (size === 1) { + switch (unit) { + case "B": + return { size: `${size}B`, totalHistoryFiles: 1 } + case "K": + return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 } + case "M": + return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 } + case "G": + return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 } + default: + return undefined + } + } + + if (size % 2 === 0) { + return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 } + } + + return { size: `1${unit}`, totalHistoryFiles: size - 1 } +} + +export function localFileDestination() { + const fileInfo = getSingleFileMaxSizeInfo(env.ROLLING_LOG_MAX_SIZE) + const outFile = rfs.createStream(logsFileName, { + // As we have a rolling size, we want to half the max size + size: fileInfo?.size, + path: logsPath, + maxFiles: fileInfo?.totalHistoryFiles || 1, + immutable: true, + history: budibaseLogsHistoryFileName, + initialRotation: false, + }) + + return outFile +} + +export function getLogReadStream() { + const streams = [] + const historyFile = getFullPath(budibaseLogsHistoryFileName) + if (fs.existsSync(historyFile)) { + const fileContent = fs.readFileSync(historyFile, "utf-8") + const historyFiles = fileContent.split("\n") + for (const historyFile of historyFiles.filter(x => x)) { + streams.push(fs.readFileSync(historyFile)) + } + } + + streams.push(fs.readFileSync(getFullPath(logsFileName))) + + const combinedContent = Buffer.concat(streams) + return combinedContent +} diff --git a/packages/backend-core/src/logging/tests/system.spec.ts b/packages/backend-core/src/logging/tests/system.spec.ts new file mode 100644 index 0000000000..b84d8e8456 --- /dev/null +++ b/packages/backend-core/src/logging/tests/system.spec.ts @@ -0,0 +1,61 @@ +import { getSingleFileMaxSizeInfo } from "../system" + +describe("system", () => { + describe("getSingleFileMaxSizeInfo", () => { + it.each([ + ["100B", "50B"], + ["200K", "100K"], + ["20M", "10M"], + ["4G", "2G"], + ])( + "Halving even number (%s) returns halved size and 1 history file (%s)", + (totalValue, expectedMaxSize) => { + const result = getSingleFileMaxSizeInfo(totalValue) + expect(result).toEqual({ + size: expectedMaxSize, + totalHistoryFiles: 1, + }) + } + ) + + it.each([ + ["5B", "1B", 4], + ["17K", "1K", 16], + ["21M", "1M", 20], + ["3G", "1G", 2], + ])( + "Halving an odd number (%s) returns as many files as size (-1) (%s)", + (totalValue, expectedMaxSize, totalHistoryFiles) => { + const result = getSingleFileMaxSizeInfo(totalValue) + expect(result).toEqual({ + size: expectedMaxSize, + totalHistoryFiles, + }) + } + ) + + it.each([ + ["1B", "1B"], + ["1K", "500B"], + ["1M", "500K"], + ["1G", "500M"], + ])( + "Halving '%s' returns halved unit (%s)", + (totalValue, expectedMaxSize) => { + const result = getSingleFileMaxSizeInfo(totalValue) + expect(result).toEqual({ + size: expectedMaxSize, + totalHistoryFiles: 1, + }) + } + ) + + it.each([[undefined], [""], ["50"], ["wrongvalue"]])( + "Halving wrongly formatted value ('%s') returns undefined", + totalValue => { + const result = getSingleFileMaxSizeInfo(totalValue!) + expect(result).toBeUndefined() + } + ) + }) +}) diff --git a/packages/builder/src/pages/builder/portal/account/systemLogs/index.svelte b/packages/builder/src/pages/builder/portal/account/systemLogs/index.svelte new file mode 100644 index 0000000000..8643a65f17 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/systemLogs/index.svelte @@ -0,0 +1,40 @@ + + + + Download your latest logs to share with the Budibase team +
+ +
+ + + diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js index 40580cd0ec..04151f66d4 100644 --- a/packages/builder/src/stores/portal/menu.js +++ b/packages/builder/src/stores/portal/menu.js @@ -85,6 +85,13 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { title: "Audit Logs", href: "/builder/portal/account/auditLogs", }) + + if (!$admin.cloud) { + accountSubPages.push({ + title: "System Logs", + href: "/builder/portal/account/systemLogs", + }) + } } if ($admin.cloud && $auth?.user?.accountPortalAccess) { accountSubPages.push({ diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index 7b823d28c2..5bffb82f4d 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -30,6 +30,7 @@ import { buildBackupsEndpoints } from "./backups" import { buildEnvironmentVariableEndpoints } from "./environmentVariables" import { buildEventEndpoints } from "./events" import { buildAuditLogsEndpoints } from "./auditLogs" +import { buildLogsEndpoints } from "./logs" /** * Random identifier to uniquely identify a session in a tab. This is @@ -277,5 +278,6 @@ export const createAPIClient = config => { ...buildEnvironmentVariableEndpoints(API), ...buildEventEndpoints(API), ...buildAuditLogsEndpoints(API), + ...buildLogsEndpoints(API), } } diff --git a/packages/frontend-core/src/api/logs.js b/packages/frontend-core/src/api/logs.js new file mode 100644 index 0000000000..b6cb98627c --- /dev/null +++ b/packages/frontend-core/src/api/logs.js @@ -0,0 +1,14 @@ +export const buildLogsEndpoints = API => ({ + /** + * Gets a stream for the system logs. + */ + getSystemLogs: async () => { + return await API.get({ + url: "/api/system/logs", + json: false, + parseResponse: async response => { + return response + }, + }) + }, +}) diff --git a/packages/frontend-core/src/utils/download.js b/packages/frontend-core/src/utils/download.js index 681bc8648e..89c8572253 100644 --- a/packages/frontend-core/src/utils/download.js +++ b/packages/frontend-core/src/utils/download.js @@ -11,3 +11,26 @@ export function downloadText(filename, text) { URL.revokeObjectURL(url) } + +export async function downloadStream(streamResponse) { + const blob = await streamResponse.blob() + + const contentDisposition = streamResponse.headers.get("Content-Disposition") + + const matches = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/.exec( + contentDisposition + ) + + const filename = matches[1].replace(/['"]/g, "") + + const resBlob = new Blob([blob]) + + const blobUrl = URL.createObjectURL(resBlob) + + const link = document.createElement("a") + link.href = blobUrl + link.download = filename + link.click() + + URL.revokeObjectURL(blobUrl) +} diff --git a/packages/frontend-core/src/utils/index.js b/packages/frontend-core/src/utils/index.js index dd04dd6c28..3f00c00e47 100644 --- a/packages/frontend-core/src/utils/index.js +++ b/packages/frontend-core/src/utils/index.js @@ -5,4 +5,4 @@ export * as RoleUtils from "./roles" export * as Utils from "./utils" export { memo, derivedMemo } from "./memo" export { createWebsocket } from "./websocket" -export { downloadText } from "./download" +export * from "./download" diff --git a/packages/worker/src/api/controllers/system/logs.ts b/packages/worker/src/api/controllers/system/logs.ts new file mode 100644 index 0000000000..a5607d545a --- /dev/null +++ b/packages/worker/src/api/controllers/system/logs.ts @@ -0,0 +1,13 @@ +import { UserCtx } from "@budibase/types" +import { installation, logging } from "@budibase/backend-core" + +export async function getLogs(ctx: UserCtx) { + const logReadStream = logging.system.getLogReadStream() + + const { installId } = await installation.getInstall() + + const fileName = `${installId}-${Date.now()}.log` + + ctx.set("content-disposition", `attachment; filename=${fileName}`) + ctx.body = logReadStream +} diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index 4131f14c74..cbd6c96558 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -16,6 +16,9 @@ import licenseRoutes from "./global/license" import migrationRoutes from "./system/migrations" import accountRoutes from "./system/accounts" import restoreRoutes from "./system/restore" +import systemLogRoutes from "./system/logs" + +import env from "../../environment" export const routes: Router[] = [ configRoutes, @@ -38,3 +41,7 @@ export const routes: Router[] = [ eventRoutes, pro.scim, ] + +if (env.SELF_HOSTED) { + routes.push(systemLogRoutes) +} diff --git a/packages/worker/src/api/routes/system/logs.ts b/packages/worker/src/api/routes/system/logs.ts new file mode 100644 index 0000000000..dcb95b5c8a --- /dev/null +++ b/packages/worker/src/api/routes/system/logs.ts @@ -0,0 +1,9 @@ +import Router from "@koa/router" +import { middleware } from "@budibase/backend-core" +import * as controller from "../../controllers/system/logs" + +const router: Router = new Router() + +router.get("/api/system/logs", middleware.adminOnly, controller.getLogs) + +export default router diff --git a/yarn.lock b/yarn.lock index 9537bb81fa..c6c6235d02 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23197,6 +23197,11 @@ rollup@^3.18.0: optionalDependencies: fsevents "~2.3.2" +rotating-file-stream@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/rotating-file-stream/-/rotating-file-stream-3.1.0.tgz#6cf50e1671de82a396de6d31d39a6f2445f45fba" + integrity sha512-TkMF6cP1/QDcon9D71mjxHoflNuznNOrY5JJQfuxkKklZRmoow/lWBLNxXVjb6KcjAU8BDCV145buLgOx9Px1Q== + rrweb-cssom@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz#ed298055b97cbddcdeb278f904857629dec5e0e1"