Merge pull request #11185 from Budibase/budi-7113-self-host-console-log-export
Self host console log export
This commit is contained in:
commit
d858fa817e
17 changed files with 298 additions and 15 deletions
|
@ -28,3 +28,4 @@ BB_ADMIN_USER_PASSWORD=
|
|||
|
||||
# A path that is watched for plugin bundles. Any bundles found are imported automatically/
|
||||
PLUGINS_DIR=
|
||||
ROLLING_LOG_MAX_SIZE=
|
|
@ -51,6 +51,7 @@
|
|||
"pouchdb": "7.3.0",
|
||||
"pouchdb-find": "7.2.2",
|
||||
"redlock": "4.2.0",
|
||||
"rotating-file-stream": "3.1.0",
|
||||
"sanitize-s3-objectkey": "0.0.1",
|
||||
"semver": "7.3.7",
|
||||
"tar-fs": "2.1.1",
|
||||
|
|
|
@ -47,7 +47,10 @@ function httpLogging() {
|
|||
return process.env.HTTP_LOGGING
|
||||
}
|
||||
|
||||
function findVersion() {
|
||||
function getPackageJsonFields(): {
|
||||
VERSION: string
|
||||
SERVICE_NAME: string
|
||||
} {
|
||||
function findFileInAncestors(
|
||||
fileName: string,
|
||||
currentDir: string
|
||||
|
@ -69,10 +72,14 @@ function findVersion() {
|
|||
try {
|
||||
const packageJsonFile = findFileInAncestors("package.json", process.cwd())
|
||||
const content = readFileSync(packageJsonFile!, "utf-8")
|
||||
return JSON.parse(content).version
|
||||
const parsedContent = JSON.parse(content)
|
||||
return {
|
||||
VERSION: parsedContent.version,
|
||||
SERVICE_NAME: parsedContent.name,
|
||||
}
|
||||
} catch {
|
||||
// throwing an error here is confusing/causes backend-core to be hard to import
|
||||
return undefined
|
||||
return { VERSION: "", SERVICE_NAME: "" }
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -154,13 +161,14 @@ const environment = {
|
|||
ENABLE_SSO_MAINTENANCE_MODE: selfHosted
|
||||
? process.env.ENABLE_SSO_MAINTENANCE_MODE
|
||||
: false,
|
||||
VERSION: findVersion(),
|
||||
...getPackageJsonFields(),
|
||||
DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
|
||||
_set(key: any, value: any) {
|
||||
process.env[key] = value
|
||||
// @ts-ignore
|
||||
environment[key] = value
|
||||
},
|
||||
ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
|
||||
}
|
||||
|
||||
// clean up any environment variable edge cases
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
export * as correlation from "./correlation/correlation"
|
||||
export { logger } from "./pino/logger"
|
||||
export * from "./alerts"
|
||||
export * as system from "./system"
|
||||
|
||||
// turn off or on context logging i.e. tenantId, appId etc
|
||||
export let LOG_CONTEXT = true
|
||||
|
|
|
@ -1,10 +1,15 @@
|
|||
import env from "../../environment"
|
||||
import pino, { LoggerOptions } from "pino"
|
||||
import pinoPretty from "pino-pretty"
|
||||
|
||||
import { IdentityType } from "@budibase/types"
|
||||
|
||||
import env from "../../environment"
|
||||
import * as context from "../../context"
|
||||
import * as correlation from "../correlation"
|
||||
import { IdentityType } from "@budibase/types"
|
||||
import { LOG_CONTEXT } from "../index"
|
||||
|
||||
import { localFileDestination } from "../system"
|
||||
|
||||
// LOGGER
|
||||
|
||||
let pinoInstance: pino.Logger | undefined
|
||||
|
@ -16,22 +21,27 @@ if (!env.DISABLE_PINO_LOGGER) {
|
|||
return { level: label.toUpperCase() }
|
||||
},
|
||||
bindings: () => {
|
||||
return {}
|
||||
return {
|
||||
service: env.SERVICE_NAME,
|
||||
}
|
||||
},
|
||||
},
|
||||
timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
|
||||
}
|
||||
|
||||
const destinations: pino.DestinationStream[] = []
|
||||
|
||||
if (env.isDev()) {
|
||||
pinoOptions.transport = {
|
||||
target: "pino-pretty",
|
||||
options: {
|
||||
singleLine: true,
|
||||
},
|
||||
}
|
||||
destinations.push(pinoPretty({ singleLine: true }))
|
||||
}
|
||||
|
||||
pinoInstance = pino(pinoOptions)
|
||||
if (env.SELF_HOSTED) {
|
||||
destinations.push(localFileDestination())
|
||||
}
|
||||
|
||||
pinoInstance = destinations.length
|
||||
? pino(pinoOptions, pino.multistream(destinations))
|
||||
: pino(pinoOptions)
|
||||
|
||||
// CONSOLE OVERRIDES
|
||||
|
||||
|
|
81
packages/backend-core/src/logging/system.ts
Normal file
81
packages/backend-core/src/logging/system.ts
Normal file
|
@ -0,0 +1,81 @@
|
|||
import fs from "fs"
|
||||
import path from "path"
|
||||
import * as rfs from "rotating-file-stream"
|
||||
|
||||
import env from "../environment"
|
||||
import { budibaseTempDir } from "../objectStore"
|
||||
|
||||
const logsFileName = `budibase.log`
|
||||
const budibaseLogsHistoryFileName = "budibase-logs-history.txt"
|
||||
|
||||
const logsPath = path.join(budibaseTempDir(), "systemlogs")
|
||||
|
||||
function getFullPath(fileName: string) {
|
||||
return path.join(logsPath, fileName)
|
||||
}
|
||||
|
||||
export function getSingleFileMaxSizeInfo(totalMaxSize: string) {
|
||||
const regex = /(\d+)([A-Za-z])/
|
||||
const match = totalMaxSize?.match(regex)
|
||||
if (!match) {
|
||||
console.warn(`totalMaxSize does not have a valid value`, {
|
||||
totalMaxSize,
|
||||
})
|
||||
return undefined
|
||||
}
|
||||
|
||||
const size = +match[1]
|
||||
const unit = match[2]
|
||||
if (size === 1) {
|
||||
switch (unit) {
|
||||
case "B":
|
||||
return { size: `${size}B`, totalHistoryFiles: 1 }
|
||||
case "K":
|
||||
return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 }
|
||||
case "M":
|
||||
return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 }
|
||||
case "G":
|
||||
return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 }
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
if (size % 2 === 0) {
|
||||
return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 }
|
||||
}
|
||||
|
||||
return { size: `1${unit}`, totalHistoryFiles: size - 1 }
|
||||
}
|
||||
|
||||
export function localFileDestination() {
|
||||
const fileInfo = getSingleFileMaxSizeInfo(env.ROLLING_LOG_MAX_SIZE)
|
||||
const outFile = rfs.createStream(logsFileName, {
|
||||
// As we have a rolling size, we want to half the max size
|
||||
size: fileInfo?.size,
|
||||
path: logsPath,
|
||||
maxFiles: fileInfo?.totalHistoryFiles || 1,
|
||||
immutable: true,
|
||||
history: budibaseLogsHistoryFileName,
|
||||
initialRotation: false,
|
||||
})
|
||||
|
||||
return outFile
|
||||
}
|
||||
|
||||
export function getLogReadStream() {
|
||||
const streams = []
|
||||
const historyFile = getFullPath(budibaseLogsHistoryFileName)
|
||||
if (fs.existsSync(historyFile)) {
|
||||
const fileContent = fs.readFileSync(historyFile, "utf-8")
|
||||
const historyFiles = fileContent.split("\n")
|
||||
for (const historyFile of historyFiles.filter(x => x)) {
|
||||
streams.push(fs.readFileSync(historyFile))
|
||||
}
|
||||
}
|
||||
|
||||
streams.push(fs.readFileSync(getFullPath(logsFileName)))
|
||||
|
||||
const combinedContent = Buffer.concat(streams)
|
||||
return combinedContent
|
||||
}
|
61
packages/backend-core/src/logging/tests/system.spec.ts
Normal file
61
packages/backend-core/src/logging/tests/system.spec.ts
Normal file
|
@ -0,0 +1,61 @@
|
|||
import { getSingleFileMaxSizeInfo } from "../system"
|
||||
|
||||
describe("system", () => {
|
||||
describe("getSingleFileMaxSizeInfo", () => {
|
||||
it.each([
|
||||
["100B", "50B"],
|
||||
["200K", "100K"],
|
||||
["20M", "10M"],
|
||||
["4G", "2G"],
|
||||
])(
|
||||
"Halving even number (%s) returns halved size and 1 history file (%s)",
|
||||
(totalValue, expectedMaxSize) => {
|
||||
const result = getSingleFileMaxSizeInfo(totalValue)
|
||||
expect(result).toEqual({
|
||||
size: expectedMaxSize,
|
||||
totalHistoryFiles: 1,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
it.each([
|
||||
["5B", "1B", 4],
|
||||
["17K", "1K", 16],
|
||||
["21M", "1M", 20],
|
||||
["3G", "1G", 2],
|
||||
])(
|
||||
"Halving an odd number (%s) returns as many files as size (-1) (%s)",
|
||||
(totalValue, expectedMaxSize, totalHistoryFiles) => {
|
||||
const result = getSingleFileMaxSizeInfo(totalValue)
|
||||
expect(result).toEqual({
|
||||
size: expectedMaxSize,
|
||||
totalHistoryFiles,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
it.each([
|
||||
["1B", "1B"],
|
||||
["1K", "500B"],
|
||||
["1M", "500K"],
|
||||
["1G", "500M"],
|
||||
])(
|
||||
"Halving '%s' returns halved unit (%s)",
|
||||
(totalValue, expectedMaxSize) => {
|
||||
const result = getSingleFileMaxSizeInfo(totalValue)
|
||||
expect(result).toEqual({
|
||||
size: expectedMaxSize,
|
||||
totalHistoryFiles: 1,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
it.each([[undefined], [""], ["50"], ["wrongvalue"]])(
|
||||
"Halving wrongly formatted value ('%s') returns undefined",
|
||||
totalValue => {
|
||||
const result = getSingleFileMaxSizeInfo(totalValue!)
|
||||
expect(result).toBeUndefined()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
|
@ -0,0 +1,40 @@
|
|||
<script>
|
||||
import { Layout, Body, Button } from "@budibase/bbui"
|
||||
import { downloadStream } from "@budibase/frontend-core"
|
||||
import Spinner from "components/common/Spinner.svelte"
|
||||
|
||||
import { API } from "api"
|
||||
|
||||
let loading = false
|
||||
|
||||
async function download() {
|
||||
loading = true
|
||||
try {
|
||||
await downloadStream(await API.getSystemLogs())
|
||||
} finally {
|
||||
loading = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<Layout noPadding>
|
||||
<Body>Download your latest logs to share with the Budibase team</Body>
|
||||
<div class="download-button">
|
||||
<Button cta on:click={download} disabled={loading}>
|
||||
<div class="button-content">
|
||||
{#if loading}
|
||||
<Spinner size="10" />
|
||||
{/if}
|
||||
Download system logs
|
||||
</div>
|
||||
</Button>
|
||||
</div>
|
||||
</Layout>
|
||||
|
||||
<style>
|
||||
.button-content {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--spacing-m);
|
||||
}
|
||||
</style>
|
|
@ -85,6 +85,13 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
|
|||
title: "Audit Logs",
|
||||
href: "/builder/portal/account/auditLogs",
|
||||
})
|
||||
|
||||
if (!$admin.cloud) {
|
||||
accountSubPages.push({
|
||||
title: "System Logs",
|
||||
href: "/builder/portal/account/systemLogs",
|
||||
})
|
||||
}
|
||||
}
|
||||
if ($admin.cloud && $auth?.user?.accountPortalAccess) {
|
||||
accountSubPages.push({
|
||||
|
|
|
@ -30,6 +30,7 @@ import { buildBackupsEndpoints } from "./backups"
|
|||
import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
|
||||
import { buildEventEndpoints } from "./events"
|
||||
import { buildAuditLogsEndpoints } from "./auditLogs"
|
||||
import { buildLogsEndpoints } from "./logs"
|
||||
|
||||
/**
|
||||
* Random identifier to uniquely identify a session in a tab. This is
|
||||
|
@ -277,5 +278,6 @@ export const createAPIClient = config => {
|
|||
...buildEnvironmentVariableEndpoints(API),
|
||||
...buildEventEndpoints(API),
|
||||
...buildAuditLogsEndpoints(API),
|
||||
...buildLogsEndpoints(API),
|
||||
}
|
||||
}
|
||||
|
|
14
packages/frontend-core/src/api/logs.js
Normal file
14
packages/frontend-core/src/api/logs.js
Normal file
|
@ -0,0 +1,14 @@
|
|||
export const buildLogsEndpoints = API => ({
|
||||
/**
|
||||
* Gets a stream for the system logs.
|
||||
*/
|
||||
getSystemLogs: async () => {
|
||||
return await API.get({
|
||||
url: "/api/system/logs",
|
||||
json: false,
|
||||
parseResponse: async response => {
|
||||
return response
|
||||
},
|
||||
})
|
||||
},
|
||||
})
|
|
@ -11,3 +11,26 @@ export function downloadText(filename, text) {
|
|||
|
||||
URL.revokeObjectURL(url)
|
||||
}
|
||||
|
||||
export async function downloadStream(streamResponse) {
|
||||
const blob = await streamResponse.blob()
|
||||
|
||||
const contentDisposition = streamResponse.headers.get("Content-Disposition")
|
||||
|
||||
const matches = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/.exec(
|
||||
contentDisposition
|
||||
)
|
||||
|
||||
const filename = matches[1].replace(/['"]/g, "")
|
||||
|
||||
const resBlob = new Blob([blob])
|
||||
|
||||
const blobUrl = URL.createObjectURL(resBlob)
|
||||
|
||||
const link = document.createElement("a")
|
||||
link.href = blobUrl
|
||||
link.download = filename
|
||||
link.click()
|
||||
|
||||
URL.revokeObjectURL(blobUrl)
|
||||
}
|
||||
|
|
|
@ -5,4 +5,4 @@ export * as RoleUtils from "./roles"
|
|||
export * as Utils from "./utils"
|
||||
export { memo, derivedMemo } from "./memo"
|
||||
export { createWebsocket } from "./websocket"
|
||||
export { downloadText } from "./download"
|
||||
export * from "./download"
|
||||
|
|
13
packages/worker/src/api/controllers/system/logs.ts
Normal file
13
packages/worker/src/api/controllers/system/logs.ts
Normal file
|
@ -0,0 +1,13 @@
|
|||
import { UserCtx } from "@budibase/types"
|
||||
import { installation, logging } from "@budibase/backend-core"
|
||||
|
||||
export async function getLogs(ctx: UserCtx) {
|
||||
const logReadStream = logging.system.getLogReadStream()
|
||||
|
||||
const { installId } = await installation.getInstall()
|
||||
|
||||
const fileName = `${installId}-${Date.now()}.log`
|
||||
|
||||
ctx.set("content-disposition", `attachment; filename=${fileName}`)
|
||||
ctx.body = logReadStream
|
||||
}
|
|
@ -16,6 +16,9 @@ import licenseRoutes from "./global/license"
|
|||
import migrationRoutes from "./system/migrations"
|
||||
import accountRoutes from "./system/accounts"
|
||||
import restoreRoutes from "./system/restore"
|
||||
import systemLogRoutes from "./system/logs"
|
||||
|
||||
import env from "../../environment"
|
||||
|
||||
export const routes: Router[] = [
|
||||
configRoutes,
|
||||
|
@ -38,3 +41,7 @@ export const routes: Router[] = [
|
|||
eventRoutes,
|
||||
pro.scim,
|
||||
]
|
||||
|
||||
if (env.SELF_HOSTED) {
|
||||
routes.push(systemLogRoutes)
|
||||
}
|
||||
|
|
9
packages/worker/src/api/routes/system/logs.ts
Normal file
9
packages/worker/src/api/routes/system/logs.ts
Normal file
|
@ -0,0 +1,9 @@
|
|||
import Router from "@koa/router"
|
||||
import { middleware } from "@budibase/backend-core"
|
||||
import * as controller from "../../controllers/system/logs"
|
||||
|
||||
const router: Router = new Router()
|
||||
|
||||
router.get("/api/system/logs", middleware.adminOnly, controller.getLogs)
|
||||
|
||||
export default router
|
|
@ -23197,6 +23197,11 @@ rollup@^3.18.0:
|
|||
optionalDependencies:
|
||||
fsevents "~2.3.2"
|
||||
|
||||
rotating-file-stream@3.1.0:
|
||||
version "3.1.0"
|
||||
resolved "https://registry.yarnpkg.com/rotating-file-stream/-/rotating-file-stream-3.1.0.tgz#6cf50e1671de82a396de6d31d39a6f2445f45fba"
|
||||
integrity sha512-TkMF6cP1/QDcon9D71mjxHoflNuznNOrY5JJQfuxkKklZRmoow/lWBLNxXVjb6KcjAU8BDCV145buLgOx9Px1Q==
|
||||
|
||||
rrweb-cssom@^0.6.0:
|
||||
version "0.6.0"
|
||||
resolved "https://registry.yarnpkg.com/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz#ed298055b97cbddcdeb278f904857629dec5e0e1"
|
||||
|
|
Loading…
Reference in a new issue