From a908125ea3716e17795e3d4a2f1e6ff771d36399 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 20 Feb 2023 19:56:04 +0000 Subject: [PATCH 1/6] add metrics endpoint to api --- .../src/api/controllers/public/metrics.ts | 122 ++++++++++++++++++ .../server/src/api/routes/public/index.ts | 2 + .../server/src/api/routes/public/metrics.ts | 28 ++++ 3 files changed, 152 insertions(+) create mode 100644 packages/server/src/api/controllers/public/metrics.ts create mode 100644 packages/server/src/api/routes/public/metrics.ts diff --git a/packages/server/src/api/controllers/public/metrics.ts b/packages/server/src/api/controllers/public/metrics.ts new file mode 100644 index 0000000000..264df7f415 --- /dev/null +++ b/packages/server/src/api/controllers/public/metrics.ts @@ -0,0 +1,122 @@ +import { App, BBContext } from "@budibase/types" +import { db as dbCore, context } from "@budibase/backend-core" +import os from "os" + +export async function fetch(ctx: BBContext) { + const allDatabases = await dbCore.getAllDbs() + const devAppIDs = await dbCore.getDevAppIDs({ idsOnly: true }) + const prodAppIDs = await dbCore.getProdAppIDs({ idsOnly: true }) + const allAppIds = await dbCore.getAllApps({ idsOnly: true }) + + var outputString = "" + + const freeMem = os.freemem() + const totalMem = os.totalmem() + const usedMem = totalMem - freeMem + const uptime = os.uptime() + + // **** budibase_os_uptime **** + outputString += convertToOpenMetrics( + "budibase_os_uptime", + "Time in seconds that the host operating system has been up", + "counter", + uptime + ) + + // **** budibase_os_free_mem **** + outputString += convertToOpenMetrics( + "budibase_os_free_mem", + "Bytes of memory free for usage on the host operating system", + "gauge", + freeMem + ) + + // **** budibase_os_total_mem **** + outputString += convertToOpenMetrics( + "budibase_os_total_mem", + "Total bytes of memory on the host operating system", + "gauge", + totalMem + ) + + // **** budibase_os_used_mem **** + outputString += convertToOpenMetrics( + "budibase_os_used_mem", + "Total bytes of memory in use on the host operating system", + "gauge", + usedMem + ) + + // **** budibase_os_load1 **** + outputString += convertToOpenMetrics( + "budibase_os_load1", + "Host operating system load average", + "gauge", + os.loadavg()[0] + ) + + // **** budibase_os_load5 **** + outputString += convertToOpenMetrics( + "budibase_os_load5", + "Host operating system load average", + "gauge", + os.loadavg()[1] + ) + // **** budibase_os_load15 **** + outputString += convertToOpenMetrics( + "budibase_os_load15", + "Host operating system load average", + "gauge", + os.loadavg()[2] + ) + + // **** budibase_tenant_app_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_app_count", + "The number of apps created by a user", + "gauge", + allAppIds.length + ) + + // **** budibase_tenant_production_app_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_production_app_count", + "The number of apps a user has published", + "gauge", + prodAppIDs.length + ) + + // **** budibase_tenant_dev_app_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_dev_app_count", + "The number of apps a user has unpublished in development", + "gauge", + devAppIDs.length + ) + + // **** budibase_tenant_db_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_db_count", + "The number of couchdb databases including global tables such as _users", + "gauge", + allDatabases.length + ) + + ctx.body = outputString +} + +export function convertToOpenMetrics( + metricName: string, + metricHelp: string, + metricType: string, + metricValue: number +) { + return ` + # HELP ${metricName} ${metricHelp}. + # TYPE ${metricName} ${metricType} + ${metricName} ${metricValue}` +} + +export default { + fetch, +} diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 79e7731752..e67c10fe02 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -1,4 +1,5 @@ import appEndpoints from "./applications" +import metricEndpoints from "./metrics" import queryEndpoints from "./queries" import tableEndpoints from "./tables" import rowEndpoints from "./rows" @@ -120,6 +121,7 @@ function applyRoutes( } applyRoutes(appEndpoints, PermissionType.APP, "appId") +applyRoutes(metricEndpoints, PermissionType.APP, "appId") applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId") applyRoutes(userEndpoints, PermissionType.USER, "userId") applyRoutes(queryEndpoints, PermissionType.QUERY, "queryId") diff --git a/packages/server/src/api/routes/public/metrics.ts b/packages/server/src/api/routes/public/metrics.ts new file mode 100644 index 0000000000..67cae14388 --- /dev/null +++ b/packages/server/src/api/routes/public/metrics.ts @@ -0,0 +1,28 @@ +import controller from "../../controllers/public/metrics" +import Endpoint from "./utils/Endpoint" + +const read = [] + +/** + * @openapi + * /metrics: + * get: + * operationId: getById + * summary: Retrieve Budibase tenant metrics + * description: Output metrics in openMetrics format compatible with Prometheus + * tags: + * - metrics + * responses: + * 200: + * description: Returns tenant metrics. + * content: + * text/plain: + * schema: + * $ref: '#/components/schemas/tableOutput' + * examples: + * table: + * $ref: '#/components/examples/table' + */ +read.push(new Endpoint("get", "/metrics", controller.fetch)) + +export default { read } From dde1237ed220e47202d7a29f5b92b8c7c15b4698 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Wed, 22 Feb 2023 21:10:35 +0000 Subject: [PATCH 2/6] allUsers function & more metrics --- packages/backend-core/src/users.ts | 12 ++ .../src/api/controllers/public/metrics.ts | 147 ++++++++++++++++-- .../server/src/api/routes/public/metrics.ts | 7 +- 3 files changed, 150 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 1720a79a83..7502ece47a 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -5,6 +5,8 @@ import { generateAppUserID, queryGlobalView, UNICODE_MAX, + DocumentType, + SEPARATOR, } from "./db" import { BulkDocsResponse, User } from "@budibase/types" import { getGlobalDB } from "./context" @@ -19,6 +21,16 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => { ).rows.map(row => row.doc) as User[] } +export const getAllUserIds = async () => { + const db = getGlobalDB() + const startKey = `${DocumentType.USER}${SEPARATOR}` + const response = await db.allDocs({ + startkey: startKey, + endkey: `${startKey}${UNICODE_MAX}`, + }) + return response.rows.map(row => row.id) +} + export const bulkUpdateGlobalUsers = async (users: User[]) => { const db = getGlobalDB() return (await db.bulkDocs(users)) as BulkDocsResponse diff --git a/packages/server/src/api/controllers/public/metrics.ts b/packages/server/src/api/controllers/public/metrics.ts index 264df7f415..65f8849f7f 100644 --- a/packages/server/src/api/controllers/public/metrics.ts +++ b/packages/server/src/api/controllers/public/metrics.ts @@ -1,19 +1,43 @@ import { App, BBContext } from "@budibase/types" -import { db as dbCore, context } from "@budibase/backend-core" +import { users as userCore, db as dbCore } from "@budibase/backend-core" +import { quotas, licensing } from "@budibase/pro" + import os from "os" export async function fetch(ctx: BBContext) { + // *** OPERATING SYSTEM *** + const freeMem = os.freemem() + const totalMem = os.totalmem() + const usedMem = totalMem - freeMem + const uptime = os.uptime() + + // *** APPS *** const allDatabases = await dbCore.getAllDbs() const devAppIDs = await dbCore.getDevAppIDs({ idsOnly: true }) const prodAppIDs = await dbCore.getProdAppIDs({ idsOnly: true }) const allAppIds = await dbCore.getAllApps({ idsOnly: true }) - var outputString = "" + // *** USERS *** + const usersObject = await userCore.getAllUserIds() - const freeMem = os.freemem() - const totalMem = os.totalmem() - const usedMem = totalMem - freeMem - const uptime = os.uptime() + // *** QUOTAS *** + const usage = await quotas.getQuotaUsage() + const license = await licensing.cache.getCachedLicense() + const appsQuotaUsage = usage.usageQuota.apps + const rowsQuotaUsage = usage.usageQuota.rows + const pluginsQuotaUsage = usage.usageQuota.plugins + const userGroupsQuotaUsage = usage.usageQuota.userGroups + const queryQuotaUsage = usage.monthly.current.queries + const automationsQuotaUsage = usage.monthly.current.automations + const appsQuotaLimit = license.quotas.usage.static.apps.value + const rowsQuotaLimit = license.quotas.usage.static.rows.value + const userGroupsQuotaLimit = license.quotas.usage.static.userGroups.value + const pluginsQuotaLimit = license.quotas.usage.static.plugins.value + const queryQuotaLimit = license.quotas.usage.monthly.queries.value + const automationsQuotaLimit = license.quotas.usage.monthly.automations.value + + // *** BUILD THE OUTPUT STRING *** + var outputString = "" // **** budibase_os_uptime **** outputString += convertToOpenMetrics( @@ -70,6 +94,14 @@ export async function fetch(ctx: BBContext) { os.loadavg()[2] ) + // **** budibase_tenant_user_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_user_count", + "The number of users created", + "gauge", + usersObject.length + ) + // **** budibase_tenant_app_count **** outputString += convertToOpenMetrics( "budibase_tenant_app_count", @@ -102,6 +134,102 @@ export async function fetch(ctx: BBContext) { allDatabases.length ) + // **** budibase_quota_usage_apps **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_apps", + "The number of apps created", + "gauge", + appsQuotaUsage + ) + + // **** budibase_quota_limit_apps **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_apps", + "The limit on the number of apps that can be created", + "gauge", + appsQuotaLimit == -1 ? 3000000 : appsQuotaLimit + ) + + // **** budibase_quota_usage_rows **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_rows", + "The number of database rows used from the quota", + "gauge", + rowsQuotaUsage + ) + + // **** budibase_quota_limit_rows **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_rows", + "The limit on the number of rows that can be created", + "gauge", + rowsQuotaLimit == -1 ? 3000000 : rowsQuotaLimit + ) + + // **** budibase_quota_usage_plugins **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_plugins", + "The number of plugins in use", + "gauge", + pluginsQuotaUsage + ) + + // **** budibase_quota_limit_plugins **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_plugins", + "The limit on the number of plugins that can be created", + "gauge", + pluginsQuotaLimit == -1 ? 3000000 : pluginsQuotaLimit + ) + + // **** budibase_quota_usage_user_groups **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_user_groups", + "The number of user groups created", + "gauge", + userGroupsQuotaUsage + ) + + // **** budibase_quota_limit_user_groups **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_user_groups", + "The limit on the number of user groups that can be created", + "gauge", + userGroupsQuotaLimit == -1 ? 3000000 : userGroupsQuotaLimit + ) + + // **** budibase_quota_usage_queries **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_queries", + "The number of queries used in the current month", + "gauge", + queryQuotaUsage + ) + + // **** budibase_quota_limit_queries **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_queries", + "The limit on the number of queries for the current month", + "gauge", + queryQuotaLimit == -1 ? 3000000 : queryQuotaLimit + ) + + // **** budibase_quota_usage_automations **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_automations", + "The number of automations used in the current month", + "gauge", + automationsQuotaUsage + ) + + // **** budibase_quota_limit_automations **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_automations", + "The limit on the number of automations that can be created", + "gauge", + automationsQuotaLimit == -1 ? 3000000 : automationsQuotaLimit + ) + ctx.body = outputString } @@ -111,10 +239,9 @@ export function convertToOpenMetrics( metricType: string, metricValue: number ) { - return ` - # HELP ${metricName} ${metricHelp}. - # TYPE ${metricName} ${metricType} - ${metricName} ${metricValue}` + return `# HELP ${metricName} ${metricHelp}. +# TYPE ${metricName} ${metricType} +${metricName} ${metricValue}\n` } export default { diff --git a/packages/server/src/api/routes/public/metrics.ts b/packages/server/src/api/routes/public/metrics.ts index 67cae14388..35745e3b13 100644 --- a/packages/server/src/api/routes/public/metrics.ts +++ b/packages/server/src/api/routes/public/metrics.ts @@ -9,7 +9,7 @@ const read = [] * get: * operationId: getById * summary: Retrieve Budibase tenant metrics - * description: Output metrics in openMetrics format compatible with Prometheus + * description: Output metrics in OpenMetrics format compatible with Prometheus * tags: * - metrics * responses: @@ -17,11 +17,6 @@ const read = [] * description: Returns tenant metrics. * content: * text/plain: - * schema: - * $ref: '#/components/schemas/tableOutput' - * examples: - * table: - * $ref: '#/components/examples/table' */ read.push(new Endpoint("get", "/metrics", controller.fetch)) From cbc377b0f4ad6f2146dd2842d43b0383606770a0 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 6 Mar 2023 11:51:49 +0000 Subject: [PATCH 3/6] unit tests --- packages/server/specs/openapi.json | 30 +++++++++--- packages/server/specs/openapi.yaml | 18 +++++-- .../src/api/controllers/public/metrics.ts | 20 ++++---- .../server/src/api/routes/public/index.ts | 11 ++++- .../src/api/routes/tests/metrics.spec.js | 49 +++++++++++++++++++ 5 files changed, 108 insertions(+), 20 deletions(-) create mode 100644 packages/server/src/api/routes/tests/metrics.spec.js diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 9a0d69e352..e1b977b113 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -817,7 +817,6 @@ "type": "string", "enum": [ "string", - "barcodeqr", "longform", "options", "number", @@ -829,7 +828,8 @@ "formula", "auto", "json", - "internal" + "internal", + "barcodeqr" ], "description": "Defines the type of the column, most explain themselves, a link column is a relationship." }, @@ -1021,7 +1021,6 @@ "type": "string", "enum": [ "string", - "barcodeqr", "longform", "options", "number", @@ -1033,7 +1032,8 @@ "formula", "auto", "json", - "internal" + "internal", + "barcodeqr" ], "description": "Defines the type of the column, most explain themselves, a link column is a relationship." }, @@ -1236,7 +1236,6 @@ "type": "string", "enum": [ "string", - "barcodeqr", "longform", "options", "number", @@ -1248,7 +1247,8 @@ "formula", "auto", "json", - "internal" + "internal", + "barcodeqr" ], "description": "Defines the type of the column, most explain themselves, a link column is a relationship." }, @@ -2054,6 +2054,24 @@ } } }, + "/metrics": { + "get": { + "operationId": "getById", + "summary": "Retrieve Budibase tenant metrics", + "description": "Output metrics in OpenMetrics format compatible with Prometheus", + "tags": [ + "metrics" + ], + "responses": { + "200": { + "description": "Returns tenant metrics.", + "content": { + "text/plain": null + } + } + } + } + }, "/queries/{queryId}": { "post": { "operationId": "execute", diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 69e44d881c..dc92c818b7 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -603,7 +603,6 @@ components: type: string enum: - string - - barcodeqr - longform - options - number @@ -616,6 +615,7 @@ components: - auto - json - internal + - barcodeqr description: Defines the type of the column, most explain themselves, a link column is a relationship. constraints: @@ -766,7 +766,6 @@ components: type: string enum: - string - - barcodeqr - longform - options - number @@ -779,6 +778,7 @@ components: - auto - json - internal + - barcodeqr description: Defines the type of the column, most explain themselves, a link column is a relationship. constraints: @@ -936,7 +936,6 @@ components: type: string enum: - string - - barcodeqr - longform - options - number @@ -949,6 +948,7 @@ components: - auto - json - internal + - barcodeqr description: Defines the type of the column, most explain themselves, a link column is a relationship. constraints: @@ -1531,6 +1531,18 @@ paths: examples: applications: $ref: "#/components/examples/applications" + /metrics: + get: + operationId: getById + summary: Retrieve Budibase tenant metrics + description: Output metrics in OpenMetrics format compatible with Prometheus + tags: + - metrics + responses: + "200": + description: Returns tenant metrics. + content: + ? text/plain "/queries/{queryId}": post: operationId: execute diff --git a/packages/server/src/api/controllers/public/metrics.ts b/packages/server/src/api/controllers/public/metrics.ts index 65f8849f7f..be78b86fc4 100644 --- a/packages/server/src/api/controllers/public/metrics.ts +++ b/packages/server/src/api/controllers/public/metrics.ts @@ -13,8 +13,8 @@ export async function fetch(ctx: BBContext) { // *** APPS *** const allDatabases = await dbCore.getAllDbs() - const devAppIDs = await dbCore.getDevAppIDs({ idsOnly: true }) - const prodAppIDs = await dbCore.getProdAppIDs({ idsOnly: true }) + const devAppIDs = await dbCore.getDevAppIDs() + const prodAppIDs = await dbCore.getProdAppIDs() const allAppIds = await dbCore.getAllApps({ idsOnly: true }) // *** USERS *** @@ -147,7 +147,7 @@ export async function fetch(ctx: BBContext) { "budibase_quota_limit_apps", "The limit on the number of apps that can be created", "gauge", - appsQuotaLimit == -1 ? 3000000 : appsQuotaLimit + appsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : appsQuotaLimit ) // **** budibase_quota_usage_rows **** @@ -163,7 +163,7 @@ export async function fetch(ctx: BBContext) { "budibase_quota_limit_rows", "The limit on the number of rows that can be created", "gauge", - rowsQuotaLimit == -1 ? 3000000 : rowsQuotaLimit + rowsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : rowsQuotaLimit ) // **** budibase_quota_usage_plugins **** @@ -179,7 +179,7 @@ export async function fetch(ctx: BBContext) { "budibase_quota_limit_plugins", "The limit on the number of plugins that can be created", "gauge", - pluginsQuotaLimit == -1 ? 3000000 : pluginsQuotaLimit + pluginsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : pluginsQuotaLimit ) // **** budibase_quota_usage_user_groups **** @@ -195,7 +195,7 @@ export async function fetch(ctx: BBContext) { "budibase_quota_limit_user_groups", "The limit on the number of user groups that can be created", "gauge", - userGroupsQuotaLimit == -1 ? 3000000 : userGroupsQuotaLimit + userGroupsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : userGroupsQuotaLimit ) // **** budibase_quota_usage_queries **** @@ -211,7 +211,7 @@ export async function fetch(ctx: BBContext) { "budibase_quota_limit_queries", "The limit on the number of queries for the current month", "gauge", - queryQuotaLimit == -1 ? 3000000 : queryQuotaLimit + queryQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : queryQuotaLimit ) // **** budibase_quota_usage_automations **** @@ -227,10 +227,12 @@ export async function fetch(ctx: BBContext) { "budibase_quota_limit_automations", "The limit on the number of automations that can be created", "gauge", - automationsQuotaLimit == -1 ? 3000000 : automationsQuotaLimit + automationsQuotaLimit == -1 + ? Number.MAX_SAFE_INTEGER + : automationsQuotaLimit ) - ctx.body = outputString + ctx.set('Content-Type', 'text/plain'); } export function convertToOpenMetrics( diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index e67c10fe02..e2bce1e730 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -13,7 +13,7 @@ import env from "../../../environment" // below imports don't have declaration files const Router = require("@koa/router") const { RateLimit, Stores } = require("koa2-ratelimit") -import { redis, permissions } from "@budibase/backend-core" +import { middleware, redis, permissions } from "@budibase/backend-core" const { PermissionType, PermissionLevel } = permissions const PREFIX = "/api/public/v1" @@ -92,6 +92,13 @@ function addToRouter(endpoints: any) { } } +function applyAdminRoutes(endpoints: any) { + addMiddleware(endpoints.read, middleware.builderOrAdmin) + addMiddleware(endpoints.write, middleware.builderOrAdmin) + addToRouter(endpoints.read) + addToRouter(endpoints.write) +} + function applyRoutes( endpoints: any, permType: string, @@ -120,8 +127,8 @@ function applyRoutes( addToRouter(endpoints.write) } +applyAdminRoutes(metricEndpoints) applyRoutes(appEndpoints, PermissionType.APP, "appId") -applyRoutes(metricEndpoints, PermissionType.APP, "appId") applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId") applyRoutes(userEndpoints, PermissionType.USER, "userId") applyRoutes(queryEndpoints, PermissionType.QUERY, "queryId") diff --git a/packages/server/src/api/routes/tests/metrics.spec.js b/packages/server/src/api/routes/tests/metrics.spec.js new file mode 100644 index 0000000000..6e689f040a --- /dev/null +++ b/packages/server/src/api/routes/tests/metrics.spec.js @@ -0,0 +1,49 @@ +const { roles, utils } = require("@budibase/backend-core") +const { checkBuilderEndpoint } = require("./utilities/TestFunctions") +const setup = require("./utilities") +const { BUILTIN_ROLE_IDS } = roles + +jest.setTimeout(30000) + +// jest.mock("../../../utilities/workerRequests", () => ({ +// getGlobalUsers: jest.fn(() => { +// return {} +// }), +// getGlobalSelf: jest.fn(() => { +// return {} +// }), +// deleteGlobalUser: jest.fn(), +// })) + +describe("/metrics", () => { + let request = setup.getRequest() + let config = setup.getConfig() + + afterAll(setup.afterAll) + + // For some reason this cannot be a beforeAll or the test "should be able to update the user" fail + beforeEach(async () => { + await config.init() + }) + + describe("get", () => { + it("returns a list of metrics", async () => { + const res = await request + .get(`/api/public/v1/metrics`) + .set(config.defaultHeaders()) + .expect("Content-Type", /text\/plain/) + .expect(200) + expect(res.text).toContain("budibase_tenant_user_count") + }) + + it("should apply authorization to endpoint", async () => { + // await config.createUser() + const res = await request + .get(`/api/public/v1/metrics`) + .set(config.publicHeaders()) + .expect(403) + + }) + }) + +}) From 13ae9501019d08e2d9ef6f19df8d3f30e37c44c5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 6 Mar 2023 15:31:27 +0000 Subject: [PATCH 4/6] Updating test case. --- .../routes/{ => public}/tests/metrics.spec.js | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) rename packages/server/src/api/routes/{ => public}/tests/metrics.spec.js (57%) diff --git a/packages/server/src/api/routes/tests/metrics.spec.js b/packages/server/src/api/routes/public/tests/metrics.spec.js similarity index 57% rename from packages/server/src/api/routes/tests/metrics.spec.js rename to packages/server/src/api/routes/public/tests/metrics.spec.js index 6e689f040a..8e226ec958 100644 --- a/packages/server/src/api/routes/tests/metrics.spec.js +++ b/packages/server/src/api/routes/public/tests/metrics.spec.js @@ -1,20 +1,7 @@ -const { roles, utils } = require("@budibase/backend-core") -const { checkBuilderEndpoint } = require("./utilities/TestFunctions") -const setup = require("./utilities") -const { BUILTIN_ROLE_IDS } = roles +const setup = require("../../tests/utilities") jest.setTimeout(30000) -// jest.mock("../../../utilities/workerRequests", () => ({ -// getGlobalUsers: jest.fn(() => { -// return {} -// }), -// getGlobalSelf: jest.fn(() => { -// return {} -// }), -// deleteGlobalUser: jest.fn(), -// })) - describe("/metrics", () => { let request = setup.getRequest() let config = setup.getConfig() @@ -36,13 +23,11 @@ describe("/metrics", () => { expect(res.text).toContain("budibase_tenant_user_count") }) - it("should apply authorization to endpoint", async () => { - // await config.createUser() - const res = await request + it("endpoint should not be publicly exposed", async () => { + await request .get(`/api/public/v1/metrics`) .set(config.publicHeaders()) .expect(403) - }) }) From 36683459b8b940739b70febe9ba425898490522d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 6 Mar 2023 15:55:55 +0000 Subject: [PATCH 5/6] Updating metrics endpoint to have examples of expected data format - show OpenMetrics response. --- packages/server/specs/openapi.json | 23 ++- packages/server/specs/openapi.yaml | 159 +++++++++++++++++- .../server/specs/resources/application.ts | 11 ++ packages/server/specs/resources/index.ts | 2 + packages/server/specs/resources/metrics.ts | 81 +++++++++ .../src/api/controllers/public/metrics.ts | 2 +- .../server/src/api/routes/public/metrics.ts | 5 + packages/server/src/definitions/openapi.ts | 15 ++ 8 files changed, 295 insertions(+), 3 deletions(-) create mode 100644 packages/server/specs/resources/metrics.ts diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 121828b21c..275c1af4f1 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -115,6 +115,15 @@ ] } }, + "deploymentOutput": { + "value": { + "data": { + "_id": "ef12381f934b4f129675cdbb76eff3c2", + "status": "SUCCESS", + "appUrl": "/app-url" + } + } + }, "inputRow": { "value": { "_id": "ro_ta_5b1649e42a5b41dea4ef7742a36a7a70_e6dc7e38cf1343b2b56760265201cda4", @@ -413,6 +422,9 @@ } ] } + }, + "metrics": { + "value": "# HELP budibase_os_uptime Time in seconds that the host operating system has been up.\n# TYPE budibase_os_uptime counter\nbudibase_os_uptime 54958\n# HELP budibase_os_free_mem Bytes of memory free for usage on the host operating system.\n# TYPE budibase_os_free_mem gauge\nbudibase_os_free_mem 804507648\n# HELP budibase_os_total_mem Total bytes of memory on the host operating system.\n# TYPE budibase_os_total_mem gauge\nbudibase_os_total_mem 16742404096\n# HELP budibase_os_used_mem Total bytes of memory in use on the host operating system.\n# TYPE budibase_os_used_mem gauge\nbudibase_os_used_mem 15937896448\n# HELP budibase_os_load1 Host operating system load average.\n# TYPE budibase_os_load1 gauge\nbudibase_os_load1 1.91\n# HELP budibase_os_load5 Host operating system load average.\n# TYPE budibase_os_load5 gauge\nbudibase_os_load5 1.75\n# HELP budibase_os_load15 Host operating system load average.\n# TYPE budibase_os_load15 gauge\nbudibase_os_load15 1.56\n# HELP budibase_tenant_user_count The number of users created.\n# TYPE budibase_tenant_user_count gauge\nbudibase_tenant_user_count 1\n# HELP budibase_tenant_app_count The number of apps created by a user.\n# TYPE budibase_tenant_app_count gauge\nbudibase_tenant_app_count 2\n# HELP budibase_tenant_production_app_count The number of apps a user has published.\n# TYPE budibase_tenant_production_app_count gauge\nbudibase_tenant_production_app_count 1\n# HELP budibase_tenant_dev_app_count The number of apps a user has unpublished in development.\n# TYPE budibase_tenant_dev_app_count gauge\nbudibase_tenant_dev_app_count 1\n# HELP budibase_tenant_db_count The number of couchdb databases including global tables such as _users.\n# TYPE budibase_tenant_db_count gauge\nbudibase_tenant_db_count 3\n# HELP budibase_quota_usage_apps The number of apps created.\n# TYPE budibase_quota_usage_apps gauge\nbudibase_quota_usage_apps 1\n# HELP budibase_quota_limit_apps The limit on the number of apps that can be created.\n# TYPE budibase_quota_limit_apps gauge\nbudibase_quota_limit_apps 9007199254740991\n# HELP budibase_quota_usage_rows The number of database rows used from the quota.\n# TYPE budibase_quota_usage_rows gauge\nbudibase_quota_usage_rows 0\n# HELP budibase_quota_limit_rows The limit on the number of rows that can be created.\n# TYPE budibase_quota_limit_rows gauge\nbudibase_quota_limit_rows 9007199254740991\n# HELP budibase_quota_usage_plugins The number of plugins in use.\n# TYPE budibase_quota_usage_plugins gauge\nbudibase_quota_usage_plugins 0\n# HELP budibase_quota_limit_plugins The limit on the number of plugins that can be created.\n# TYPE budibase_quota_limit_plugins gauge\nbudibase_quota_limit_plugins 9007199254740991\n# HELP budibase_quota_usage_user_groups The number of user groups created.\n# TYPE budibase_quota_usage_user_groups gauge\nbudibase_quota_usage_user_groups 0\n# HELP budibase_quota_limit_user_groups The limit on the number of user groups that can be created.\n# TYPE budibase_quota_limit_user_groups gauge\nbudibase_quota_limit_user_groups 9007199254740991\n# HELP budibase_quota_usage_queries The number of queries used in the current month.\n# TYPE budibase_quota_usage_queries gauge\nbudibase_quota_usage_queries 0\n# HELP budibase_quota_limit_queries The limit on the number of queries for the current month.\n# TYPE budibase_quota_limit_queries gauge\nbudibase_quota_limit_queries 9007199254740991\n# HELP budibase_quota_usage_automations The number of automations used in the current month.\n# TYPE budibase_quota_usage_automations gauge\nbudibase_quota_usage_automations 0\n# HELP budibase_quota_limit_automations The limit on the number of automations that can be created.\n# TYPE budibase_quota_limit_automations gauge\nbudibase_quota_limit_automations 9007199254740991\n" } }, "securitySchemes": { @@ -2066,7 +2078,16 @@ "200": { "description": "Returns tenant metrics.", "content": { - "text/plain": null + "text/plain": { + "schema": { + "type": "string" + }, + "examples": { + "metrics": { + "$ref": "#/components/examples/metrics" + } + } + } } } } diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 3243959c65..5647872d6b 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -85,6 +85,12 @@ components: updatedAt: 2022-02-22T13:00:54.035Z createdAt: 2022-02-11T18:02:26.961Z status: development + deploymentOutput: + value: + data: + _id: ef12381f934b4f129675cdbb76eff3c2 + status: SUCCESS + appUrl: /app-url inputRow: value: _id: ro_ta_5b1649e42a5b41dea4ef7742a36a7a70_e6dc7e38cf1343b2b56760265201cda4 @@ -290,6 +296,152 @@ components: name: Admin permissionId: admin inherits: POWER + metrics: + value: > + # HELP budibase_os_uptime Time in seconds that the host operating system + has been up. + + # TYPE budibase_os_uptime counter + + budibase_os_uptime 54958 + + # HELP budibase_os_free_mem Bytes of memory free for usage on the host operating system. + + # TYPE budibase_os_free_mem gauge + + budibase_os_free_mem 804507648 + + # HELP budibase_os_total_mem Total bytes of memory on the host operating system. + + # TYPE budibase_os_total_mem gauge + + budibase_os_total_mem 16742404096 + + # HELP budibase_os_used_mem Total bytes of memory in use on the host operating system. + + # TYPE budibase_os_used_mem gauge + + budibase_os_used_mem 15937896448 + + # HELP budibase_os_load1 Host operating system load average. + + # TYPE budibase_os_load1 gauge + + budibase_os_load1 1.91 + + # HELP budibase_os_load5 Host operating system load average. + + # TYPE budibase_os_load5 gauge + + budibase_os_load5 1.75 + + # HELP budibase_os_load15 Host operating system load average. + + # TYPE budibase_os_load15 gauge + + budibase_os_load15 1.56 + + # HELP budibase_tenant_user_count The number of users created. + + # TYPE budibase_tenant_user_count gauge + + budibase_tenant_user_count 1 + + # HELP budibase_tenant_app_count The number of apps created by a user. + + # TYPE budibase_tenant_app_count gauge + + budibase_tenant_app_count 2 + + # HELP budibase_tenant_production_app_count The number of apps a user has published. + + # TYPE budibase_tenant_production_app_count gauge + + budibase_tenant_production_app_count 1 + + # HELP budibase_tenant_dev_app_count The number of apps a user has unpublished in development. + + # TYPE budibase_tenant_dev_app_count gauge + + budibase_tenant_dev_app_count 1 + + # HELP budibase_tenant_db_count The number of couchdb databases including global tables such as _users. + + # TYPE budibase_tenant_db_count gauge + + budibase_tenant_db_count 3 + + # HELP budibase_quota_usage_apps The number of apps created. + + # TYPE budibase_quota_usage_apps gauge + + budibase_quota_usage_apps 1 + + # HELP budibase_quota_limit_apps The limit on the number of apps that can be created. + + # TYPE budibase_quota_limit_apps gauge + + budibase_quota_limit_apps 9007199254740991 + + # HELP budibase_quota_usage_rows The number of database rows used from the quota. + + # TYPE budibase_quota_usage_rows gauge + + budibase_quota_usage_rows 0 + + # HELP budibase_quota_limit_rows The limit on the number of rows that can be created. + + # TYPE budibase_quota_limit_rows gauge + + budibase_quota_limit_rows 9007199254740991 + + # HELP budibase_quota_usage_plugins The number of plugins in use. + + # TYPE budibase_quota_usage_plugins gauge + + budibase_quota_usage_plugins 0 + + # HELP budibase_quota_limit_plugins The limit on the number of plugins that can be created. + + # TYPE budibase_quota_limit_plugins gauge + + budibase_quota_limit_plugins 9007199254740991 + + # HELP budibase_quota_usage_user_groups The number of user groups created. + + # TYPE budibase_quota_usage_user_groups gauge + + budibase_quota_usage_user_groups 0 + + # HELP budibase_quota_limit_user_groups The limit on the number of user groups that can be created. + + # TYPE budibase_quota_limit_user_groups gauge + + budibase_quota_limit_user_groups 9007199254740991 + + # HELP budibase_quota_usage_queries The number of queries used in the current month. + + # TYPE budibase_quota_usage_queries gauge + + budibase_quota_usage_queries 0 + + # HELP budibase_quota_limit_queries The limit on the number of queries for the current month. + + # TYPE budibase_quota_limit_queries gauge + + budibase_quota_limit_queries 9007199254740991 + + # HELP budibase_quota_usage_automations The number of automations used in the current month. + + # TYPE budibase_quota_usage_automations gauge + + budibase_quota_usage_automations 0 + + # HELP budibase_quota_limit_automations The limit on the number of automations that can be created. + + # TYPE budibase_quota_limit_automations gauge + + budibase_quota_limit_automations 9007199254740991 securitySchemes: ApiKeyAuth: type: apiKey @@ -1542,7 +1694,12 @@ paths: "200": description: Returns tenant metrics. content: - ? text/plain + text/plain: + schema: + type: string + examples: + metrics: + $ref: "#/components/examples/metrics" "/queries/{queryId}": post: operationId: queryExecute diff --git a/packages/server/specs/resources/application.ts b/packages/server/specs/resources/application.ts index aa037da154..cd7a68c049 100644 --- a/packages/server/specs/resources/application.ts +++ b/packages/server/specs/resources/application.ts @@ -15,6 +15,12 @@ const application = { lockedBy: userResource.getExamples().user.value.user, } +const deployment = { + _id: "ef12381f934b4f129675cdbb76eff3c2", + status: "SUCCESS", + appUrl: "/app-url", +} + const base = { name: { description: "The name of the app.", @@ -108,6 +114,11 @@ export default new Resource() data: [application], }, }, + deploymentOutput: { + value: { + data: deployment, + }, + }, }) .setSchemas({ application: applicationSchema, diff --git a/packages/server/specs/resources/index.ts b/packages/server/specs/resources/index.ts index f1729f4541..6b8a1aa437 100644 --- a/packages/server/specs/resources/index.ts +++ b/packages/server/specs/resources/index.ts @@ -3,6 +3,7 @@ import row from "./row" import table from "./table" import query from "./query" import user from "./user" +import metrics from "./metrics" import misc from "./misc" export const examples = { @@ -12,6 +13,7 @@ export const examples = { ...query.getExamples(), ...user.getExamples(), ...misc.getExamples(), + ...metrics.getExamples(), } export const schemas = { diff --git a/packages/server/specs/resources/metrics.ts b/packages/server/specs/resources/metrics.ts new file mode 100644 index 0000000000..1dfbe101d8 --- /dev/null +++ b/packages/server/specs/resources/metrics.ts @@ -0,0 +1,81 @@ +import Resource from "./utils/Resource" + +const metricsResponse = + "# HELP budibase_os_uptime Time in seconds that the host operating system has been up.\n" + + "# TYPE budibase_os_uptime counter\n" + + "budibase_os_uptime 54958\n" + + "# HELP budibase_os_free_mem Bytes of memory free for usage on the host operating system.\n" + + "# TYPE budibase_os_free_mem gauge\n" + + "budibase_os_free_mem 804507648\n" + + "# HELP budibase_os_total_mem Total bytes of memory on the host operating system.\n" + + "# TYPE budibase_os_total_mem gauge\n" + + "budibase_os_total_mem 16742404096\n" + + "# HELP budibase_os_used_mem Total bytes of memory in use on the host operating system.\n" + + "# TYPE budibase_os_used_mem gauge\n" + + "budibase_os_used_mem 15937896448\n" + + "# HELP budibase_os_load1 Host operating system load average.\n" + + "# TYPE budibase_os_load1 gauge\n" + + "budibase_os_load1 1.91\n" + + "# HELP budibase_os_load5 Host operating system load average.\n" + + "# TYPE budibase_os_load5 gauge\n" + + "budibase_os_load5 1.75\n" + + "# HELP budibase_os_load15 Host operating system load average.\n" + + "# TYPE budibase_os_load15 gauge\n" + + "budibase_os_load15 1.56\n" + + "# HELP budibase_tenant_user_count The number of users created.\n" + + "# TYPE budibase_tenant_user_count gauge\n" + + "budibase_tenant_user_count 1\n" + + "# HELP budibase_tenant_app_count The number of apps created by a user.\n" + + "# TYPE budibase_tenant_app_count gauge\n" + + "budibase_tenant_app_count 2\n" + + "# HELP budibase_tenant_production_app_count The number of apps a user has published.\n" + + "# TYPE budibase_tenant_production_app_count gauge\n" + + "budibase_tenant_production_app_count 1\n" + + "# HELP budibase_tenant_dev_app_count The number of apps a user has unpublished in development.\n" + + "# TYPE budibase_tenant_dev_app_count gauge\n" + + "budibase_tenant_dev_app_count 1\n" + + "# HELP budibase_tenant_db_count The number of couchdb databases including global tables such as _users.\n" + + "# TYPE budibase_tenant_db_count gauge\n" + + "budibase_tenant_db_count 3\n" + + "# HELP budibase_quota_usage_apps The number of apps created.\n" + + "# TYPE budibase_quota_usage_apps gauge\n" + + "budibase_quota_usage_apps 1\n" + + "# HELP budibase_quota_limit_apps The limit on the number of apps that can be created.\n" + + "# TYPE budibase_quota_limit_apps gauge\n" + + "budibase_quota_limit_apps 9007199254740991\n" + + "# HELP budibase_quota_usage_rows The number of database rows used from the quota.\n" + + "# TYPE budibase_quota_usage_rows gauge\n" + + "budibase_quota_usage_rows 0\n" + + "# HELP budibase_quota_limit_rows The limit on the number of rows that can be created.\n" + + "# TYPE budibase_quota_limit_rows gauge\n" + + "budibase_quota_limit_rows 9007199254740991\n" + + "# HELP budibase_quota_usage_plugins The number of plugins in use.\n" + + "# TYPE budibase_quota_usage_plugins gauge\n" + + "budibase_quota_usage_plugins 0\n" + + "# HELP budibase_quota_limit_plugins The limit on the number of plugins that can be created.\n" + + "# TYPE budibase_quota_limit_plugins gauge\n" + + "budibase_quota_limit_plugins 9007199254740991\n" + + "# HELP budibase_quota_usage_user_groups The number of user groups created.\n" + + "# TYPE budibase_quota_usage_user_groups gauge\n" + + "budibase_quota_usage_user_groups 0\n" + + "# HELP budibase_quota_limit_user_groups The limit on the number of user groups that can be created.\n" + + "# TYPE budibase_quota_limit_user_groups gauge\n" + + "budibase_quota_limit_user_groups 9007199254740991\n" + + "# HELP budibase_quota_usage_queries The number of queries used in the current month.\n" + + "# TYPE budibase_quota_usage_queries gauge\n" + + "budibase_quota_usage_queries 0\n" + + "# HELP budibase_quota_limit_queries The limit on the number of queries for the current month.\n" + + "# TYPE budibase_quota_limit_queries gauge\n" + + "budibase_quota_limit_queries 9007199254740991\n" + + "# HELP budibase_quota_usage_automations The number of automations used in the current month.\n" + + "# TYPE budibase_quota_usage_automations gauge\n" + + "budibase_quota_usage_automations 0\n" + + "# HELP budibase_quota_limit_automations The limit on the number of automations that can be created.\n" + + "# TYPE budibase_quota_limit_automations gauge\n" + + "budibase_quota_limit_automations 9007199254740991\n" + +export default new Resource().setExamples({ + metrics: { + value: metricsResponse, + }, +}) diff --git a/packages/server/src/api/controllers/public/metrics.ts b/packages/server/src/api/controllers/public/metrics.ts index be78b86fc4..f702a47831 100644 --- a/packages/server/src/api/controllers/public/metrics.ts +++ b/packages/server/src/api/controllers/public/metrics.ts @@ -232,7 +232,7 @@ export async function fetch(ctx: BBContext) { : automationsQuotaLimit ) ctx.body = outputString - ctx.set('Content-Type', 'text/plain'); + ctx.set("Content-Type", "text/plain") } export function convertToOpenMetrics( diff --git a/packages/server/src/api/routes/public/metrics.ts b/packages/server/src/api/routes/public/metrics.ts index 35745e3b13..ff993d2ec6 100644 --- a/packages/server/src/api/routes/public/metrics.ts +++ b/packages/server/src/api/routes/public/metrics.ts @@ -17,6 +17,11 @@ const read = [] * description: Returns tenant metrics. * content: * text/plain: + * schema: + * type: string + * examples: + * metrics: + * $ref: '#/components/examples/metrics' */ read.push(new Endpoint("get", "/metrics", controller.fetch)) diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index 8c44131e96..b7b77b42d6 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -22,6 +22,10 @@ export interface paths { /** Based on application properties (currently only name) search for applications. */ post: operations["appSearch"]; }; + "/metrics": { + /** Output metrics in OpenMetrics format compatible with Prometheus */ + get: operations["getById"]; + }; "/queries/{queryId}": { /** Queries which have been created within a Budibase app can be executed using this, */ post: operations["queryExecute"]; @@ -844,6 +848,17 @@ export interface operations { }; }; }; + /** Output metrics in OpenMetrics format compatible with Prometheus */ + getById: { + responses: { + /** Returns tenant metrics. */ + 200: { + content: { + "text/plain": string; + }; + }; + }; + }; /** Queries which have been created within a Budibase app can be executed using this, */ queryExecute: { parameters: { From 0026f86cc667a538886aaf772bf7338005645751 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 6 Mar 2023 17:41:24 +0000 Subject: [PATCH 6/6] Fixing operation ID. --- packages/server/specs/openapi.json | 2 +- packages/server/specs/openapi.yaml | 2 +- packages/server/src/api/controllers/public/metrics.ts | 4 ++-- packages/server/src/api/routes/public/metrics.ts | 2 +- packages/server/src/definitions/openapi.ts | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 275c1af4f1..bcff78e861 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -2068,7 +2068,7 @@ }, "/metrics": { "get": { - "operationId": "getById", + "operationId": "metricsGet", "summary": "Retrieve Budibase tenant metrics", "description": "Output metrics in OpenMetrics format compatible with Prometheus", "tags": [ diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 5647872d6b..a9daed6c76 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -1685,7 +1685,7 @@ paths: $ref: "#/components/examples/applications" /metrics: get: - operationId: getById + operationId: metricsGet summary: Retrieve Budibase tenant metrics description: Output metrics in OpenMetrics format compatible with Prometheus tags: diff --git a/packages/server/src/api/controllers/public/metrics.ts b/packages/server/src/api/controllers/public/metrics.ts index f702a47831..b6a8de3b7c 100644 --- a/packages/server/src/api/controllers/public/metrics.ts +++ b/packages/server/src/api/controllers/public/metrics.ts @@ -1,10 +1,10 @@ -import { App, BBContext } from "@budibase/types" +import { Ctx } from "@budibase/types" import { users as userCore, db as dbCore } from "@budibase/backend-core" import { quotas, licensing } from "@budibase/pro" import os from "os" -export async function fetch(ctx: BBContext) { +export async function fetch(ctx: Ctx) { // *** OPERATING SYSTEM *** const freeMem = os.freemem() const totalMem = os.totalmem() diff --git a/packages/server/src/api/routes/public/metrics.ts b/packages/server/src/api/routes/public/metrics.ts index ff993d2ec6..841da0e804 100644 --- a/packages/server/src/api/routes/public/metrics.ts +++ b/packages/server/src/api/routes/public/metrics.ts @@ -7,7 +7,7 @@ const read = [] * @openapi * /metrics: * get: - * operationId: getById + * operationId: metricsGet * summary: Retrieve Budibase tenant metrics * description: Output metrics in OpenMetrics format compatible with Prometheus * tags: diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index b7b77b42d6..5ca4990647 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -24,7 +24,7 @@ export interface paths { }; "/metrics": { /** Output metrics in OpenMetrics format compatible with Prometheus */ - get: operations["getById"]; + get: operations["metricsGet"]; }; "/queries/{queryId}": { /** Queries which have been created within a Budibase app can be executed using this, */ @@ -849,7 +849,7 @@ export interface operations { }; }; /** Output metrics in OpenMetrics format compatible with Prometheus */ - getById: { + metricsGet: { responses: { /** Returns tenant metrics. */ 200: {