From aede23d44e46a51823ec8914bed8c39a6a3bae32 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Sun, 22 May 2022 18:29:02 +0100 Subject: [PATCH] Improving performance of load script, can generate thousands of users a second. --- packages/backend-core/src/utils.js | 41 +++++++++--- packages/server/scripts/load/users.js | 62 ++++++++++++------- .../src/api/controllers/public/users.ts | 27 +------- packages/server/src/environment.js | 6 +- packages/server/src/utilities/users.js | 21 +++++++ packages/worker/src/environment.js | 6 +- 6 files changed, 106 insertions(+), 57 deletions(-) diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js index 5c922c42ad..e764f35803 100644 --- a/packages/backend-core/src/utils.js +++ b/packages/backend-core/src/utils.js @@ -197,11 +197,16 @@ exports.getBuildersCount = async () => { return builders.length } -exports.saveUser = async ( +const DEFAULT_SAVE_USER = { + hashPassword: true, + requirePassword: true, + bulkCreate: false, +} + +exports.internalSaveUser = async ( user, tenantId, - hashPassword = true, - requirePassword = true + { hashPassword, requirePassword, bulkCreate } = DEFAULT_SAVE_USER ) => { if (!tenantId) { throw "No tenancy specified." @@ -213,7 +218,10 @@ exports.saveUser = async ( let { email, password, _id } = user // make sure another user isn't using the same email let dbUser - if (email) { + // user can't exist in bulk creation + if (bulkCreate) { + dbUser = null + } else if (email) { // check budibase users inside the tenant dbUser = await exports.getGlobalUserByEmail(email) if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) { @@ -267,11 +275,17 @@ exports.saveUser = async ( user.status = UserStatus.ACTIVE } try { - const response = await db.put({ + const putOpts = { password: hashedPassword, ...user, - }) - await tryAddTenant(tenantId, _id, email) + } + if (bulkCreate) { + return putOpts + } + const response = await db.put(putOpts) + if (env.MULTI_TENANCY) { + await tryAddTenant(tenantId, _id, email) + } await userCache.invalidateUser(response.id) return { _id: response.id, @@ -288,6 +302,19 @@ exports.saveUser = async ( }) } +// maintained for api compat, don't want to change function signature +exports.saveUser = async ( + user, + tenantId, + hashPassword = true, + requirePassword = true +) => { + return exports.internalSaveUser(user, tenantId, { + hashPassword, + requirePassword, + }) +} + /** * Logs a user out from budibase. Re-used across account portal and builder. */ diff --git a/packages/server/scripts/load/users.js b/packages/server/scripts/load/users.js index 7c3964b6e7..3bc8056986 100644 --- a/packages/server/scripts/load/users.js +++ b/packages/server/scripts/load/users.js @@ -1,13 +1,21 @@ -const fetch = require("node-fetch") -const { getProdAppID } = require("@budibase/backend-core/db") +// get the JWT secret etc +require("../../src/environment") +require("@budibase/backend-core").init() +const { + getProdAppID, + generateGlobalUserID, +} = require("@budibase/backend-core/db") +const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy") +const { internalSaveUser } = require("@budibase/backend-core/utils") +const { publicApiUserFix } = require("../../src/utilities/users") +const { hash } = require("@budibase/backend-core/utils") const USER_LOAD_NUMBER = 10000 -const BATCH_SIZE = 25 -const SERVER_URL = "http://localhost:4001" +const BATCH_SIZE = 200 const PASSWORD = "test" +const TENANT_ID = "default" const APP_ID = process.argv[2] -const API_KEY = process.argv[3] const words = [ "test", @@ -31,17 +39,15 @@ if (!APP_ID) { console.error("Must supply app ID as first CLI option!") process.exit(-1) } -if (!API_KEY) { - console.error("Must supply API key as second CLI option!") - process.exit(-1) -} const WORD_1 = words[Math.floor(Math.random() * words.length)] const WORD_2 = words[Math.floor(Math.random() * words.length)] +let HASHED_PASSWORD function generateUser(count) { return { - password: PASSWORD, + _id: generateGlobalUserID(), + password: HASHED_PASSWORD, email: `${WORD_1}${count}@${WORD_2}.com`, roles: { [getProdAppID(APP_ID)]: "BASIC", @@ -54,23 +60,31 @@ function generateUser(count) { } async function run() { - for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) { - let promises = [] - for (let j = 0; j < BATCH_SIZE; j++) { - promises.push( - fetch(`${SERVER_URL}/api/public/v1/users`, { - method: "POST", - body: JSON.stringify(generateUser(i + j)), - headers: { - "x-budibase-api-key": API_KEY, - "Content-Type": "application/json", + HASHED_PASSWORD = await hash(PASSWORD) + return doInTenant(TENANT_ID, async () => { + const db = getGlobalDB() + for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) { + let userSavePromises = [] + for (let j = 0; j < BATCH_SIZE; j++) { + // like the public API + const ctx = publicApiUserFix({ + request: { + body: generateUser(i + j), }, }) - ) + userSavePromises.push( + internalSaveUser(ctx.request.body, TENANT_ID, { + hashPassword: false, + requirePassword: true, + bulkCreate: true, + }) + ) + } + const users = await Promise.all(userSavePromises) + await db.bulkDocs(users) + console.log(`${i + BATCH_SIZE} users have been created.`) } - await Promise.all(promises) - console.log(`${i + BATCH_SIZE} users have been created.`) - } + }) } run() diff --git a/packages/server/src/api/controllers/public/users.ts b/packages/server/src/api/controllers/public/users.ts index f199dcb761..129d2c883f 100644 --- a/packages/server/src/api/controllers/public/users.ts +++ b/packages/server/src/api/controllers/public/users.ts @@ -4,30 +4,9 @@ import { readGlobalUser, saveGlobalUser, } from "../../../utilities/workerRequests" +import { publicApiUserFix } from "../../../utilities/users" import { search as stringSearch } from "./utils" -const { getProdAppID } = require("@budibase/backend-core/db") - -function fixUser(ctx: any) { - if (!ctx.request.body) { - return ctx - } - if (!ctx.request.body._id && ctx.params.userId) { - ctx.request.body._id = ctx.params.userId - } - if (!ctx.request.body.roles) { - ctx.request.body.roles = {} - } else { - const newRoles: { [key: string]: string } = {} - for (let [appId, role] of Object.entries(ctx.request.body.roles)) { - // @ts-ignore - newRoles[getProdAppID(appId)] = role - } - ctx.request.body.roles = newRoles - } - return ctx -} - function getUser(ctx: any, userId?: string) { if (userId) { ctx.params = { userId } @@ -45,7 +24,7 @@ export async function search(ctx: any, next: any) { } export async function create(ctx: any, next: any) { - const response = await saveGlobalUser(fixUser(ctx)) + const response = await saveGlobalUser(publicApiUserFix(ctx)) ctx.body = await getUser(ctx, response._id) await next() } @@ -61,7 +40,7 @@ export async function update(ctx: any, next: any) { ...ctx.request.body, _rev: user._rev, } - const response = await saveGlobalUser(fixUser(ctx)) + const response = await saveGlobalUser(publicApiUserFix(ctx)) ctx.body = await getUser(ctx, response._id) await next() } diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 96f395f153..9fa8692298 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -1,3 +1,5 @@ +const { join } = require("path") + function isTest() { return ( process.env.NODE_ENV === "jest" || @@ -20,7 +22,9 @@ function isCypress() { let LOADED = false if (!LOADED && isDev() && !isTest()) { - require("dotenv").config() + require("dotenv").config({ + path: join(__dirname, "..", ".env"), + }) LOADED = true } diff --git a/packages/server/src/utilities/users.js b/packages/server/src/utilities/users.js index b3601986d8..e769441322 100644 --- a/packages/server/src/utilities/users.js +++ b/packages/server/src/utilities/users.js @@ -1,6 +1,7 @@ const { InternalTables } = require("../db/utils") const { getGlobalUser } = require("../utilities/global") const { getAppDB } = require("@budibase/backend-core/context") +const { getProdAppID } = require("@budibase/backend-core/db") exports.getFullUser = async (ctx, userId) => { const global = await getGlobalUser(userId) @@ -22,3 +23,23 @@ exports.getFullUser = async (ctx, userId) => { _id: userId, } } + +exports.publicApiUserFix = ctx => { + if (!ctx.request.body) { + return ctx + } + if (!ctx.request.body._id && ctx.params.userId) { + ctx.request.body._id = ctx.params.userId + } + if (!ctx.request.body.roles) { + ctx.request.body.roles = {} + } else { + const newRoles = {} + for (let [appId, role] of Object.entries(ctx.request.body.roles)) { + // @ts-ignore + newRoles[getProdAppID(appId)] = role + } + ctx.request.body.roles = newRoles + } + return ctx +} diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index c965863a54..8ef12e3877 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -1,3 +1,5 @@ +const { join } = require("path") + function isDev() { return process.env.NODE_ENV !== "production" } @@ -12,7 +14,9 @@ function isTest() { let LOADED = false if (!LOADED && isDev() && !isTest()) { - require("dotenv").config() + require("dotenv").config({ + path: join(__dirname, "..", ".env"), + }) LOADED = true }