CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-08-09 07:08:01 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #6021 from Budibase/hotfixes/perf-bg

Browse source 
Hotfixes/perf bg
This commit is contained in:
Martin McKeaveney 2022-05-22 19:21:50 +01:00 committed by GitHub
commit fe353b5fce
10 changed files with 196 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
packages/backend-core/src/utils.js
View file

@ -197,11 +197,16 @@ exports.getBuildersCount = async () => {
return builders.length
}
exports.saveUser = async (
const DEFAULT_SAVE_USER = {
hashPassword: true,
requirePassword: true,
bulkCreate: false,
}
exports.internalSaveUser = async (
user,
tenantId,
hashPassword = true,
requirePassword = true
{ hashPassword, requirePassword, bulkCreate } = DEFAULT_SAVE_USER
) => {
if (!tenantId) {
throw "No tenancy specified."
@ -213,7 +218,10 @@ exports.saveUser = async (
let { email, password, _id } = user
// make sure another user isn't using the same email
let dbUser
if (email) {
// user can't exist in bulk creation
if (bulkCreate) {
dbUser = null
} else if (email) {
// check budibase users inside the tenant
dbUser = await exports.getGlobalUserByEmail(email)
if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) {
@ -267,11 +275,17 @@ exports.saveUser = async (
user.status = UserStatus.ACTIVE
}
try {
const response = await db.put({
const putOpts = {
password: hashedPassword,
...user,
})
await tryAddTenant(tenantId, _id, email)
}
if (bulkCreate) {
return putOpts
}
const response = await db.put(putOpts)
if (env.MULTI_TENANCY) {
await tryAddTenant(tenantId, _id, email)
}
await userCache.invalidateUser(response.id)
return {
_id: response.id,
@ -288,6 +302,19 @@ exports.saveUser = async (
})
}
// maintained for api compat, don't want to change function signature
exports.saveUser = async (
user,
tenantId,
hashPassword = true,
requirePassword = true
) => {
return exports.internalSaveUser(user, tenantId, {
hashPassword,
requirePassword,
})
}
/**
* Logs a user out from budibase. Re-used across account portal and builder.
*/

97
packages/server/scripts/load/users.js Normal file
View file

@ -0,0 +1,97 @@
// get the JWT secret etc
require("../../src/environment")
require("@budibase/backend-core").init()
const {
getProdAppID,
generateGlobalUserID,
} = require("@budibase/backend-core/db")
const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
const { internalSaveUser } = require("@budibase/backend-core/utils")
const { publicApiUserFix } = require("../../src/utilities/users")
const { hash } = require("@budibase/backend-core/utils")
const USER_LOAD_NUMBER = 10000
const BATCH_SIZE = 200
const PASSWORD = "test"
const TENANT_ID = "default"
const APP_ID = process.argv[2]
const words = [
"test",
"testing",
"budi",
"mail",
"age",
"risk",
"load",
"uno",
"arm",
"leg",
"pen",
"glass",
"box",
"chicken",
"bottle",
]
if (!APP_ID) {
console.error("Must supply app ID as first CLI option!")
process.exit(-1)
}
const WORD_1 = words[Math.floor(Math.random() * words.length)]
const WORD_2 = words[Math.floor(Math.random() * words.length)]
let HASHED_PASSWORD
function generateUser(count) {
return {
_id: generateGlobalUserID(),
password: HASHED_PASSWORD,
email: `${WORD_1}${count}@${WORD_2}.com`,
roles: {
[getProdAppID(APP_ID)]: "BASIC",
},
status: "active",
forceResetPassword: false,
firstName: "John",
lastName: "Smith",
}
}
async function run() {
HASHED_PASSWORD = await hash(PASSWORD)
return doInTenant(TENANT_ID, async () => {
const db = getGlobalDB()
for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) {
let userSavePromises = []
for (let j = 0; j < BATCH_SIZE; j++) {
// like the public API
const ctx = publicApiUserFix({
request: {
body: generateUser(i + j),
},
})
userSavePromises.push(
internalSaveUser(ctx.request.body, TENANT_ID, {
hashPassword: false,
requirePassword: true,
bulkCreate: true,
})
)
}
const users = await Promise.all(userSavePromises)
await db.bulkDocs(users)
console.log(`${i + BATCH_SIZE} users have been created.`)
}
})
}
run()
.then(() => {
console.log(`Generated ${USER_LOAD_NUMBER} users!`)
})
.catch(err => {
console.error("Failed for reason: ", err)
process.exit(-1)
})

9
packages/server/src/api/controllers/application.ts
View file

@ -439,6 +439,15 @@ export const destroy = async (ctx: any) => {
}
export const sync = async (ctx: any, next: any) => {
if (env.DISABLE_AUTO_PROD_APP_SYNC) {
ctx.status = 200
ctx.body = {
message:
"App sync disabled. You can reenable with the DISABLE_AUTO_PROD_APP_SYNC environment variable.",
}
return next()
}
const appId = ctx.params.appId
if (!isDevAppID(appId)) {
ctx.throw(400, "This action cannot be performed for production apps")

27
packages/server/src/api/controllers/public/users.ts
View file

@ -4,30 +4,9 @@ import {
readGlobalUser,
saveGlobalUser,
} from "../../../utilities/workerRequests"
import { publicApiUserFix } from "../../../utilities/users"
import { search as stringSearch } from "./utils"
const { getProdAppID } = require("@budibase/backend-core/db")
function fixUser(ctx: any) {
if (!ctx.request.body) {
return ctx
}
if (!ctx.request.body._id && ctx.params.userId) {
ctx.request.body._id = ctx.params.userId
}
if (!ctx.request.body.roles) {
ctx.request.body.roles = {}
} else {
const newRoles: { [key: string]: string } = {}
for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
// @ts-ignore
newRoles[getProdAppID(appId)] = role
}
ctx.request.body.roles = newRoles
}
return ctx
}
function getUser(ctx: any, userId?: string) {
if (userId) {
ctx.params = { userId }
@ -45,7 +24,7 @@ export async function search(ctx: any, next: any) {
}
export async function create(ctx: any, next: any) {
const response = await saveGlobalUser(fixUser(ctx))
const response = await saveGlobalUser(publicApiUserFix(ctx))
ctx.body = await getUser(ctx, response._id)
await next()
}
@ -61,7 +40,7 @@ export async function update(ctx: any, next: any) {
...ctx.request.body,
_rev: user._rev,
}
const response = await saveGlobalUser(fixUser(ctx))
const response = await saveGlobalUser(publicApiUserFix(ctx))
ctx.body = await getUser(ctx, response._id)
await next()
}

7
packages/server/src/environment.js
View file

@ -1,3 +1,5 @@
const { join } = require("path")
function isTest() {
return (
process.env.NODE_ENV === "jest" ||
@ -20,7 +22,9 @@ function isCypress() {
let LOADED = false
if (!LOADED && isDev() && !isTest()) {
require("dotenv").config()
require("dotenv").config({
path: join(__dirname, "..", ".env"),
})
LOADED = true
}
@ -57,6 +61,7 @@ module.exports = {
JEST_WORKER_ID: process.env.JEST_WORKER_ID,
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC,
// minor
SALT_ROUNDS: process.env.SALT_ROUNDS,
LOGGER: process.env.LOGGER,

21
packages/server/src/utilities/users.js
View file

@ -1,6 +1,7 @@
const { InternalTables } = require("../db/utils")
const { getGlobalUser } = require("../utilities/global")
const { getAppDB } = require("@budibase/backend-core/context")
const { getProdAppID } = require("@budibase/backend-core/db")
exports.getFullUser = async (ctx, userId) => {
const global = await getGlobalUser(userId)
@ -22,3 +23,23 @@ exports.getFullUser = async (ctx, userId) => {
_id: userId,
}
}
exports.publicApiUserFix = ctx => {
if (!ctx.request.body) {
return ctx
}
if (!ctx.request.body._id && ctx.params.userId) {
ctx.request.body._id = ctx.params.userId
}
if (!ctx.request.body.roles) {
ctx.request.body.roles = {}
} else {
const newRoles = {}
for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
// @ts-ignore
newRoles[getProdAppID(appId)] = role
}
ctx.request.body.roles = newRoles
}
return ctx
}

18
packages/server/yarn.lock
View file

@ -1014,10 +1014,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.164":
version "1.0.164"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6"
integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg==
"@budibase/backend-core@1.0.167":
version "1.0.167"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879"
integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1091,12 +1091,12 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/pro@1.0.164":
version "1.0.164"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a"
integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA==
"@budibase/pro@1.0.167":
version "1.0.167"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84"
integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA==
dependencies:
"@budibase/backend-core" "1.0.164"
"@budibase/backend-core" "1.0.167"
node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139":

5
packages/worker/src/api/controllers/global/configs.js
View file

@ -271,13 +271,14 @@ exports.configChecklist = async function (ctx) {
const oidcConfig = await getScopedFullConfig(db, {
type: Configs.OIDC,
})
// They have set up an global user
const users = await db.allDocs(
getGlobalUserParams(null, {
include_docs: true,
limit: 1,
})
)
const adminUser = users.rows.some(row => row.doc.admin)
ctx.body = {
apps: {
@ -291,7 +292,7 @@ exports.configChecklist = async function (ctx) {
link: "/builder/portal/manage/email",
},
adminUser: {
checked: adminUser,
checked: users && users.rows.length >= 1,
label: "Create your first user",
link: "/builder/portal/manage/users",
},

6
packages/worker/src/environment.js
View file

@ -1,3 +1,5 @@
const { join } = require("path")
function isDev() {
return process.env.NODE_ENV !== "production"
}
@ -12,7 +14,9 @@ function isTest() {
let LOADED = false
if (!LOADED && isDev() && !isTest()) {
require("dotenv").config()
require("dotenv").config({
path: join(__dirname, "..", ".env"),
})
LOADED = true
}

18
packages/worker/yarn.lock
View file

@ -293,10 +293,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.164":
version "1.0.164"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6"
integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg==
"@budibase/backend-core@1.0.167":
version "1.0.167"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879"
integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -321,12 +321,12 @@
uuid "^8.3.2"
zlib "^1.0.5"
"@budibase/pro@1.0.164":
version "1.0.164"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a"
integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA==
"@budibase/pro@1.0.167":
version "1.0.167"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84"
integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA==
dependencies:
"@budibase/backend-core" "1.0.164"
"@budibase/backend-core" "1.0.167"
node-fetch "^2.6.1"
"@cspotcode/source-map-consumer@0.8.0":