diff --git a/packages/backend-core/src/cache/tests/user.spec.ts b/packages/backend-core/src/cache/tests/user.spec.ts new file mode 100644 index 0000000000..80e5bc3063 --- /dev/null +++ b/packages/backend-core/src/cache/tests/user.spec.ts @@ -0,0 +1,145 @@ +import { User } from "@budibase/types" +import { generator, structures } from "../../../tests" +import { DBTestConfiguration } from "../../../tests/extra" +import { getUsers } from "../user" +import { getGlobalDB } from "../../context" +import _ from "lodash" + +import * as redis from "../../redis/init" +import { UserDB } from "../../users" + +const config = new DBTestConfiguration() + +describe("user cache", () => { + describe("getUsers", () => { + const users: User[] = [] + beforeAll(async () => { + const userCount = 10 + const userIds = generator.arrayOf(() => generator.guid(), { + min: userCount, + max: userCount, + }) + + await config.doInTenant(async () => { + const db = getGlobalDB() + for (const userId of userIds) { + const user = structures.users.user({ _id: userId }) + await db.put(user) + users.push(user) + } + }) + }) + + beforeEach(async () => { + jest.clearAllMocks() + + const redisClient = await redis.getUserClient() + await redisClient.clear() + }) + + it("when no user is in cache, all of them are retrieved from db", async () => { + const usersToRequest = _.sampleSize(users, 5) + + const userIdsToRequest = usersToRequest.map(x => x._id!) + + jest.spyOn(UserDB, "bulkGet") + + const results = await config.doInTenant(() => getUsers(userIdsToRequest)) + + expect(results.users).toHaveLength(5) + expect(results).toEqual({ + users: usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })), + }) + + expect(UserDB.bulkGet).toBeCalledTimes(1) + expect(UserDB.bulkGet).toBeCalledWith(userIdsToRequest) + }) + + it("on a second all, all of them are retrieved from cache", async () => { + const usersToRequest = _.sampleSize(users, 5) + + const userIdsToRequest = usersToRequest.map(x => x._id!) + + jest.spyOn(UserDB, "bulkGet") + + await config.doInTenant(() => getUsers(userIdsToRequest)) + const resultsFromCache = await config.doInTenant(() => + getUsers(userIdsToRequest) + ) + + expect(resultsFromCache.users).toHaveLength(5) + expect(resultsFromCache).toEqual({ + users: expect.arrayContaining( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ), + }) + + expect(UserDB.bulkGet).toBeCalledTimes(1) + }) + + it("when some users are cached, only the missing ones are retrieved from db", async () => { + const usersToRequest = _.sampleSize(users, 5) + + const userIdsToRequest = usersToRequest.map(x => x._id!) + + jest.spyOn(UserDB, "bulkGet") + + await config.doInTenant(() => + getUsers([userIdsToRequest[0], userIdsToRequest[3]]) + ) + ;(UserDB.bulkGet as jest.Mock).mockClear() + + const results = await config.doInTenant(() => getUsers(userIdsToRequest)) + + expect(results.users).toHaveLength(5) + expect(results).toEqual({ + users: expect.arrayContaining( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ), + }) + + expect(UserDB.bulkGet).toBeCalledTimes(1) + expect(UserDB.bulkGet).toBeCalledWith([ + userIdsToRequest[1], + userIdsToRequest[2], + userIdsToRequest[4], + ]) + }) + + it("requesting existing and unexisting ids will return found ones", async () => { + const usersToRequest = _.sampleSize(users, 3) + const missingIds = [generator.guid(), generator.guid()] + + const userIdsToRequest = _.shuffle([ + ...missingIds, + ...usersToRequest.map(x => x._id!), + ]) + + const results = await config.doInTenant(() => getUsers(userIdsToRequest)) + + expect(results.users).toHaveLength(3) + expect(results).toEqual({ + users: expect.arrayContaining( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ), + notFoundIds: expect.arrayContaining(missingIds), + }) + }) + }) +}) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index e2af78adfd..b3fd7c08cd 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -6,6 +6,7 @@ import env from "../environment" import * as accounts from "../accounts" import { UserDB } from "../users" import { sdk } from "@budibase/shared-core" +import { User } from "@budibase/types" const EXPIRY_SECONDS = 3600 @@ -27,6 +28,35 @@ async function populateFromDB(userId: string, tenantId: string) { return user } +async function populateUsersFromDB( + userIds: string[] +): Promise<{ users: User[]; notFoundIds?: string[] }> { + const getUsersResponse = await UserDB.bulkGet(userIds) + + // Handle missed user ids + const notFoundIds = userIds.filter((uid, i) => !getUsersResponse[i]) + + const users = getUsersResponse.filter(x => x) + + await Promise.all( + users.map(async (user: any) => { + user.budibaseAccess = true + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + const account = await accounts.getAccount(user.email) + if (account) { + user.account = account + user.accountPortalAccess = true + } + } + }) + ) + + if (notFoundIds.length) { + return { users, notFoundIds } + } + return { users } +} + /** * Get the requested user by id. * Use redis cache to first read the user. @@ -77,6 +107,36 @@ export async function getUser( return user } +/** + * Get the requested users by id. + * Use redis cache to first read the users. + * If not present fallback to loading the users directly and re-caching. + * @param {*} userIds the ids of the user to get + * @param {*} tenantId the tenant of the users to get + * @returns + */ +export async function getUsers( + userIds: string[] +): Promise<{ users: User[]; notFoundIds?: string[] }> { + const client = await redis.getUserClient() + // try cache + let usersFromCache = await client.bulkGet(userIds) + const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid]) + const users = Object.values(usersFromCache) + let notFoundIds + + if (missingUsersFromCache.length) { + const usersFromDb = await populateUsersFromDB(missingUsersFromCache) + + notFoundIds = usersFromDb.notFoundIds + for (const userToCache of usersFromDb.users) { + await client.store(userToCache._id!, userToCache, EXPIRY_SECONDS) + } + users.push(...usersFromDb.users) + } + return { users, notFoundIds: notFoundIds } +} + export async function invalidateUser(userId: string) { const client = await redis.getUserClient() await client.delete(userId) diff --git a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts index 484a118cbd..c3ddf220e6 100644 --- a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts +++ b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts @@ -102,6 +102,7 @@ describe("sso", () => { // modified external id to match user format ssoUser._id = "us_" + details.userId + delete ssoUser.userId // new sso user won't have a password delete ssoUser.password diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 5056a5d549..78817d0aa0 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -250,7 +250,7 @@ class RedisWrapper { const prefixedKeys = keys.map(key => addDbPrefix(db, key)) let response = await this.getClient().mget(prefixedKeys) if (Array.isArray(response)) { - let final: any = {} + let final: Record = {} let count = 0 for (let result of response) { if (result) { diff --git a/packages/backend-core/tests/core/utilities/structures/shared.ts b/packages/backend-core/tests/core/utilities/structures/shared.ts deleted file mode 100644 index de0e19486c..0000000000 --- a/packages/backend-core/tests/core/utilities/structures/shared.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { User } from "@budibase/types" -import { generator } from "./generator" -import { uuid } from "./common" - -export const newEmail = () => { - return `${uuid()}@test.com` -} - -export const user = (userProps?: any): User => { - return { - email: newEmail(), - password: "test", - roles: { app_test: "admin" }, - firstName: generator.first(), - lastName: generator.last(), - pictureUrl: "http://test.com", - ...userProps, - } -} diff --git a/packages/backend-core/tests/core/utilities/structures/sso.ts b/packages/backend-core/tests/core/utilities/structures/sso.ts index 4d13635f09..2e3af712a9 100644 --- a/packages/backend-core/tests/core/utilities/structures/sso.ts +++ b/packages/backend-core/tests/core/utilities/structures/sso.ts @@ -13,8 +13,7 @@ import { } from "@budibase/types" import { generator } from "./generator" import { email, uuid } from "./common" -import * as shared from "./shared" -import { user } from "./shared" +import * as users from "./users" import sample from "lodash/sample" export function OAuth(): OAuth2 { @@ -26,7 +25,7 @@ export function OAuth(): OAuth2 { export function authDetails(userDoc?: User): SSOAuthDetails { if (!userDoc) { - userDoc = user() + userDoc = users.user() } const userId = userDoc._id || uuid() @@ -52,7 +51,7 @@ export function providerType(): SSOProviderType { export function ssoProfile(user?: User): SSOProfile { if (!user) { - user = shared.user() + user = users.user() } return { id: user._id!, diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 0a4f2e8b54..420a9fde0e 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -4,11 +4,33 @@ import { BuilderUser, SSOAuthDetails, SSOUser, + User, } from "@budibase/types" -import { user } from "./shared" import { authDetails } from "./sso" +import { uuid } from "./common" +import { generator } from "./generator" +import { tenant } from "." +import { generateGlobalUserID } from "../../../../src/docIds" -export { user, newEmail } from "./shared" +export const newEmail = () => { + return `${uuid()}@test.com` +} + +export const user = (userProps?: Partial>): User => { + const userId = userProps?._id || generateGlobalUserID() + return { + _id: userId, + userId, + email: newEmail(), + password: "test", + roles: { app_test: "admin" }, + firstName: generator.first(), + lastName: generator.last(), + pictureUrl: "http://test.com", + tenantId: tenant.id(), + ...userProps, + } +} export const adminUser = (userProps?: any): AdminUser => { return {