diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index d670e222d3..fc35575ec6 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -25,6 +25,13 @@ jobs: lint: runs-on: ubuntu-latest steps: + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 35000 + swap-size-mb: 1024 + remove-android: 'true' + remove-dotnet: 'true' - name: Checkout repo and submodules uses: actions/checkout@v3 if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' diff --git a/lerna.json b/lerna.json index a3a8bda618..a57875986a 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.7-alpha.2", + "version": "2.10.9-alpha.3", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/src/cache/tests/user.spec.ts b/packages/backend-core/src/cache/tests/user.spec.ts new file mode 100644 index 0000000000..45537694c7 --- /dev/null +++ b/packages/backend-core/src/cache/tests/user.spec.ts @@ -0,0 +1,135 @@ +import { User } from "@budibase/types" +import { cache, tenancy } from "../.." +import { generator, structures } from "../../../tests" +import { DBTestConfiguration } from "../../../tests/extra" +import { getUsers } from "../user" +import { getGlobalDB, getGlobalDBName } from "../../context" +import _ from "lodash" +import { getDB } from "../../db" +import type * as TenancyType from "../../tenancy" +import * as redis from "../../redis/init" + +const config = new DBTestConfiguration() + +// This mock is required to ensure that getTenantDB returns always as a singleton. +// This will allow us to spy on the db +const staticDb = getDB(getGlobalDBName(config.tenantId)) +jest.mock("../../tenancy", (): typeof TenancyType => ({ + ...jest.requireActual("../../tenancy"), + getTenantDB: jest.fn().mockImplementation(() => staticDb), +})) + +describe("user cache", () => { + describe("getUsers", () => { + const users: User[] = [] + beforeAll(async () => { + const userCount = 10 + const userIds = generator.arrayOf(() => generator.guid(), { + min: userCount, + max: userCount, + }) + + await config.doInTenant(async () => { + const db = getGlobalDB() + for (const userId of userIds) { + const user = structures.users.user({ _id: userId }) + await db.put(user) + users.push(user) + } + }) + }) + + beforeEach(async () => { + jest.clearAllMocks() + + const redisClient = await redis.getUserClient() + await redisClient.clear() + }) + + it("when no user is in cache, all of them are retrieved from db", async () => { + const usersToRequest = _.sampleSize(users, 5) + + const userIdsToRequest = usersToRequest.map(x => x._id!) + + jest.spyOn(staticDb, "allDocs") + + const results = await getUsers(userIdsToRequest, config.tenantId) + + expect(results).toHaveLength(5) + expect(results).toEqual( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ) + + expect(tenancy.getTenantDB).toBeCalledTimes(1) + expect(tenancy.getTenantDB).toBeCalledWith(config.tenantId) + expect(staticDb.allDocs).toBeCalledTimes(1) + expect(staticDb.allDocs).toBeCalledWith({ + keys: userIdsToRequest, + include_docs: true, + limit: 5, + }) + }) + + it("on a second all, all of them are retrieved from cache", async () => { + const usersToRequest = _.sampleSize(users, 5) + + const userIdsToRequest = usersToRequest.map(x => x._id!) + + jest.spyOn(staticDb, "allDocs") + + await getUsers(userIdsToRequest, config.tenantId) + const resultsFromCache = await getUsers(userIdsToRequest, config.tenantId) + + expect(resultsFromCache).toHaveLength(5) + expect(resultsFromCache).toEqual( + expect.arrayContaining( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ) + ) + + expect(staticDb.allDocs).toBeCalledTimes(1) + }) + + it("when some users are cached, only the missing ones are retrieved from db", async () => { + const usersToRequest = _.sampleSize(users, 5) + + const userIdsToRequest = usersToRequest.map(x => x._id!) + + jest.spyOn(staticDb, "allDocs") + + await getUsers( + [userIdsToRequest[0], userIdsToRequest[3]], + config.tenantId + ) + ;(staticDb.allDocs as jest.Mock).mockClear() + + const results = await getUsers(userIdsToRequest, config.tenantId) + + expect(results).toHaveLength(5) + expect(results).toEqual( + expect.arrayContaining( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ) + ) + + expect(staticDb.allDocs).toBeCalledTimes(1) + expect(staticDb.allDocs).toBeCalledWith({ + keys: [userIdsToRequest[1], userIdsToRequest[2], userIdsToRequest[4]], + include_docs: true, + limit: 3, + }) + }) + }) +}) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index e2af78adfd..9742b41b65 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -27,6 +27,31 @@ async function populateFromDB(userId: string, tenantId: string) { return user } +async function populateUsersFromDB(userIds: string[], tenantId: string) { + const db = tenancy.getTenantDB(tenantId) + const allDocsResponse = await db.allDocs({ + keys: userIds, + include_docs: true, + limit: userIds.length, + }) + + const users = allDocsResponse.rows.map(r => r.doc) + await Promise.all( + users.map(async user => { + user.budibaseAccess = true + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + const account = await accounts.getAccount(user.email) + if (account) { + user.account = account + user.accountPortalAccess = true + } + } + }) + ) + + return users +} + /** * Get the requested user by id. * Use redis cache to first read the user. @@ -77,6 +102,34 @@ export async function getUser( return user } +/** + * Get the requested users by id. + * Use redis cache to first read the users. + * If not present fallback to loading the users directly and re-caching. + * @param {*} userIds the ids of the user to get + * @param {*} tenantId the tenant of the users to get + * @returns + */ +export async function getUsers(userIds: string[], tenantId: string) { + const client = await redis.getUserClient() + // try cache + let usersFromCache = await client.bulkGet(userIds) + const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid]) + const users = Object.values(usersFromCache) + + if (missingUsersFromCache.length) { + const usersFromDb = await populateUsersFromDB( + missingUsersFromCache, + tenantId + ) + for (const userToCache of usersFromDb) { + await client.store(userToCache._id, userToCache, EXPIRY_SECONDS) + } + users.push(...usersFromDb) + } + return users +} + export async function invalidateUser(userId: string) { const client = await redis.getUserClient() await client.delete(userId) diff --git a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts index 484a118cbd..c3ddf220e6 100644 --- a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts +++ b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts @@ -102,6 +102,7 @@ describe("sso", () => { // modified external id to match user format ssoUser._id = "us_" + details.userId + delete ssoUser.userId // new sso user won't have a password delete ssoUser.password diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 5056a5d549..78817d0aa0 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -250,7 +250,7 @@ class RedisWrapper { const prefixedKeys = keys.map(key => addDbPrefix(db, key)) let response = await this.getClient().mget(prefixedKeys) if (Array.isArray(response)) { - let final: any = {} + let final: Record = {} let count = 0 for (let result of response) { if (result) { diff --git a/packages/backend-core/tests/core/utilities/structures/shared.ts b/packages/backend-core/tests/core/utilities/structures/shared.ts deleted file mode 100644 index 6e6292e3f4..0000000000 --- a/packages/backend-core/tests/core/utilities/structures/shared.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { User } from "@budibase/types" -import { generator } from "./generator" -import { uuid } from "./common" -import { tenant } from "." - -export const newEmail = () => { - return `${uuid()}@test.com` -} - -export const user = (userProps?: Partial): User => { - return { - email: newEmail(), - password: "test", - roles: { app_test: "admin" }, - firstName: generator.first(), - lastName: generator.last(), - pictureUrl: "http://test.com", - tenantId: tenant.id(), - ...userProps, - } -} diff --git a/packages/backend-core/tests/core/utilities/structures/sso.ts b/packages/backend-core/tests/core/utilities/structures/sso.ts index 4d13635f09..2e3af712a9 100644 --- a/packages/backend-core/tests/core/utilities/structures/sso.ts +++ b/packages/backend-core/tests/core/utilities/structures/sso.ts @@ -13,8 +13,7 @@ import { } from "@budibase/types" import { generator } from "./generator" import { email, uuid } from "./common" -import * as shared from "./shared" -import { user } from "./shared" +import * as users from "./users" import sample from "lodash/sample" export function OAuth(): OAuth2 { @@ -26,7 +25,7 @@ export function OAuth(): OAuth2 { export function authDetails(userDoc?: User): SSOAuthDetails { if (!userDoc) { - userDoc = user() + userDoc = users.user() } const userId = userDoc._id || uuid() @@ -52,7 +51,7 @@ export function providerType(): SSOProviderType { export function ssoProfile(user?: User): SSOProfile { if (!user) { - user = shared.user() + user = users.user() } return { id: user._id!, diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 0a4f2e8b54..420a9fde0e 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -4,11 +4,33 @@ import { BuilderUser, SSOAuthDetails, SSOUser, + User, } from "@budibase/types" -import { user } from "./shared" import { authDetails } from "./sso" +import { uuid } from "./common" +import { generator } from "./generator" +import { tenant } from "." +import { generateGlobalUserID } from "../../../../src/docIds" -export { user, newEmail } from "./shared" +export const newEmail = () => { + return `${uuid()}@test.com` +} + +export const user = (userProps?: Partial>): User => { + const userId = userProps?._id || generateGlobalUserID() + return { + _id: userId, + userId, + email: newEmail(), + password: "test", + roles: { app_test: "admin" }, + firstName: generator.first(), + lastName: generator.last(), + pictureUrl: "http://test.com", + tenantId: tenant.id(), + ...userProps, + } +} export const adminUser = (userProps?: any): AdminUser => { return { diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 0b87960ab5..dc6c910be8 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -82,7 +82,7 @@ "@spectrum-css/typography": "3.0.1", "@spectrum-css/underlay": "2.0.9", "@spectrum-css/vars": "3.0.1", - "dayjs": "^1.10.4", + "dayjs": "^1.10.8", "easymde": "^2.16.1", "svelte-flatpickr": "3.2.3", "svelte-portal": "^1.0.0", diff --git a/packages/builder/package.json b/packages/builder/package.json index a6bf81201d..43f1ae3bff 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -69,7 +69,7 @@ "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", "codemirror": "^5.59.0", - "dayjs": "^1.11.2", + "dayjs": "^1.10.8", "downloadjs": "1.4.7", "fast-json-patch": "^3.1.1", "lodash": "4.17.21", diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte index 7c4d3db7ce..c93a41f541 100644 --- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte @@ -126,7 +126,7 @@ user, prodAppId ) - const isAppBuilder = sdk.users.hasAppBuilderPermissions(user, prodAppId) + const isAppBuilder = user.builder?.apps?.includes(prodAppId) let role if (isAdminOrGlobalBuilder) { role = Constants.Roles.ADMIN diff --git a/packages/builder/src/pages/builder/portal/users/users/[userId].svelte b/packages/builder/src/pages/builder/portal/users/users/[userId].svelte index 2a74cd9de5..ec10ec8316 100644 --- a/packages/builder/src/pages/builder/portal/users/users/[userId].svelte +++ b/packages/builder/src/pages/builder/portal/users/users/[userId].svelte @@ -111,7 +111,7 @@ }) } return availableApps.map(app => { - const prodAppId = apps.getProdAppID(app.appId) + const prodAppId = apps.getProdAppID(app.devId) return { name: app.name, devId: app.devId, diff --git a/packages/client/package.json b/packages/client/package.json index a5ee304610..698c7bd929 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -33,7 +33,7 @@ "@spectrum-css/typography": "^3.0.2", "@spectrum-css/vars": "^3.0.1", "apexcharts": "^3.22.1", - "dayjs": "^1.10.5", + "dayjs": "^1.10.8", "downloadjs": "1.4.7", "html5-qrcode": "^2.2.1", "leaflet": "^1.7.1", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index ca7e0a6d2b..1f15bb72c5 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -8,7 +8,7 @@ "dependencies": { "@budibase/bbui": "0.0.0", "@budibase/shared-core": "0.0.0", - "dayjs": "^1.11.7", + "dayjs": "^1.10.8", "lodash": "^4.17.21", "socket.io-client": "^4.6.1", "svelte": "^3.46.2" diff --git a/packages/frontend-core/src/components/grid/cells/DateCell.svelte b/packages/frontend-core/src/components/grid/cells/DateCell.svelte index 9144f5b769..53b159ee30 100644 --- a/packages/frontend-core/src/components/grid/cells/DateCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/DateCell.svelte @@ -1,5 +1,5 @@