Merge branch 'feature/user-column-type' into BUDI-7455/populate_user_refs_on_get

.github/workflows/budibase_ci.yml

@@ -25,6 +25,13 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-android: 'true'
+          remove-dotnet: 'true'
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.7-alpha.2",
+  "version": "2.10.9-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/cache/tests/user.spec.ts

@@ -0,0 +1,135 @@
+import { User } from "@budibase/types"
+import { cache, tenancy } from "../.."
+import { generator, structures } from "../../../tests"
+import { DBTestConfiguration } from "../../../tests/extra"
+import { getUsers } from "../user"
+import { getGlobalDB, getGlobalDBName } from "../../context"
+import _ from "lodash"
+import { getDB } from "../../db"
+import type * as TenancyType from "../../tenancy"
+import * as redis from "../../redis/init"
+
+const config = new DBTestConfiguration()
+
+// This mock is required to ensure that getTenantDB returns always as a singleton.
+// This will allow us to spy on the db
+const staticDb = getDB(getGlobalDBName(config.tenantId))
+jest.mock("../../tenancy", (): typeof TenancyType => ({
+  ...jest.requireActual("../../tenancy"),
+  getTenantDB: jest.fn().mockImplementation(() => staticDb),
+}))
+
+describe("user cache", () => {
+  describe("getUsers", () => {
+    const users: User[] = []
+    beforeAll(async () => {
+      const userCount = 10
+      const userIds = generator.arrayOf(() => generator.guid(), {
+        min: userCount,
+        max: userCount,
+      })
+
+      await config.doInTenant(async () => {
+        const db = getGlobalDB()
+        for (const userId of userIds) {
+          const user = structures.users.user({ _id: userId })
+          await db.put(user)
+          users.push(user)
+        }
+      })
+    })
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+
+      const redisClient = await redis.getUserClient()
+      await redisClient.clear()
+    })
+
+    it("when no user is in cache, all of them are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(staticDb, "allDocs")
+
+      const results = await getUsers(userIdsToRequest, config.tenantId)
+
+      expect(results).toHaveLength(5)
+      expect(results).toEqual(
+        usersToRequest.map(u => ({
+          ...u,
+          budibaseAccess: true,
+          _rev: expect.any(String),
+        }))
+      )
+
+      expect(tenancy.getTenantDB).toBeCalledTimes(1)
+      expect(tenancy.getTenantDB).toBeCalledWith(config.tenantId)
+      expect(staticDb.allDocs).toBeCalledTimes(1)
+      expect(staticDb.allDocs).toBeCalledWith({
+        keys: userIdsToRequest,
+        include_docs: true,
+        limit: 5,
+      })
+    })
+
+    it("on a second all, all of them are retrieved from cache", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(staticDb, "allDocs")
+
+      await getUsers(userIdsToRequest, config.tenantId)
+      const resultsFromCache = await getUsers(userIdsToRequest, config.tenantId)
+
+      expect(resultsFromCache).toHaveLength(5)
+      expect(resultsFromCache).toEqual(
+        expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        )
+      )
+
+      expect(staticDb.allDocs).toBeCalledTimes(1)
+    })
+
+    it("when some users are cached, only the missing ones are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(staticDb, "allDocs")
+
+      await getUsers(
+        [userIdsToRequest[0], userIdsToRequest[3]],
+        config.tenantId
+      )
+      ;(staticDb.allDocs as jest.Mock).mockClear()
+
+      const results = await getUsers(userIdsToRequest, config.tenantId)
+
+      expect(results).toHaveLength(5)
+      expect(results).toEqual(
+        expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        )
+      )
+
+      expect(staticDb.allDocs).toBeCalledTimes(1)
+      expect(staticDb.allDocs).toBeCalledWith({
+        keys: [userIdsToRequest[1], userIdsToRequest[2], userIdsToRequest[4]],
+        include_docs: true,
+        limit: 3,
+      })
+    })
+  })
+})

packages/backend-core/src/cache/user.ts

@@ -27,6 +27,31 @@ async function populateFromDB(userId: string, tenantId: string) {
   return user
 }
 
+async function populateUsersFromDB(userIds: string[], tenantId: string) {
+  const db = tenancy.getTenantDB(tenantId)
+  const allDocsResponse = await db.allDocs<any>({
+    keys: userIds,
+    include_docs: true,
+    limit: userIds.length,
+  })
+
+  const users = allDocsResponse.rows.map(r => r.doc)
+  await Promise.all(
+    users.map(async user => {
+      user.budibaseAccess = true
+      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+        const account = await accounts.getAccount(user.email)
+        if (account) {
+          user.account = account
+          user.accountPortalAccess = true
+        }
+      }
+    })
+  )
+
+  return users
+}
+
 /**
  * Get the requested user by id.
  * Use redis cache to first read the user.
@@ -77,6 +102,34 @@ export async function getUser(
   return user
 }
 
+/**
+ * Get the requested users by id.
+ * Use redis cache to first read the users.
+ * If not present fallback to loading the users directly and re-caching.
+ * @param {*} userIds the ids of the user to get
+ * @param {*} tenantId the tenant of the users to get
+ * @returns
+ */
+export async function getUsers(userIds: string[], tenantId: string) {
+  const client = await redis.getUserClient()
+  // try cache
+  let usersFromCache = await client.bulkGet(userIds)
+  const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
+  const users = Object.values(usersFromCache)
+
+  if (missingUsersFromCache.length) {
+    const usersFromDb = await populateUsersFromDB(
+      missingUsersFromCache,
+      tenantId
+    )
+    for (const userToCache of usersFromDb) {
+      await client.store(userToCache._id, userToCache, EXPIRY_SECONDS)
+    }
+    users.push(...usersFromDb)
+  }
+  return users
+}
+
 export async function invalidateUser(userId: string) {
   const client = await redis.getUserClient()
   await client.delete(userId)

packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts

@@ -102,6 +102,7 @@ describe("sso", () => {
 
           // modified external id to match user format
           ssoUser._id = "us_" + details.userId
+          delete ssoUser.userId
 
           // new sso user won't have a password
           delete ssoUser.password

packages/backend-core/src/redis/redis.ts

@@ -250,7 +250,7 @@ class RedisWrapper {
     const prefixedKeys = keys.map(key => addDbPrefix(db, key))
     let response = await this.getClient().mget(prefixedKeys)
     if (Array.isArray(response)) {
-      let final: any = {}
+      let final: Record<string, any> = {}
       let count = 0
       for (let result of response) {
         if (result) {

packages/backend-core/tests/core/utilities/structures/shared.ts

@@ -1,21 +0,0 @@
-import { User } from "@budibase/types"
-import { generator } from "./generator"
-import { uuid } from "./common"
-import { tenant } from "."
-
-export const newEmail = () => {
-  return `${uuid()}@test.com`
-}
-
-export const user = (userProps?: Partial<User>): User => {
-  return {
-    email: newEmail(),
-    password: "test",
-    roles: { app_test: "admin" },
-    firstName: generator.first(),
-    lastName: generator.last(),
-    pictureUrl: "http://test.com",
-    tenantId: tenant.id(),
-    ...userProps,
-  }
-}

packages/backend-core/tests/core/utilities/structures/sso.ts

@@ -13,8 +13,7 @@ import {
 } from "@budibase/types"
 import { generator } from "./generator"
 import { email, uuid } from "./common"
-import * as shared from "./shared"
-import { user } from "./shared"
+import * as users from "./users"
 import sample from "lodash/sample"
 
 export function OAuth(): OAuth2 {
@@ -26,7 +25,7 @@ export function OAuth(): OAuth2 {
 
 export function authDetails(userDoc?: User): SSOAuthDetails {
   if (!userDoc) {
-    userDoc = user()
+    userDoc = users.user()
   }
 
   const userId = userDoc._id || uuid()
@@ -52,7 +51,7 @@ export function providerType(): SSOProviderType {
 
 export function ssoProfile(user?: User): SSOProfile {
   if (!user) {
-    user = shared.user()
+    user = users.user()
   }
   return {
     id: user._id!,

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -4,11 +4,33 @@ import {
   BuilderUser,
   SSOAuthDetails,
   SSOUser,
+  User,
 } from "@budibase/types"
-import { user } from "./shared"
 import { authDetails } from "./sso"
+import { uuid } from "./common"
+import { generator } from "./generator"
+import { tenant } from "."
+import { generateGlobalUserID } from "../../../../src/docIds"
 
-export { user, newEmail } from "./shared"
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
+  const userId = userProps?._id || generateGlobalUserID()
+  return {
+    _id: userId,
+    userId,
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    tenantId: tenant.id(),
+    ...userProps,
+  }
+}
 
 export const adminUser = (userProps?: any): AdminUser => {
   return {

packages/bbui/package.json

@@ -82,7 +82,7 @@
     "@spectrum-css/typography": "3.0.1",
     "@spectrum-css/underlay": "2.0.9",
     "@spectrum-css/vars": "3.0.1",
-    "dayjs": "^1.10.4",
+    "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
     "svelte-flatpickr": "3.2.3",
     "svelte-portal": "^1.0.0",

packages/builder/package.json

@@ -69,7 +69,7 @@
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",
-    "dayjs": "^1.11.2",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "fast-json-patch": "^3.1.1",
     "lodash": "4.17.21",

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -126,7 +126,7 @@
           user,
           prodAppId
         )
-        const isAppBuilder = sdk.users.hasAppBuilderPermissions(user, prodAppId)
+        const isAppBuilder = user.builder?.apps?.includes(prodAppId)
         let role
         if (isAdminOrGlobalBuilder) {
           role = Constants.Roles.ADMIN

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -111,7 +111,7 @@
       })
     }
     return availableApps.map(app => {
-      const prodAppId = apps.getProdAppID(app.appId)
+      const prodAppId = apps.getProdAppID(app.devId)
       return {
         name: app.name,
         devId: app.devId,

packages/client/package.json

@@ -33,7 +33,7 @@
     "@spectrum-css/typography": "^3.0.2",
     "@spectrum-css/vars": "^3.0.1",
     "apexcharts": "^3.22.1",
-    "dayjs": "^1.10.5",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",

packages/frontend-core/package.json

@@ -8,7 +8,7 @@
   "dependencies": {
     "@budibase/bbui": "0.0.0",
     "@budibase/shared-core": "0.0.0",
-    "dayjs": "^1.11.7",
+    "dayjs": "^1.10.8",
     "lodash": "^4.17.21",
     "socket.io-client": "^4.6.1",
     "svelte": "^3.46.2"

packages/frontend-core/src/components/grid/cells/DateCell.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { dayjs } from "dayjs"
+  import dayjs from "dayjs"
   import { CoreDatePicker, Icon } from "@budibase/bbui"
   import { onMount } from "svelte"
 

packages/server/scripts/integrations/postgres/init.sql

@@ -1,7 +1,7 @@
 SELECT 'CREATE DATABASE main'
 WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
 CREATE SCHEMA "test-1";
-CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer');
+CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer', 'support');
 CREATE TABLE Persons (
     PersonID SERIAL PRIMARY KEY,
     LastName varchar(255),
@@ -51,6 +51,7 @@ CREATE TABLE CompositeTable (
 );
 INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa');
 INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer');
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0);
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE);
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE);
 INSERT INTO Products (ProductName) VALUES ('Computers');

packages/server/src/api/controllers/row/external.ts

@@ -28,20 +28,8 @@ export async function handleRequest(
 ) {
   // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
   if (opts && opts.filters) {
-    for (let filterField of NoEmptyFilterStrings) {
-      if (!opts.filters[filterField]) {
-        continue
-      }
-      // @ts-ignore
-      for (let [key, value] of Object.entries(opts.filters[filterField])) {
-        if (!value || value === "") {
-          // @ts-ignore
-          delete opts.filters[filterField][key]
-        }
-      }
-    }
+    opts.filters = utils.removeEmptyFilters(opts.filters)
   }
-
   if (
     !dataFilters.hasFilters(opts?.filters) &&
     opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE

packages/server/src/api/controllers/row/tests/utils.spec.ts

@@ -0,0 +1,21 @@
+import * as utils from "../utils"
+
+describe("removeEmptyFilters", () => {
+  it("0 should not be removed", () => {
+    const filters = utils.removeEmptyFilters({
+      equal: {
+        column: 0,
+      },
+    })
+    expect((filters.equal as any).column).toBe(0)
+  })
+
+  it("empty string should be removed", () => {
+    const filters = utils.removeEmptyFilters({
+      equal: {
+        column: "",
+      },
+    })
+    expect(Object.values(filters.equal as any).length).toBe(0)
+  })
+})

packages/server/src/api/controllers/row/utils.ts

@@ -1,8 +1,15 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import { context } from "@budibase/backend-core"
-import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types"
-import { FieldTypes } from "../../../constants"
+import {
+  Ctx,
+  FieldType,
+  Row,
+  SearchFilters,
+  Table,
+  UserCtx,
+} from "@budibase/types"
+import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
 import sdk from "../../../sdk"
 
 import validateJs from "validate.js"
@@ -139,3 +146,32 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
+
+// don't do a pure falsy check, as 0 is included
+// https://github.com/Budibase/budibase/issues/10118
+export function removeEmptyFilters(filters: SearchFilters) {
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!filters[filterField]) {
+      continue
+    }
+
+    for (let filterType of Object.keys(filters)) {
+      if (filterType !== filterField) {
+        continue
+      }
+      // don't know which one we're checking, type could be anything
+      const value = filters[filterType] as unknown
+      if (typeof value === "object") {
+        for (let [key, value] of Object.entries(
+          filters[filterType] as object
+        )) {
+          if (value == null || value === "") {
+            // @ts-ignore
+            delete filters[filterField][key]
+          }
+        }
+      }
+    }
+  }
+  return filters
+}

packages/server/src/integrations/mysql.ts

@@ -158,6 +158,12 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     ) {
       config.ssl.rejectUnauthorized = config.rejectUnauthorized
     }
+    // The MySQL library we use doesn't directly document the parameters that can be passed in the ssl
+    // object, it instead points to an older library that it says it is mostly API compatible with, that
+    // older library actually documents what parameters can be passed in the ssl object.
+    // https://github.com/sidorares/node-mysql2#api-and-configuration
+    // https://github.com/mysqljs/mysql#ssl-options
+
     // @ts-ignore
     delete config.rejectUnauthorized
     this.config = {

packages/string-templates/package.json

@@ -26,7 +26,7 @@
   },
   "dependencies": {
     "@budibase/handlebars-helpers": "^0.11.9",
-    "dayjs": "^1.10.4",
+    "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
     "handlebars-utils": "^1.0.6",
     "lodash": "^4.17.20",

packages/worker/src/api/routes/global/tests/scim.spec.ts

@@ -10,6 +10,8 @@ import {
 import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"
 
+// this test can 409 - retries reduce issues with this
+jest.retryTimes(2)
 jest.setTimeout(30000)
 
 mocks.licenses.useScimIntegration()

yarn.lock

@@ -9779,11 +9779,16 @@ dateformat@^4.5.1, dateformat@^4.6.3:
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5"
   integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==
 
-dayjs@^1.10.4, dayjs@^1.10.5, dayjs@^1.11.2, dayjs@^1.11.7:
+dayjs@^1.10.4, dayjs@^1.10.5:
   version "1.11.7"
   resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2"
   integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==
 
+dayjs@^1.10.8:
+  version "1.11.9"
+  resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.9.tgz#9ca491933fadd0a60a2c19f6c237c03517d71d1a"
+  integrity sha512-QvzAURSbQ0pKdIye2txOzNaHmxtUBXerpY0FJsFXUMKbIZeFm5ht1LS/jFsrncjnmtv8HsG0W2g6c0zUjZWmpA==
+
 dd-trace@3.13.2:
   version "3.13.2"
   resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-3.13.2.tgz#95b1ec480ab9ac406e1da7591a8c6f678d3799fd"