From 8e8c913dfb97d91d860d01e7c653da8c398873cb Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 6 Sep 2023 18:17:44 +0100
Subject: [PATCH 01/46] Updating the yarn dev process to not include a removal
 of the dist directory as part of the streamed watchers - this seems to create
 a problem for shared core and moving this up a level to a parallel run before
 the watchers massively improves stability.

---
 package.json                      | 2 +-
 packages/server/package.json      | 2 +-
 packages/shared-core/package.json | 2 +-
 packages/types/package.json       | 2 +-
 packages/worker/package.json      | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index b0c428fe70..b263fb6585 100644
--- a/package.json
+++ b/package.json
@@ -48,7 +48,7 @@
     "kill-builder": "kill-port 3000",
     "kill-server": "kill-port 4001 4002",
     "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all && lerna run --stream dev:builder",
+    "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev:builder",
     "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
diff --git a/packages/server/package.json b/packages/server/package.json
index ecf31175dc..39f52342a4 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -26,7 +26,7 @@
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon",
+    "dev:builder": "yarn run dev:stack:up && nodemon",
     "dev:built": "yarn run dev:stack:up && yarn run run:docker",
     "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json
index 8edea4870a..160a42f086 100644
--- a/packages/shared-core/package.json
+++ b/packages/shared-core/package.json
@@ -10,7 +10,7 @@
     "prebuild": "rimraf dist/",
     "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "dev:builder": "yarn prebuild && tsc -p tsconfig.json --watch --preserveWatchOutput",
+    "dev:builder": "tsc -p tsconfig.json --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null"
   },
   "dependencies": {
diff --git a/packages/types/package.json b/packages/types/package.json
index bb0163ae13..919cc736fc 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -10,7 +10,7 @@
     "prebuild": "rimraf dist/",
     "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "dev:builder": "yarn prebuild && tsc -p tsconfig.json --watch --preserveWatchOutput",
+    "dev:builder": "tsc -p tsconfig.json --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null"
   },
   "jest": {},
diff --git a/packages/worker/package.json b/packages/worker/package.json
index 3b66df264e..15ab1c1ff9 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -22,7 +22,7 @@
     "predocker": "yarn build && cp ../../yarn.lock ./dist/",
     "build:docker": "yarn predocker && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
-    "dev:builder": "npm run dev:stack:init && rimraf dist/ && nodemon",
+    "dev:builder": "npm run dev:stack:init && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watch",

From aeac7aaa0c0f8845bbfd187e207795c25af7237d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 12 Sep 2023 13:54:39 +0100
Subject: [PATCH 02/46] Making sure that not all falsy values are considered
 invalid filter inputs.

---
 .../server/scripts/integrations/postgres/init.sql   |  3 ++-
 packages/server/src/api/controllers/row/external.ts |  4 ++--
 packages/server/src/api/controllers/row/utils.ts    |  6 ++++++
 yarn.lock                                           | 13 +++++++++++++
 4 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/packages/server/scripts/integrations/postgres/init.sql b/packages/server/scripts/integrations/postgres/init.sql
index 6cb3f51269..f89ad2812d 100644
--- a/packages/server/scripts/integrations/postgres/init.sql
+++ b/packages/server/scripts/integrations/postgres/init.sql
@@ -1,7 +1,7 @@
 SELECT 'CREATE DATABASE main'
 WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
 CREATE SCHEMA "test-1";
-CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer');
+CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer', 'support');
 CREATE TABLE Persons (
     PersonID SERIAL PRIMARY KEY,
     LastName varchar(255),
@@ -51,6 +51,7 @@ CREATE TABLE CompositeTable (
 );
 INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa');
 INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer');
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0);
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE);
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE);
 INSERT INTO Products (ProductName) VALUES ('Computers');
diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 491cbfa8f9..44921a5114 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -32,12 +32,12 @@ export async function handleRequest(
       }
       // @ts-ignore
       for (let [key, value] of Object.entries(opts.filters[filterField])) {
-        if (!value || value === "") {
+        if (utils.invalidFilter(value)) {
           // @ts-ignore
           delete opts.filters[filterField][key]
         }
       }
-    }
+    
   }
 
   if (
diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts
index e85ec4553c..12c84bfb7b 100644
--- a/packages/server/src/api/controllers/row/utils.ts
+++ b/packages/server/src/api/controllers/row/utils.ts
@@ -139,3 +139,9 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
+
+// don't do a pure falsy check, as 0 is included
+// https://github.com/Budibase/budibase/issues/10118
+export function invalidFilter(value: any) {
+  return value == null || value === ""
+}
diff --git a/yarn.lock b/yarn.lock
index ab86a87560..8c93661665 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6269,6 +6269,14 @@
     "@types/tedious" "*"
     tarn "^3.0.1"
 
+"@types/node-fetch@2.6.1":
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
+  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
+  dependencies:
+    "@types/node" "*"
+    form-data "^3.0.0"
+
 "@types/node-fetch@2.6.4":
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
@@ -6290,6 +6298,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
   integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==
 
+"@types/node@14.18.20":
+  version "14.18.20"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
+  integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
+
 "@types/node@16.9.1":
   version "16.9.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"

From aed5694ac2715dae82e5f43f314d16b5647806a7 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 12 Sep 2023 14:06:20 +0100
Subject: [PATCH 03/46] Adding dedicated function for removing empty filters.

---
 .../src/api/controllers/row/external.ts       | 15 +-------
 .../server/src/api/controllers/row/utils.ts   | 38 +++++++++++++++++--
 2 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 44921a5114..a04584e6bd 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -18,6 +18,7 @@ import {
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 import { dataFilters } from "@budibase/shared-core"
+import { removeEmptyFilters } from "./utils"
 
 export async function handleRequest(
   operation: Operation,
@@ -26,20 +27,8 @@ export async function handleRequest(
 ) {
   // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
   if (opts && opts.filters) {
-    for (let filterField of NoEmptyFilterStrings) {
-      if (!opts.filters[filterField]) {
-        continue
-      }
-      // @ts-ignore
-      for (let [key, value] of Object.entries(opts.filters[filterField])) {
-        if (utils.invalidFilter(value)) {
-          // @ts-ignore
-          delete opts.filters[filterField][key]
-        }
-      }
-    
+    opts.filters = utils.removeEmptyFilters(opts.filters)
   }
-
   if (
     !dataFilters.hasFilters(opts?.filters) &&
     opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts
index 12c84bfb7b..cc27f4c2a3 100644
--- a/packages/server/src/api/controllers/row/utils.ts
+++ b/packages/server/src/api/controllers/row/utils.ts
@@ -1,8 +1,15 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import { context } from "@budibase/backend-core"
-import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types"
-import { FieldTypes } from "../../../constants"
+import {
+  Ctx,
+  FieldType,
+  Row,
+  SearchFilters,
+  Table,
+  UserCtx,
+} from "@budibase/types"
+import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
 import sdk from "../../../sdk"
 
 import validateJs from "validate.js"
@@ -142,6 +149,29 @@ export async function validate({
 
 // don't do a pure falsy check, as 0 is included
 // https://github.com/Budibase/budibase/issues/10118
-export function invalidFilter(value: any) {
-  return value == null || value === ""
+export function removeEmptyFilters(filters: SearchFilters) {
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!filters[filterField]) {
+      continue
+    }
+
+    for (let filterType of Object.keys(filters)) {
+      if (filterType !== filterField) {
+        continue
+      }
+      // don't know which one we're checking, type could be anything
+      const value = filters[filterType] as unknown
+      if (typeof value === "object") {
+        for (let [key, value] of Object.entries(
+          filters[filterType] as object
+        )) {
+          if (value == null || value === "") {
+            // @ts-ignore
+            delete filters[filterField][key]
+          }
+        }
+      }
+    }
+  }
+  return filters
 }

From 80e217afda1325ccf8b23f8650eead02a26fb0d0 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 12 Sep 2023 14:23:08 +0100
Subject: [PATCH 04/46] Quick test to avoid scenario in future.

---
 .../api/controllers/row/tests/utils.spec.ts   | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 packages/server/src/api/controllers/row/tests/utils.spec.ts

diff --git a/packages/server/src/api/controllers/row/tests/utils.spec.ts b/packages/server/src/api/controllers/row/tests/utils.spec.ts
new file mode 100644
index 0000000000..e0ad637e9d
--- /dev/null
+++ b/packages/server/src/api/controllers/row/tests/utils.spec.ts
@@ -0,0 +1,21 @@
+import * as utils from "../utils"
+
+describe("removeEmptyFilters", () => {
+  it("0 should not be removed", () => {
+    const filters = utils.removeEmptyFilters({
+      equal: {
+        column: 0,
+      },
+    })
+    expect((filters.equal as any).column).toBe(0)
+  })
+
+  it("empty string should be removed", () => {
+    const filters = utils.removeEmptyFilters({
+      equal: {
+        column: "",
+      },
+    })
+    expect(Object.values(filters.equal as any).length).toBe(0)
+  })
+})

From 2dd2dcc2cb0dabb3416157ef6d9c665943b7383c Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Tue, 12 Sep 2023 17:26:41 +0100
Subject: [PATCH 05/46] Role fixes

---
 .../app/[application]/_components/BuilderSidePanel.svelte       | 2 +-
 .../src/pages/builder/portal/users/users/[userId].svelte        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
index 7c4d3db7ce..c93a41f541 100644
--- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
@@ -126,7 +126,7 @@
           user,
           prodAppId
         )
-        const isAppBuilder = sdk.users.hasAppBuilderPermissions(user, prodAppId)
+        const isAppBuilder = user.builder?.apps?.includes(prodAppId)
         let role
         if (isAdminOrGlobalBuilder) {
           role = Constants.Roles.ADMIN
diff --git a/packages/builder/src/pages/builder/portal/users/users/[userId].svelte b/packages/builder/src/pages/builder/portal/users/users/[userId].svelte
index 2a74cd9de5..ec10ec8316 100644
--- a/packages/builder/src/pages/builder/portal/users/users/[userId].svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/[userId].svelte
@@ -111,7 +111,7 @@
       })
     }
     return availableApps.map(app => {
-      const prodAppId = apps.getProdAppID(app.appId)
+      const prodAppId = apps.getProdAppID(app.devId)
       return {
         name: app.name,
         devId: app.devId,

From be5e8005b2d6b2a2c0e5a8339d25e034914e506a Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 13 Sep 2023 14:06:52 +0000
Subject: [PATCH 06/46] Bump version to 2.10.7

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 63466ceedb..77f38e67ed 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.6",
+  "version": "2.10.7",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 8af3e4bf6b5d6930126cb4302e0df67a5db2f6d8 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 13 Sep 2023 15:23:30 +0100
Subject: [PATCH 07/46] Found some discussion of testcontainers being
 problematic when nearly out of disk space, we have seen issues with the
 default Github runners as they have extremely limited disk space, this should
 help a bit removing android and dotnet, two pieces of functionality we will
 never need.

---
 .github/workflows/budibase_ci.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index d670e222d3..fc35575ec6 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -25,6 +25,13 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-android: 'true'
+          remove-dotnet: 'true'
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'

From 6d74bb4063bc7580bad2c822a325f291d638aa7d Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 13 Sep 2023 18:01:20 +0000
Subject: [PATCH 08/46] Bump version to 2.10.8

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 77f38e67ed..30aa2afa3c 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.7",
+  "version": "2.10.8",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 37cb9261e8028bd62d21f0499d7b20aeba6ddd64 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 14 Sep 2023 09:28:43 +0000
Subject: [PATCH 09/46] Bump version to 2.10.9-alpha.0

---
 lerna.json | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lerna.json b/lerna.json
index d267691a6e..2394c5aa57 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,7 +1,9 @@
 {
-  "version": "2.10.8",
+  "version": "2.10.9-alpha.0",
   "npmClient": "yarn",
-  "packages": ["packages/*"],
+  "packages": [
+    "packages/*"
+  ],
   "useNx": true,
   "command": {
     "publish": {
@@ -17,4 +19,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}
\ No newline at end of file

From 4daa996044837eaf344a60f1edb7ca53e1984700 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 14 Sep 2023 10:38:14 +0100
Subject: [PATCH 10/46] Adding console log to trigger cache bust and run tests.

---
 packages/backend-core/src/index.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index ffffd8240a..c540b717da 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -50,5 +50,6 @@ export * from "./constants"
 // expose package init function
 import * as db from "./db"
 export const init = (opts: any = {}) => {
+  console.log("test")
   db.init(opts.db)
 }

From 4ca9ea97b66c0e96b4949b01566326594714de31 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 14 Sep 2023 10:55:41 +0100
Subject: [PATCH 11/46] Get server/worker tests to run.

---
 packages/server/src/index.ts | 1 +
 packages/worker/src/index.ts | 1 +
 2 files changed, 2 insertions(+)

diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
index b3fbb22bde..46ec2035a7 100644
--- a/packages/server/src/index.ts
+++ b/packages/server/src/index.ts
@@ -11,6 +11,7 @@ function runServer() {
   process.env.GLOBAL_AGENT_FORCE_GLOBAL_AGENT = "false"
   bootstrap()
   require("./app")
+  console.log("test")
 }
 
 runServer()
diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index 30596d1d76..7212e8207c 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -109,6 +109,7 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
   await redis.init()
+  console.log("test")
 })
 
 process.on("uncaughtException", err => {

From 26c6393f06c102379f3f4a0a2886272b3cda71a8 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 14 Sep 2023 11:11:05 +0100
Subject: [PATCH 12/46] Adding retries to 409 test.

---
 packages/worker/src/api/routes/global/tests/scim.spec.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/worker/src/api/routes/global/tests/scim.spec.ts b/packages/worker/src/api/routes/global/tests/scim.spec.ts
index 5686e39fa8..fba1523cd4 100644
--- a/packages/worker/src/api/routes/global/tests/scim.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts
@@ -10,6 +10,8 @@ import {
 import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"
 
+// this test can 409 - retries reduce issues with this
+jest.retryTimes(2)
 jest.setTimeout(30000)
 
 mocks.licenses.useScimIntegration()

From 5f82b79d40d37f66a132a7f12bd43ed281444e90 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 14 Sep 2023 11:42:04 +0100
Subject: [PATCH 13/46] Removing test logs.

---
 packages/backend-core/src/index.ts | 1 -
 packages/server/src/index.ts       | 1 -
 packages/worker/src/index.ts       | 1 -
 3 files changed, 3 deletions(-)

diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index c540b717da..ffffd8240a 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -50,6 +50,5 @@ export * from "./constants"
 // expose package init function
 import * as db from "./db"
 export const init = (opts: any = {}) => {
-  console.log("test")
   db.init(opts.db)
 }
diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
index 46ec2035a7..b3fbb22bde 100644
--- a/packages/server/src/index.ts
+++ b/packages/server/src/index.ts
@@ -11,7 +11,6 @@ function runServer() {
   process.env.GLOBAL_AGENT_FORCE_GLOBAL_AGENT = "false"
   bootstrap()
   require("./app")
-  console.log("test")
 }
 
 runServer()
diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index 7212e8207c..30596d1d76 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -109,7 +109,6 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
   await redis.init()
-  console.log("test")
 })
 
 process.on("uncaughtException", err => {

From f15ca351cd00ddc525055c65c2138a0c92ce3960 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 14 Sep 2023 11:03:14 +0000
Subject: [PATCH 14/46] Bump version to 2.10.9-alpha.1

---
 lerna.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lerna.json b/lerna.json
index 68cc011c5b..a428ef9af6 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.9-alpha.0",
+  "version": "2.10.9-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"
@@ -19,4 +19,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}
\ No newline at end of file

From bd5c6056d8cab9360de943716b116a3e1903fae3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 15 Sep 2023 08:04:17 +0100
Subject: [PATCH 15/46] Update dayjs dependencies to same version and fix
 import syntax in DateCell

---
 packages/bbui/package.json                    |  2 +-
 packages/builder/package.json                 |  2 +-
 packages/client/package.json                  |  2 +-
 packages/frontend-core/package.json           |  2 +-
 .../src/components/grid/cells/DateCell.svelte |  2 +-
 packages/string-templates/package.json        |  2 +-
 yarn.lock                                     | 20 ++++++-------------
 7 files changed, 12 insertions(+), 20 deletions(-)

diff --git a/packages/bbui/package.json b/packages/bbui/package.json
index 0b87960ab5..dc6c910be8 100644
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@@ -82,7 +82,7 @@
     "@spectrum-css/typography": "3.0.1",
     "@spectrum-css/underlay": "2.0.9",
     "@spectrum-css/vars": "3.0.1",
-    "dayjs": "^1.10.4",
+    "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
     "svelte-flatpickr": "3.2.3",
     "svelte-portal": "^1.0.0",
diff --git a/packages/builder/package.json b/packages/builder/package.json
index a6bf81201d..43f1ae3bff 100644
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@@ -69,7 +69,7 @@
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",
-    "dayjs": "^1.11.2",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "fast-json-patch": "^3.1.1",
     "lodash": "4.17.21",
diff --git a/packages/client/package.json b/packages/client/package.json
index a5ee304610..698c7bd929 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -33,7 +33,7 @@
     "@spectrum-css/typography": "^3.0.2",
     "@spectrum-css/vars": "^3.0.1",
     "apexcharts": "^3.22.1",
-    "dayjs": "^1.10.5",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",
diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json
index ca7e0a6d2b..1f15bb72c5 100644
--- a/packages/frontend-core/package.json
+++ b/packages/frontend-core/package.json
@@ -8,7 +8,7 @@
   "dependencies": {
     "@budibase/bbui": "0.0.0",
     "@budibase/shared-core": "0.0.0",
-    "dayjs": "^1.11.7",
+    "dayjs": "^1.10.8",
     "lodash": "^4.17.21",
     "socket.io-client": "^4.6.1",
     "svelte": "^3.46.2"
diff --git a/packages/frontend-core/src/components/grid/cells/DateCell.svelte b/packages/frontend-core/src/components/grid/cells/DateCell.svelte
index 9144f5b769..53b159ee30 100644
--- a/packages/frontend-core/src/components/grid/cells/DateCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DateCell.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { dayjs } from "dayjs"
+  import dayjs from "dayjs"
   import { CoreDatePicker, Icon } from "@budibase/bbui"
   import { onMount } from "svelte"
 
diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json
index 4dc4c17ad8..aee60f4732 100644
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@@ -26,7 +26,7 @@
   },
   "dependencies": {
     "@budibase/handlebars-helpers": "^0.11.9",
-    "dayjs": "^1.10.4",
+    "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
     "handlebars-utils": "^1.0.6",
     "lodash": "^4.17.20",
diff --git a/yarn.lock b/yarn.lock
index 8c93661665..d777dca0bd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6269,14 +6269,6 @@
     "@types/tedious" "*"
     tarn "^3.0.1"
 
-"@types/node-fetch@2.6.1":
-  version "2.6.1"
-  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
-  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
-  dependencies:
-    "@types/node" "*"
-    form-data "^3.0.0"
-
 "@types/node-fetch@2.6.4":
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
@@ -6298,11 +6290,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
   integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==
 
-"@types/node@14.18.20":
-  version "14.18.20"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
-  integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
-
 "@types/node@16.9.1":
   version "16.9.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"
@@ -9792,11 +9779,16 @@ dateformat@^4.5.1, dateformat@^4.6.3:
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5"
   integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==
 
-dayjs@^1.10.4, dayjs@^1.10.5, dayjs@^1.11.2, dayjs@^1.11.7:
+dayjs@^1.10.4, dayjs@^1.10.5:
   version "1.11.7"
   resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2"
   integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==
 
+dayjs@^1.10.8:
+  version "1.11.9"
+  resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.9.tgz#9ca491933fadd0a60a2c19f6c237c03517d71d1a"
+  integrity sha512-QvzAURSbQ0pKdIye2txOzNaHmxtUBXerpY0FJsFXUMKbIZeFm5ht1LS/jFsrncjnmtv8HsG0W2g6c0zUjZWmpA==
+
 dd-trace@3.13.2:
   version "3.13.2"
   resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-3.13.2.tgz#95b1ec480ab9ac406e1da7591a8c6f678d3799fd"

From 14d5f551c329374a12080b3c901aa39063d275f1 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 15 Sep 2023 10:27:34 +0000
Subject: [PATCH 16/46] Bump version to 2.10.9-alpha.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index a428ef9af6..1673559ee9 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.9-alpha.1",
+  "version": "2.10.9-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 10fe86e30d4c618e45e6c976ee0fa6da5330d0dd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 18 Sep 2023 12:44:58 +0200
Subject: [PATCH 17/46] Type

---
 packages/backend-core/src/redis/redis.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts
index 5056a5d549..78817d0aa0 100644
--- a/packages/backend-core/src/redis/redis.ts
+++ b/packages/backend-core/src/redis/redis.ts
@@ -250,7 +250,7 @@ class RedisWrapper {
     const prefixedKeys = keys.map(key => addDbPrefix(db, key))
     let response = await this.getClient().mget(prefixedKeys)
     if (Array.isArray(response)) {
-      let final: any = {}
+      let final: Record<string, any> = {}
       let count = 0
       for (let result of response) {
         if (result) {

From a620bca62b4ab48dd1ce68b05c42b212c666e732 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 18 Sep 2023 13:10:27 +0200
Subject: [PATCH 18/46] Refactor

---
 .../tests/core/utilities/structures/shared.ts | 19 --------------
 .../tests/core/utilities/structures/sso.ts    |  7 +++--
 .../tests/core/utilities/structures/users.ts  | 26 +++++++++++++++++--
 3 files changed, 27 insertions(+), 25 deletions(-)
 delete mode 100644 packages/backend-core/tests/core/utilities/structures/shared.ts

diff --git a/packages/backend-core/tests/core/utilities/structures/shared.ts b/packages/backend-core/tests/core/utilities/structures/shared.ts
deleted file mode 100644
index de0e19486c..0000000000
--- a/packages/backend-core/tests/core/utilities/structures/shared.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import { User } from "@budibase/types"
-import { generator } from "./generator"
-import { uuid } from "./common"
-
-export const newEmail = () => {
-  return `${uuid()}@test.com`
-}
-
-export const user = (userProps?: any): User => {
-  return {
-    email: newEmail(),
-    password: "test",
-    roles: { app_test: "admin" },
-    firstName: generator.first(),
-    lastName: generator.last(),
-    pictureUrl: "http://test.com",
-    ...userProps,
-  }
-}
diff --git a/packages/backend-core/tests/core/utilities/structures/sso.ts b/packages/backend-core/tests/core/utilities/structures/sso.ts
index 4d13635f09..2e3af712a9 100644
--- a/packages/backend-core/tests/core/utilities/structures/sso.ts
+++ b/packages/backend-core/tests/core/utilities/structures/sso.ts
@@ -13,8 +13,7 @@ import {
 } from "@budibase/types"
 import { generator } from "./generator"
 import { email, uuid } from "./common"
-import * as shared from "./shared"
-import { user } from "./shared"
+import * as users from "./users"
 import sample from "lodash/sample"
 
 export function OAuth(): OAuth2 {
@@ -26,7 +25,7 @@ export function OAuth(): OAuth2 {
 
 export function authDetails(userDoc?: User): SSOAuthDetails {
   if (!userDoc) {
-    userDoc = user()
+    userDoc = users.user()
   }
 
   const userId = userDoc._id || uuid()
@@ -52,7 +51,7 @@ export function providerType(): SSOProviderType {
 
 export function ssoProfile(user?: User): SSOProfile {
   if (!user) {
-    user = shared.user()
+    user = users.user()
   }
   return {
     id: user._id!,
diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts
index 0a4f2e8b54..420a9fde0e 100644
--- a/packages/backend-core/tests/core/utilities/structures/users.ts
+++ b/packages/backend-core/tests/core/utilities/structures/users.ts
@@ -4,11 +4,33 @@ import {
   BuilderUser,
   SSOAuthDetails,
   SSOUser,
+  User,
 } from "@budibase/types"
-import { user } from "./shared"
 import { authDetails } from "./sso"
+import { uuid } from "./common"
+import { generator } from "./generator"
+import { tenant } from "."
+import { generateGlobalUserID } from "../../../../src/docIds"
 
-export { user, newEmail } from "./shared"
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
+  const userId = userProps?._id || generateGlobalUserID()
+  return {
+    _id: userId,
+    userId,
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    tenantId: tenant.id(),
+    ...userProps,
+  }
+}
 
 export const adminUser = (userProps?: any): AdminUser => {
   return {

From 36acd69fddaa2b4dcc8ed9db697c6e0477033135 Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Mon, 18 Sep 2023 12:14:12 +0100
Subject: [PATCH 19/46] Note about MySQL SSL (#11797)

---
 packages/server/src/integrations/mysql.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index b88162c59a..8a688c5f3b 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -158,6 +158,12 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     ) {
       config.ssl.rejectUnauthorized = config.rejectUnauthorized
     }
+    // The MySQL library we use doesn't directly document the parameters that can be passed in the ssl
+    // object, it instead points to an older library that it says it is mostly API compatible with, that
+    // older library actually documents what parameters can be passed in the ssl object.
+    // https://github.com/sidorares/node-mysql2#api-and-configuration
+    // https://github.com/mysqljs/mysql#ssl-options
+
     // @ts-ignore
     delete config.rejectUnauthorized
     this.config = {

From fd518548fddb17c1e28c3f5eddf07b3f5f2a3fea Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 18 Sep 2023 11:14:32 +0000
Subject: [PATCH 20/46] Bump version to 2.10.9-alpha.3

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 1673559ee9..a57875986a 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.9-alpha.2",
+  "version": "2.10.9-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 8c16c1908b09b3efa4af5b8dfbf18227d58ff437 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 18 Sep 2023 13:39:05 +0200
Subject: [PATCH 21/46] User cache, get in bulk

---
 packages/backend-core/src/cache/user.ts | 46 +++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts
index e2af78adfd..f05cdd42ac 100644
--- a/packages/backend-core/src/cache/user.ts
+++ b/packages/backend-core/src/cache/user.ts
@@ -27,6 +27,31 @@ async function populateFromDB(userId: string, tenantId: string) {
   return user
 }
 
+async function populateUsersFromDB(userIds: string[], tenantId: string) {
+  const db = tenancy.getTenantDB(tenantId)
+  const allDocsResponse = await db.allDocs<any>({
+    keys: userIds,
+    include_docs: true,
+    limit: userIds.length,
+  })
+
+  const users = allDocsResponse.rows.map(r => r.doc)
+  await Promise.all(
+    users.map(async user => {
+      user.budibaseAccess = true
+      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+        const account = await accounts.getAccount(user.email)
+        if (account) {
+          user.account = account
+          user.accountPortalAccess = true
+        }
+      }
+    })
+  )
+
+  return users
+}
+
 /**
  * Get the requested user by id.
  * Use redis cache to first read the user.
@@ -77,6 +102,27 @@ export async function getUser(
   return user
 }
 
+/**
+ * Get the requested users by id.
+ * Use redis cache to first read the users.
+ * If not present fallback to loading the users directly and re-caching.
+ * @param {*} userIds the ids of the user to get
+ * @param {*} tenantId the tenant of the users to get
+ * @returns
+ */
+export async function getUsers(userIds: string[], tenantId: string) {
+  const client = await redis.getUserClient()
+  // try cache
+  let usersFromCache = await client.bulkGet(userIds)
+  const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
+  const usersFromDb = await populateUsersFromDB(missingUsersFromCache, tenantId)
+  for (const userToCache of usersFromDb) {
+    await client.store(userToCache._id, userToCache, EXPIRY_SECONDS)
+  }
+  const users = [...Object.values(usersFromCache), ...usersFromDb]
+  return users
+}
+
 export async function invalidateUser(userId: string) {
   const client = await redis.getUserClient()
   await client.delete(userId)

From 333b415635de506c6954841ba8a45da072558884 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 18 Sep 2023 13:46:59 +0200
Subject: [PATCH 22/46] Add tests

---
 .../backend-core/src/cache/tests/user.spec.ts | 95 +++++++++++++++++++
 packages/backend-core/src/cache/user.ts       | 15 ++-
 2 files changed, 106 insertions(+), 4 deletions(-)
 create mode 100644 packages/backend-core/src/cache/tests/user.spec.ts

diff --git a/packages/backend-core/src/cache/tests/user.spec.ts b/packages/backend-core/src/cache/tests/user.spec.ts
new file mode 100644
index 0000000000..3d746fc506
--- /dev/null
+++ b/packages/backend-core/src/cache/tests/user.spec.ts
@@ -0,0 +1,95 @@
+import { User } from "@budibase/types"
+import { tenancy } from "../.."
+import { generator, structures } from "../../../tests"
+import { DBTestConfiguration } from "../../../tests/extra"
+import { getUsers } from "../user"
+import { getGlobalDB, getGlobalDBName } from "../../context"
+import _ from "lodash"
+import { getDB } from "../../db"
+import type * as TenancyType from "../../tenancy"
+
+const config = new DBTestConfiguration()
+
+// This mock is required to ensure that getTenantDB returns always as a singleton.
+// This will allow us to spy on the db
+const staticDb = getDB(getGlobalDBName(config.tenantId))
+jest.mock("../../tenancy", (): typeof TenancyType => ({
+  ...jest.requireActual("../../tenancy"),
+  getTenantDB: jest.fn().mockImplementation(() => staticDb),
+}))
+
+describe("user cache", () => {
+  describe("getUsers", () => {
+    const users: User[] = []
+    beforeAll(async () => {
+      const userCount = 10
+      const userIds = generator.arrayOf(() => generator.guid(), {
+        min: userCount,
+        max: userCount,
+      })
+
+      await config.doInTenant(async () => {
+        const db = getGlobalDB()
+        for (const userId of userIds) {
+          const user = structures.users.user({ _id: userId })
+          await db.put(user)
+          users.push(user)
+        }
+      })
+    })
+
+    beforeEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("when no user is in cache, all of them are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(staticDb, "allDocs")
+
+      const results = await getUsers(userIdsToRequest, config.tenantId)
+
+      expect(results).toHaveLength(5)
+      expect(results).toEqual(
+        usersToRequest.map(u => ({
+          ...u,
+          budibaseAccess: true,
+          _rev: expect.any(String),
+        }))
+      )
+
+      expect(tenancy.getTenantDB).toBeCalledTimes(1)
+      expect(tenancy.getTenantDB).toBeCalledWith(config.tenantId)
+      expect(staticDb.allDocs).toBeCalledTimes(1)
+      expect(staticDb.allDocs).toBeCalledWith({
+        keys: userIdsToRequest,
+        include_docs: true,
+        limit: 5,
+      })
+    })
+
+    it("on a second all, all of them are retrieved from cache", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(staticDb, "allDocs")
+
+      await getUsers(userIdsToRequest, config.tenantId)
+      const resultsFromCache = await getUsers(userIdsToRequest, config.tenantId)
+
+      expect(resultsFromCache).toHaveLength(5)
+      expect(resultsFromCache).toEqual(
+        usersToRequest.map(u => ({
+          ...u,
+          budibaseAccess: true,
+          _rev: expect.any(String),
+        }))
+      )
+
+      expect(staticDb.allDocs).toBeCalledTimes(1)
+    })
+  })
+})
diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts
index f05cdd42ac..9742b41b65 100644
--- a/packages/backend-core/src/cache/user.ts
+++ b/packages/backend-core/src/cache/user.ts
@@ -115,11 +115,18 @@ export async function getUsers(userIds: string[], tenantId: string) {
   // try cache
   let usersFromCache = await client.bulkGet(userIds)
   const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
-  const usersFromDb = await populateUsersFromDB(missingUsersFromCache, tenantId)
-  for (const userToCache of usersFromDb) {
-    await client.store(userToCache._id, userToCache, EXPIRY_SECONDS)
+  const users = Object.values(usersFromCache)
+
+  if (missingUsersFromCache.length) {
+    const usersFromDb = await populateUsersFromDB(
+      missingUsersFromCache,
+      tenantId
+    )
+    for (const userToCache of usersFromDb) {
+      await client.store(userToCache._id, userToCache, EXPIRY_SECONDS)
+    }
+    users.push(...usersFromDb)
   }
-  const users = [...Object.values(usersFromCache), ...usersFromDb]
   return users
 }
 

From 437f8eeb43a72694dba4cef07b4ad636c9c85935 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 18 Sep 2023 13:57:59 +0200
Subject: [PATCH 23/46] Add tests

---
 .../backend-core/src/cache/tests/user.spec.ts | 54 ++++++++++++++++---
 1 file changed, 47 insertions(+), 7 deletions(-)

diff --git a/packages/backend-core/src/cache/tests/user.spec.ts b/packages/backend-core/src/cache/tests/user.spec.ts
index 3d746fc506..45537694c7 100644
--- a/packages/backend-core/src/cache/tests/user.spec.ts
+++ b/packages/backend-core/src/cache/tests/user.spec.ts
@@ -1,5 +1,5 @@
 import { User } from "@budibase/types"
-import { tenancy } from "../.."
+import { cache, tenancy } from "../.."
 import { generator, structures } from "../../../tests"
 import { DBTestConfiguration } from "../../../tests/extra"
 import { getUsers } from "../user"
@@ -7,6 +7,7 @@ import { getGlobalDB, getGlobalDBName } from "../../context"
 import _ from "lodash"
 import { getDB } from "../../db"
 import type * as TenancyType from "../../tenancy"
+import * as redis from "../../redis/init"
 
 const config = new DBTestConfiguration()
 
@@ -38,8 +39,11 @@ describe("user cache", () => {
       })
     })
 
-    beforeEach(() => {
+    beforeEach(async () => {
       jest.clearAllMocks()
+
+      const redisClient = await redis.getUserClient()
+      await redisClient.clear()
     })
 
     it("when no user is in cache, all of them are retrieved from db", async () => {
@@ -82,14 +86,50 @@ describe("user cache", () => {
 
       expect(resultsFromCache).toHaveLength(5)
       expect(resultsFromCache).toEqual(
-        usersToRequest.map(u => ({
-          ...u,
-          budibaseAccess: true,
-          _rev: expect.any(String),
-        }))
+        expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        )
       )
 
       expect(staticDb.allDocs).toBeCalledTimes(1)
     })
+
+    it("when some users are cached, only the missing ones are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(staticDb, "allDocs")
+
+      await getUsers(
+        [userIdsToRequest[0], userIdsToRequest[3]],
+        config.tenantId
+      )
+      ;(staticDb.allDocs as jest.Mock).mockClear()
+
+      const results = await getUsers(userIdsToRequest, config.tenantId)
+
+      expect(results).toHaveLength(5)
+      expect(results).toEqual(
+        expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        )
+      )
+
+      expect(staticDb.allDocs).toBeCalledTimes(1)
+      expect(staticDb.allDocs).toBeCalledWith({
+        keys: [userIdsToRequest[1], userIdsToRequest[2], userIdsToRequest[4]],
+        include_docs: true,
+        limit: 3,
+      })
+    })
   })
 })

From 4599ff3b9c0f39447310774a523c2f1668954c5a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 18 Sep 2023 14:38:20 +0200
Subject: [PATCH 24/46] Fix test

---
 .../backend-core/src/middleware/passport/sso/tests/sso.spec.ts   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts
index 484a118cbd..c3ddf220e6 100644
--- a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts
+++ b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts
@@ -102,6 +102,7 @@ describe("sso", () => {
 
           // modified external id to match user format
           ssoUser._id = "us_" + details.userId
+          delete ssoUser.userId
 
           // new sso user won't have a password
           delete ssoUser.password

From d7436d3f97ab2ee20db8e14c67b192c5b5b1a279 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 10:45:27 +0200
Subject: [PATCH 25/46] Add user type column selector

---
 .../backend/DataTable/modals/CreateEditColumn.svelte     | 6 ++++++
 packages/builder/src/constants/backend/index.js          | 9 +++++++++
 2 files changed, 15 insertions(+)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 44c37813d6..bfe7783e4f 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -20,6 +20,7 @@
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
+    DEV_FIELDS,
     RelationshipType,
     ALLOWABLE_STRING_OPTIONS,
     ALLOWABLE_NUMBER_OPTIONS,
@@ -34,6 +35,7 @@
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
+  import { admin } from "stores/portal"
 
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
@@ -70,6 +72,10 @@
     fieldName: $tables.selected.name,
   }
 
+  if ($admin.isDev) {
+    fieldDefinitions = { ...fieldDefinitions, ...cloneDeep(DEV_FIELDS) }
+  }
+
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js
index ed0549eeca..45cd2d5ed4 100644
--- a/packages/builder/src/constants/backend/index.js
+++ b/packages/builder/src/constants/backend/index.js
@@ -122,6 +122,15 @@ export const FIELDS = {
   },
 }
 
+export const DEV_FIELDS = {
+  USER: {
+    name: "User",
+    type: "internal",
+    subtype: "user",
+    icon: "User",
+  },
+}
+
 export const AUTO_COLUMN_SUB_TYPES = {
   AUTO_ID: "autoID",
   CREATED_BY: "createdBy",

From a79960bf2368c9e3879a399407fbaea057d9774a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 15:18:54 +0200
Subject: [PATCH 26/46] Typings

---
 .../backend/DataTable/modals/CreateEditColumn.svelte        | 2 +-
 packages/server/src/api/controllers/table/utils.ts          | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index bfe7783e4f..c4342f9b0b 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -66,7 +66,7 @@
   let jsonSchemaModal
   let allowedTypes = []
   let editableColumn = {
-    type: "string",
+    type: fieldDefinitions.STRING.type,
     constraints: fieldDefinitions.STRING.constraints,
     // Initial value for column name in other table for linked records
     fieldName: $tables.selected.name,
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index 0e5b784c66..e77c2b7647 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -422,13 +422,11 @@ export function hasTypeChanged(table: Table, oldTable: Table | undefined) {
   if (!oldTable) {
     return false
   }
-  let key: any
-  let field: any
-  for ([key, field] of Object.entries(oldTable.schema)) {
-    const oldType = field.type
+  for (let [key, field] of Object.entries(oldTable.schema)) {
     if (!table.schema[key]) {
       continue
     }
+    const oldType = field.type
     const newType = table.schema[key].type
     if (oldType !== newType && !areSwitchableTypes(oldType, newType)) {
       return true

From 752ae0ac2bb1ac4034323e44c2c534a7555cd1c7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 16:06:02 +0200
Subject: [PATCH 27/46] Handle internal types frontend

---
 .../DataTable/modals/CreateEditColumn.svelte  | 38 ++++++++++++++++++-
 .../src/components/grid/lib/utils.js          | 13 ++++++-
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index c4342f9b0b..f61e69943d 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -72,8 +72,28 @@
     fieldName: $tables.selected.name,
   }
 
+  const typeMapping = {}
+
   if ($admin.isDev) {
-    fieldDefinitions = { ...fieldDefinitions, ...cloneDeep(DEV_FIELDS) }
+    fieldDefinitions = {
+      ...fieldDefinitions,
+      ...Object.entries(DEV_FIELDS).reduce((p, [key, field]) => {
+        if (field.subtype) {
+          const composedType = `${field.type}_${field.subtype}`
+          p[key] = {
+            ...field,
+            type: composedType,
+          }
+          typeMapping[composedType] = {
+            type: field.type,
+            subtype: field.subtype,
+          }
+        } else {
+          p[key] = field
+        }
+        return p
+      }, {}),
+    }
   }
 
   $: if (primaryDisplay) {
@@ -82,6 +102,7 @@
 
   const initialiseField = (field, savingColumn) => {
     isCreating = !field
+
     if (field && !savingColumn) {
       editableColumn = cloneDeep(field)
       originalName = editableColumn.name ? editableColumn.name + "" : null
@@ -89,6 +110,14 @@
       primaryDisplay =
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
+
+      const mapped = Object.entries(typeMapping).find(
+        ([_, v]) => v.type === field.type && v.subtype === field.subtype
+      )
+      if (mapped) {
+        editableColumn.type = mapped[0]
+        delete editableColumn.subtype
+      }
     } else if (!savingColumn) {
       let highestNumber = 0
       Object.keys(table.schema).forEach(columnName => {
@@ -105,6 +134,7 @@
         editableColumn.name = "Column 01"
       }
     }
+
     allowedTypes = getAllowedTypes()
   }
 
@@ -186,6 +216,10 @@
 
     let saveColumn = cloneDeep(editableColumn)
 
+    if (typeMapping[saveColumn.type]) {
+      saveColumn = { ...saveColumn, ...typeMapping[saveColumn.type] }
+    }
+
     if (saveColumn.type === AUTO_TYPE) {
       saveColumn = buildAutoColumn(
         $tables.selected.name,
@@ -426,6 +460,8 @@
   onMount(() => {
     mounted = true
   })
+
+  $: console.log({ type: editableColumn.type, allowedTypes, typeMapping })
 </script>
 
 <Layout noPadding gap="S">
diff --git a/packages/frontend-core/src/components/grid/lib/utils.js b/packages/frontend-core/src/components/grid/lib/utils.js
index d4298e6a1b..feff3da625 100644
--- a/packages/frontend-core/src/components/grid/lib/utils.js
+++ b/packages/frontend-core/src/components/grid/lib/utils.js
@@ -19,12 +19,21 @@ const TypeIconMap = {
   formula: "Calculator",
   json: "Brackets",
   bigint: "TagBold",
+  internal: {
+    user: "User",
+  },
 }
 
 export const getColumnIcon = column => {
   if (column.schema.autocolumn) {
     return "MagicWand"
   }
-  const type = column.schema.type
-  return TypeIconMap[type] || "Text"
+  const { type, subtype } = column.schema
+
+  const result =
+    typeof TypeIconMap[type] === "object" && subtype
+      ? TypeIconMap[type][subtype]
+      : TypeIconMap[type]
+
+  return result || "Text"
 }

From 03c3179df449bd7d351178cd7519fd75424faa8d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 16:16:47 +0200
Subject: [PATCH 28/46] Clean

---
 .../DataTable/modals/CreateEditColumn.svelte  | 50 ++++++++++---------
 1 file changed, 27 insertions(+), 23 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index f61e69943d..d9f95b493c 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -74,28 +74,6 @@
 
   const typeMapping = {}
 
-  if ($admin.isDev) {
-    fieldDefinitions = {
-      ...fieldDefinitions,
-      ...Object.entries(DEV_FIELDS).reduce((p, [key, field]) => {
-        if (field.subtype) {
-          const composedType = `${field.type}_${field.subtype}`
-          p[key] = {
-            ...field,
-            type: composedType,
-          }
-          typeMapping[composedType] = {
-            type: field.type,
-            subtype: field.subtype,
-          }
-        } else {
-          p[key] = field
-        }
-        return p
-      }, {}),
-    }
-  }
-
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
@@ -368,9 +346,35 @@
       ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1
     ) {
       return ALLOWABLE_NUMBER_OPTIONS
-    } else if (!external) {
+    }
+
+    let devFieldDefinitions = {}
+    if ($admin.isDev) {
+      devFieldDefinitions = Object.entries(DEV_FIELDS).reduce(
+        (p, [key, field]) => {
+          if (field.subtype) {
+            const composedType = `${field.type}_${field.subtype}`
+            p[key] = {
+              ...field,
+              type: composedType,
+            }
+            typeMapping[composedType] = {
+              type: field.type,
+              subtype: field.subtype,
+            }
+          } else {
+            p[key] = field
+          }
+          return p
+        },
+        {}
+      )
+    }
+
+    if (!external) {
       return [
         ...Object.values(fieldDefinitions),
+        ...Object.values(devFieldDefinitions),
         { name: "Auto Column", type: AUTO_TYPE },
       ]
     } else {

From ca5f7cba092cf9b0dde918bb9f6f5e835245bbc4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 16:43:34 +0200
Subject: [PATCH 29/46] Clean

---
 .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index d9f95b493c..3619787a38 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -464,8 +464,6 @@
   onMount(() => {
     mounted = true
   })
-
-  $: console.log({ type: editableColumn.type, allowedTypes, typeMapping })
 </script>
 
 <Layout noPadding gap="S">

From 00d2adccec7234e0f71a1370c51ad690ed1f6135 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 16:49:41 +0200
Subject: [PATCH 30/46] Merge dev fields to fields

---
 .../DataTable/modals/CreateEditColumn.svelte  | 50 +++++++++----------
 .../builder/src/constants/backend/index.js    |  3 --
 2 files changed, 25 insertions(+), 28 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 3619787a38..eb66d4d995 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -20,7 +20,6 @@
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
-    DEV_FIELDS,
     RelationshipType,
     ALLOWABLE_STRING_OPTIONS,
     ALLOWABLE_NUMBER_OPTIONS,
@@ -73,6 +72,30 @@
   }
 
   const typeMapping = {}
+  if (!$admin.isDev) {
+    delete fieldDefinitions.USER
+  }
+
+  // Handling fields with subtypes
+  fieldDefinitions = Object.entries(fieldDefinitions).reduce(
+    (p, [key, field]) => {
+      if (field.subtype) {
+        const composedType = `${field.type}_${field.subtype}`
+        p[key] = {
+          ...field,
+          type: composedType,
+        }
+        typeMapping[composedType] = {
+          type: field.type,
+          subtype: field.subtype,
+        }
+      } else {
+        p[key] = field
+      }
+      return p
+    },
+    {}
+  )
 
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
@@ -348,33 +371,9 @@
       return ALLOWABLE_NUMBER_OPTIONS
     }
 
-    let devFieldDefinitions = {}
-    if ($admin.isDev) {
-      devFieldDefinitions = Object.entries(DEV_FIELDS).reduce(
-        (p, [key, field]) => {
-          if (field.subtype) {
-            const composedType = `${field.type}_${field.subtype}`
-            p[key] = {
-              ...field,
-              type: composedType,
-            }
-            typeMapping[composedType] = {
-              type: field.type,
-              subtype: field.subtype,
-            }
-          } else {
-            p[key] = field
-          }
-          return p
-        },
-        {}
-      )
-    }
-
     if (!external) {
       return [
         ...Object.values(fieldDefinitions),
-        ...Object.values(devFieldDefinitions),
         { name: "Auto Column", type: AUTO_TYPE },
       ]
     } else {
@@ -393,6 +392,7 @@
       if (!external || table.sql) {
         fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
       }
+      // fields = [...fields, ...Object.values(devFieldDefinitions)]
       return fields
     }
   }
diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js
index 45cd2d5ed4..ccbc925672 100644
--- a/packages/builder/src/constants/backend/index.js
+++ b/packages/builder/src/constants/backend/index.js
@@ -120,9 +120,6 @@ export const FIELDS = {
       presence: false,
     },
   },
-}
-
-export const DEV_FIELDS = {
   USER: {
     name: "User",
     type: "internal",

From 997e7640bf97c78ca4d15e4d1583ff56c08c6b30 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 16:51:15 +0200
Subject: [PATCH 31/46] Clean

---
 .../components/backend/DataTable/modals/CreateEditColumn.svelte  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index eb66d4d995..7fa016af22 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -392,7 +392,6 @@
       if (!external || table.sql) {
         fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
       }
-      // fields = [...fields, ...Object.values(devFieldDefinitions)]
       return fields
     }
   }

From bedf43fb283b003f785ecb31e98e5b06d3ca989d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 17:46:50 +0200
Subject: [PATCH 32/46] Rename user type

---
 packages/builder/src/constants/backend/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js
index ccbc925672..fbcab7fd8b 100644
--- a/packages/builder/src/constants/backend/index.js
+++ b/packages/builder/src/constants/backend/index.js
@@ -122,7 +122,7 @@ export const FIELDS = {
   },
   USER: {
     name: "User",
-    type: "internal",
+    type: "bb_reference",
     subtype: "user",
     icon: "User",
   },

From f2c31b57524124589f9b795e23bd249235ae9843 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 17:47:45 +0200
Subject: [PATCH 33/46] Add type

---
 packages/types/src/documents/app/row.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index c09dc79b95..6001695b13 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -16,6 +16,7 @@ export enum FieldType {
   INTERNAL = "internal",
   BARCODEQR = "barcodeqr",
   BIGINT = "bigint",
+  BB_REFERENCE = "bb_reference",
 }
 
 export interface RowAttachment {

From 18f667b596dd080c0bddd62a625c79fee9596ec0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 17:49:48 +0200
Subject: [PATCH 34/46] Renamings

---
 .../DataTable/modals/CreateEditColumn.svelte     | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 7fa016af22..dad5660c8e 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -35,6 +35,7 @@
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
   import { admin } from "stores/portal"
+  import { FieldType } from "@budibase/types"
 
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
@@ -71,7 +72,7 @@
     fieldName: $tables.selected.name,
   }
 
-  const typeMapping = {}
+  const bbRefTypeMapping = {}
   if (!$admin.isDev) {
     delete fieldDefinitions.USER
   }
@@ -79,13 +80,13 @@
   // Handling fields with subtypes
   fieldDefinitions = Object.entries(fieldDefinitions).reduce(
     (p, [key, field]) => {
-      if (field.subtype) {
+      if (field.type === FieldType.BB_REFERENCE) {
         const composedType = `${field.type}_${field.subtype}`
         p[key] = {
           ...field,
           type: composedType,
         }
-        typeMapping[composedType] = {
+        bbRefTypeMapping[composedType] = {
           type: field.type,
           subtype: field.subtype,
         }
@@ -112,7 +113,7 @@
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
 
-      const mapped = Object.entries(typeMapping).find(
+      const mapped = Object.entries(bbRefTypeMapping).find(
         ([_, v]) => v.type === field.type && v.subtype === field.subtype
       )
       if (mapped) {
@@ -217,8 +218,11 @@
 
     let saveColumn = cloneDeep(editableColumn)
 
-    if (typeMapping[saveColumn.type]) {
-      saveColumn = { ...saveColumn, ...typeMapping[saveColumn.type] }
+    if (bbRefTypeMapping[saveColumn.type]) {
+      saveColumn = {
+        ...saveColumn,
+        ...bbRefTypeMapping[saveColumn.type],
+      }
     }
 
     if (saveColumn.type === AUTO_TYPE) {

From 3ee95938c3bc1713b895e872469e8c7f80da258b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Sep 2023 17:51:03 +0200
Subject: [PATCH 35/46] Allow bb_refs to externals

---
 .../backend/DataTable/modals/CreateEditColumn.svelte           | 3 +++
 packages/server/src/integrations/base/sqlTable.ts              | 1 +
 2 files changed, 4 insertions(+)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index dad5660c8e..62cd7e413e 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -396,6 +396,9 @@
       if (!external || table.sql) {
         fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
       }
+      if (fieldDefinitions.USER) {
+        fields.push(fieldDefinitions.USER)
+      }
       return fields
     }
   }
diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts
index 47b09dadee..4383167f4a 100644
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@@ -41,6 +41,7 @@ function generateSchema(
       case FieldTypes.OPTIONS:
       case FieldTypes.LONGFORM:
       case FieldTypes.BARCODEQR:
+      case FieldTypes.BB_REFERENCE:
         schema.text(key)
         break
       case FieldTypes.NUMBER:

From e4caf8b7376fd3c9ccfb5fb083aed71e03d7188c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 09:56:28 +0200
Subject: [PATCH 36/46] Remove context refs in row processor

---
 .../src/api/controllers/row/external.ts       |  2 +-
 .../src/api/controllers/row/internal.ts       |  4 ++--
 .../server/src/api/controllers/table/utils.ts |  2 +-
 .../src/utilities/rowProcessor/index.ts       | 20 +++++++++----------
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index fb0e1c0bc0..0f1c3788bc 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -81,7 +81,7 @@ export async function save(ctx: UserCtx) {
 
   const table = await sdk.tables.getTable(tableId)
   const { table: updatedTable, row } = inputProcessing(
-    ctx.user,
+    ctx.user?._id,
     cloneDeep(table),
     inputs
   )
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index b80bd339e6..1441fe92a1 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -59,7 +59,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableClone = cloneDeep(dbTable)
 
   // this returns the table and row incase they have been updated
-  let { table, row } = inputProcessing(ctx.user, tableClone, combinedRow)
+  let { table, row } = inputProcessing(ctx.user?._id, tableClone, combinedRow)
   const validateResult = await sdk.rows.utils.validate({
     row,
     table,
@@ -106,7 +106,7 @@ export async function save(ctx: UserCtx) {
   // need to copy the table so it can be differenced on way out
   const tableClone = cloneDeep(dbTable)
 
-  let { table, row } = inputProcessing(ctx.user, tableClone, inputs)
+  let { table, row } = inputProcessing(ctx.user?._id, tableClone, inputs)
 
   const validateResult = await sdk.rows.utils.validate({
     row,
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index e77c2b7647..3a5b22b702 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -113,7 +113,7 @@ export function importToRows(
 
     // We use a reference to table here and update it after input processing,
     // so that we can auto increment auto IDs in imported data properly
-    const processed = inputProcessing(user, table, row, {
+    const processed = inputProcessing(user?._id, table, row, {
       noAutoRelationships: true,
     })
     row = processed.row
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 8e95a15dca..232ba94808 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -5,8 +5,8 @@ import { ObjectStoreBuckets } from "../../constants"
 import { context, db as dbCore, objectStore } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import { Row, RowAttachment, Table, ContextUser } from "@budibase/types"
-const { cloneDeep } = require("lodash/fp")
+import { Row, RowAttachment, Table } from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
 export * from "./utils"
 
 type AutoColumnProcessingOpts = {
@@ -48,12 +48,12 @@ function getRemovedAttachmentKeys(
  * for automatic ID purposes.
  */
 export function processAutoColumn(
-  user: ContextUser | null,
+  userId: string | null | undefined,
   table: Table,
   row: Row,
   opts?: AutoColumnProcessingOpts
 ) {
-  let noUser = !user || !user.userId
+  let noUser = !userId
   let isUserTable = table._id === InternalTables.USER_METADATA
   let now = new Date().toISOString()
   // if a row doesn't have a revision then it doesn't exist yet
@@ -70,8 +70,8 @@ export function processAutoColumn(
     }
     switch (schema.subtype) {
       case AutoFieldSubTypes.CREATED_BY:
-        if (creating && shouldUpdateUserFields && user) {
-          row[key] = [user.userId]
+        if (creating && shouldUpdateUserFields && userId) {
+          row[key] = [userId]
         }
         break
       case AutoFieldSubTypes.CREATED_AT:
@@ -80,8 +80,8 @@ export function processAutoColumn(
         }
         break
       case AutoFieldSubTypes.UPDATED_BY:
-        if (shouldUpdateUserFields && user) {
-          row[key] = [user.userId]
+        if (shouldUpdateUserFields && userId) {
+          row[key] = [userId]
         }
         break
       case AutoFieldSubTypes.UPDATED_AT:
@@ -131,7 +131,7 @@ export function coerce(row: any, type: string) {
  * @returns {object} the row which has been prepared to be written to the DB.
  */
 export function inputProcessing(
-  user: ContextUser | null,
+  userId: string | null | undefined,
   table: Table,
   row: Row,
   opts?: AutoColumnProcessingOpts
@@ -174,7 +174,7 @@ export function inputProcessing(
   }
 
   // handle auto columns - this returns an object like {table, row}
-  return processAutoColumn(user, table, clonedRow, opts)
+  return processAutoColumn(userId, table, clonedRow, opts)
 }
 
 /**

From 9860023c9ec6c6c51b278521b107016b18c37a99 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 10:31:54 +0200
Subject: [PATCH 37/46] inputProcessing async

---
 packages/server/src/api/controllers/application.ts |  2 +-
 .../server/src/api/controllers/row/external.ts     |  2 +-
 .../server/src/api/controllers/row/internal.ts     |  8 ++++++--
 .../src/api/controllers/table/tests/utils.spec.ts  |  4 ++--
 packages/server/src/api/controllers/table/utils.ts |  6 +++---
 .../src/db/defaultData/datasource_bb_default.ts    | 14 +++++++-------
 .../server/src/utilities/rowProcessor/index.ts     |  2 +-
 7 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 012aa7c66d..71bd034b9f 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -168,7 +168,7 @@ export const addSampleData = async (ctx: UserCtx) => {
     // Check if default datasource exists before creating it
     await sdk.datasources.get(DEFAULT_BB_DATASOURCE_ID)
   } catch (err: any) {
-    const defaultDbDocs = buildDefaultDocs()
+    const defaultDbDocs = await buildDefaultDocs()
 
     // add in the default db data docs - tables, datasource, rows and links
     await db.bulkDocs([...defaultDbDocs])
diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 0f1c3788bc..5a0a64adc6 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -80,7 +80,7 @@ export async function save(ctx: UserCtx) {
   }
 
   const table = await sdk.tables.getTable(tableId)
-  const { table: updatedTable, row } = inputProcessing(
+  const { table: updatedTable, row } = await inputProcessing(
     ctx.user?._id,
     cloneDeep(table),
     inputs
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index 1441fe92a1..f33df4536f 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -59,7 +59,11 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableClone = cloneDeep(dbTable)
 
   // this returns the table and row incase they have been updated
-  let { table, row } = inputProcessing(ctx.user?._id, tableClone, combinedRow)
+  let { table, row } = await inputProcessing(
+    ctx.user?._id,
+    tableClone,
+    combinedRow
+  )
   const validateResult = await sdk.rows.utils.validate({
     row,
     table,
@@ -106,7 +110,7 @@ export async function save(ctx: UserCtx) {
   // need to copy the table so it can be differenced on way out
   const tableClone = cloneDeep(dbTable)
 
-  let { table, row } = inputProcessing(ctx.user?._id, tableClone, inputs)
+  let { table, row } = await inputProcessing(ctx.user?._id, tableClone, inputs)
 
   const validateResult = await sdk.rows.utils.validate({
     row,
diff --git a/packages/server/src/api/controllers/table/tests/utils.spec.ts b/packages/server/src/api/controllers/table/tests/utils.spec.ts
index 957cf51ecd..931a1cd90f 100644
--- a/packages/server/src/api/controllers/table/tests/utils.spec.ts
+++ b/packages/server/src/api/controllers/table/tests/utils.spec.ts
@@ -42,7 +42,7 @@ describe("utils", () => {
 
         const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }]
 
-        const result = importToRows(data, table, config.user)
+        const result = await importToRows(data, table, config.user)
         expect(result).toEqual([
           expect.objectContaining({
             autoId: 1,
@@ -89,7 +89,7 @@ describe("utils", () => {
 
         const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }]
 
-        const result = importToRows(data, table)
+        const result = await importToRows(data, table)
         expect(result).toHaveLength(3)
       })
     })
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index 3a5b22b702..e0d564db2a 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -99,7 +99,7 @@ export function makeSureTableUpToDate(table: any, tableToSave: any) {
   return tableToSave
 }
 
-export function importToRows(
+export async function importToRows(
   data: any[],
   table: Table,
   user: ContextUser | null = null
@@ -113,7 +113,7 @@ export function importToRows(
 
     // We use a reference to table here and update it after input processing,
     // so that we can auto increment auto IDs in imported data properly
-    const processed = inputProcessing(user?._id, table, row, {
+    const processed = await inputProcessing(user?._id, table, row, {
       noAutoRelationships: true,
     })
     row = processed.row
@@ -158,7 +158,7 @@ export async function handleDataImport(
   const db = context.getAppDB()
   const data = parse(rows, schema)
 
-  let finalData: any = importToRows(data, table, user)
+  let finalData: any = await importToRows(data, table, user)
 
   //Set IDs of finalData to match existing row if an update is expected
   if (identifierFields.length > 0) {
diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts
index d01f598ce4..a4821667ff 100644
--- a/packages/server/src/db/defaultData/datasource_bb_default.ts
+++ b/packages/server/src/db/defaultData/datasource_bb_default.ts
@@ -34,9 +34,9 @@ function syncLastIds(table: Table, rowCount: number) {
   })
 }
 
-function tableImport(table: Table, data: Row[]) {
+async function tableImport(table: Table, data: Row[]) {
   const cloneTable = cloneDeep(table)
-  const rowDocs = importToRows(data, cloneTable)
+  const rowDocs = await importToRows(data, cloneTable)
   syncLastIds(cloneTable, rowDocs.length)
   return { rows: rowDocs, table: cloneTable }
 }
@@ -601,20 +601,20 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
   },
 }
 
-export function buildDefaultDocs() {
-  const inventoryData = tableImport(
+export async function buildDefaultDocs() {
+  const inventoryData = await tableImport(
     DEFAULT_INVENTORY_TABLE_SCHEMA,
     inventoryImport
   )
 
-  const employeeData = tableImport(
+  const employeeData = await tableImport(
     DEFAULT_EMPLOYEE_TABLE_SCHEMA,
     employeeImport
   )
 
-  const jobData = tableImport(DEFAULT_JOBS_TABLE_SCHEMA, jobsImport)
+  const jobData = await tableImport(DEFAULT_JOBS_TABLE_SCHEMA, jobsImport)
 
-  const expensesData = tableImport(
+  const expensesData = await tableImport(
     DEFAULT_EXPENSES_TABLE_SCHEMA,
     expensesImport
   )
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 232ba94808..92afd0dcb1 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -130,7 +130,7 @@ export function coerce(row: any, type: string) {
  * @param {object} opts some input processing options (like disabling auto-column relationships).
  * @returns {object} the row which has been prepared to be written to the DB.
  */
-export function inputProcessing(
+export async function inputProcessing(
   userId: string | null | undefined,
   table: Table,
   row: Row,

From a1bb33bb4a79449d24a8c9dd6d52377a06f20b72 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 10:33:36 +0200
Subject: [PATCH 38/46] Process bb ref on input processing

---
 .../rowProcessor/bbReferenceProcessor.ts      |  6 ++
 .../src/utilities/rowProcessor/index.ts       | 10 ++-
 .../tests/inputProcessing.spec.ts             | 67 +++++++++++++++++++
 packages/types/src/documents/app/row.ts       | 10 +++
 4 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
 create mode 100644 packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts

diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
new file mode 100644
index 0000000000..ae31810856
--- /dev/null
+++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
@@ -0,0 +1,6 @@
+import { FieldSubtype } from "@budibase/types"
+
+export async function processOutputBBReferences(
+  value: string,
+  subtype: FieldSubtype
+) {}
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 92afd0dcb1..702c6c1d7d 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -5,8 +5,9 @@ import { ObjectStoreBuckets } from "../../constants"
 import { context, db as dbCore, objectStore } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import { Row, RowAttachment, Table } from "@budibase/types"
+import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
+import { processOutputBBReferences } from "./bbReferenceProcessor"
 export * from "./utils"
 
 type AutoColumnProcessingOpts = {
@@ -166,6 +167,13 @@ export async function inputProcessing(
         })
       }
     }
+
+    if (field.type === FieldTypes.BB_REFERENCE) {
+      clonedRow[key] = await processOutputBBReferences(
+        value,
+        field.subtype as FieldSubtype
+      )
+    }
   }
 
   if (!clonedRow._id || !clonedRow._rev) {
diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
new file mode 100644
index 0000000000..d04221c1d0
--- /dev/null
+++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
@@ -0,0 +1,67 @@
+import TestConfiguration from "../../../tests/utilities/TestConfiguration"
+import { inputProcessing } from ".."
+import { generator, structures } from "@budibase/backend-core/tests"
+import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types"
+import * as bbReferenceProcessor from "../bbReferenceProcessor"
+
+const config = new TestConfiguration()
+
+jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
+  processOutputBBReferences: jest.fn(),
+}))
+
+describe("rowProcessor - inputProcessing", () => {
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  it("populate user BB references", async () => {
+    const userId = generator.guid()
+
+    const table: Table = {
+      _id: generator.guid(),
+      name: "TestTable",
+      type: "table",
+      schema: {
+        name: {
+          type: FieldType.STRING,
+          name: "name",
+          constraints: {
+            presence: true,
+            type: "string",
+          },
+        },
+        user: {
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          name: "user",
+          constraints: {
+            presence: true,
+            type: "string",
+          },
+        },
+      },
+    }
+
+    const newRow = {
+      name: "Jack",
+      user: "123",
+    }
+
+    const user = structures.users.user()
+
+    ;(
+      bbReferenceProcessor.processOutputBBReferences as jest.Mock
+    ).mockResolvedValue(user)
+
+    const { row } = await inputProcessing(userId, table, newRow)
+
+    expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1)
+    expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledWith(
+      "123",
+      "user"
+    )
+
+    expect(row).toEqual({ ...newRow, user })
+  })
+})
diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index 6001695b13..708cbc3b9e 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -34,3 +34,13 @@ export interface Row extends Document {
   _viewId?: string
   [key: string]: any
 }
+
+export enum FieldSubtype {
+  USER = "user",
+}
+
+export const FieldTypeSubtypes = {
+  BB_REFERENCE: {
+    USER: FieldSubtype.USER,
+  },
+}

From 33d8b34ed75d72e8b43de881d5d20b9c95b28fa8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 10:41:39 +0200
Subject: [PATCH 39/46] Add tests

---
 .../tests/inputProcessing.spec.ts             | 85 ++++++++++++++++++-
 1 file changed, 84 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
index d04221c1d0..10e4a95cf9 100644
--- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
@@ -15,7 +15,11 @@ describe("rowProcessor - inputProcessing", () => {
     await config.init()
   })
 
-  it("populate user BB references", async () => {
+  beforeEach(() => {
+    jest.resetAllMocks()
+  })
+
+  it("processes BB references if on the schema and it's populated", async () => {
     const userId = generator.guid()
 
     const table: Table = {
@@ -64,4 +68,83 @@ describe("rowProcessor - inputProcessing", () => {
 
     expect(row).toEqual({ ...newRow, user })
   })
+
+  it("it does not processe BB references if on the schema but it is not populated", async () => {
+    const userId = generator.guid()
+
+    const table: Table = {
+      _id: generator.guid(),
+      name: "TestTable",
+      type: "table",
+      schema: {
+        name: {
+          type: FieldType.STRING,
+          name: "name",
+          constraints: {
+            presence: true,
+            type: "string",
+          },
+        },
+        user: {
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          name: "user",
+          constraints: {
+            presence: false,
+            type: "string",
+          },
+        },
+      },
+    }
+
+    const newRow = {
+      name: "Jack",
+    }
+
+    const { row } = await inputProcessing(userId, table, newRow)
+
+    expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
+    expect(row).toEqual({ ...newRow, user: undefined })
+  })
+
+  it("it does not processe BB references if not in the schema", async () => {
+    const userId = generator.guid()
+
+    const table: Table = {
+      _id: generator.guid(),
+      name: "TestTable",
+      type: "table",
+      schema: {
+        name: {
+          type: FieldType.STRING,
+          name: "name",
+          constraints: {
+            presence: true,
+            type: "string",
+          },
+        },
+        user: {
+          type: FieldType.NUMBER,
+          name: "user",
+          constraints: {
+            presence: true,
+            type: "string",
+          },
+        },
+      },
+    }
+
+    const newRow = {
+      name: "Jack",
+      user: "123",
+    }
+
+    const { row } = await inputProcessing(userId, table, newRow)
+
+    expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
+    expect(row).toEqual({
+      name: "Jack",
+      user: 123,
+    })
+  })
 })

From 05315df183ac277bff785fde3aa620324388b4f1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 10:44:50 +0200
Subject: [PATCH 40/46] Clean test

---
 .../utilities/rowProcessor/tests/inputProcessing.spec.ts   | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
index 10e4a95cf9..a13dbbb769 100644
--- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
@@ -1,20 +1,13 @@
-import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 import { inputProcessing } from ".."
 import { generator, structures } from "@budibase/backend-core/tests"
 import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types"
 import * as bbReferenceProcessor from "../bbReferenceProcessor"
 
-const config = new TestConfiguration()
-
 jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
   processOutputBBReferences: jest.fn(),
 }))
 
 describe("rowProcessor - inputProcessing", () => {
-  beforeAll(async () => {
-    await config.init()
-  })
-
   beforeEach(() => {
     jest.resetAllMocks()
   })

From 20ab7fb24db186da8aa2a4cb9dfae851c566cc57 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 10:54:43 +0200
Subject: [PATCH 41/46] Renames

---
 packages/server/src/utilities/rowProcessor/index.ts  |  4 ++--
 .../rowProcessor/tests/inputProcessing.spec.ts       | 12 ++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 702c6c1d7d..a69bebd0a2 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -7,7 +7,7 @@ import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
 import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
-import { processOutputBBReferences } from "./bbReferenceProcessor"
+import { processInputBBReferences } from "./bbReferenceProcessor"
 export * from "./utils"
 
 type AutoColumnProcessingOpts = {
@@ -169,7 +169,7 @@ export async function inputProcessing(
     }
 
     if (field.type === FieldTypes.BB_REFERENCE) {
-      clonedRow[key] = await processOutputBBReferences(
+      clonedRow[key] = await processInputBBReferences(
         value,
         field.subtype as FieldSubtype
       )
diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
index a13dbbb769..e291b62663 100644
--- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
@@ -4,7 +4,7 @@ import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types"
 import * as bbReferenceProcessor from "../bbReferenceProcessor"
 
 jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
-  processOutputBBReferences: jest.fn(),
+  processInputBBReferences: jest.fn(),
 }))
 
 describe("rowProcessor - inputProcessing", () => {
@@ -48,13 +48,13 @@ describe("rowProcessor - inputProcessing", () => {
     const user = structures.users.user()
 
     ;(
-      bbReferenceProcessor.processOutputBBReferences as jest.Mock
+      bbReferenceProcessor.processInputBBReferences as jest.Mock
     ).mockResolvedValue(user)
 
     const { row } = await inputProcessing(userId, table, newRow)
 
-    expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1)
-    expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledWith(
+    expect(bbReferenceProcessor.processInputBBReferences).toBeCalledTimes(1)
+    expect(bbReferenceProcessor.processInputBBReferences).toBeCalledWith(
       "123",
       "user"
     )
@@ -96,7 +96,7 @@ describe("rowProcessor - inputProcessing", () => {
 
     const { row } = await inputProcessing(userId, table, newRow)
 
-    expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
+    expect(bbReferenceProcessor.processInputBBReferences).not.toBeCalled()
     expect(row).toEqual({ ...newRow, user: undefined })
   })
 
@@ -134,7 +134,7 @@ describe("rowProcessor - inputProcessing", () => {
 
     const { row } = await inputProcessing(userId, table, newRow)
 
-    expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
+    expect(bbReferenceProcessor.processInputBBReferences).not.toBeCalled()
     expect(row).toEqual({
       name: "Jack",
       user: 123,

From 2f9843bd4cc2c39baa39fdfcb85bf6b8d8e707a3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 11:21:10 +0200
Subject: [PATCH 42/46] Handle ids

---
 .../rowProcessor/bbReferenceProcessor.ts      | 25 +++++++-
 .../tests/bbReferenceProcessor.spec.ts        | 62 +++++++++++++++++++
 2 files changed, 85 insertions(+), 2 deletions(-)
 create mode 100644 packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts

diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
index ae31810856..2d296cea3f 100644
--- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
+++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
@@ -1,6 +1,27 @@
+import { cache } from "@budibase/backend-core"
+import { utils } from "@budibase/shared-core"
 import { FieldSubtype } from "@budibase/types"
 
-export async function processOutputBBReferences(
+export async function processInputBBReferences(
   value: string,
   subtype: FieldSubtype
-) {}
+) {
+  const result = []
+  const ids = value.split(",").map((id: string) => id.trim())
+
+  switch (subtype) {
+    case FieldSubtype.USER:
+      for (const id of ids) {
+        result.push(await cache.user.getUser(id))
+      }
+      break
+    default:
+      utils.unreachable(subtype)
+  }
+
+  if (result.length > 1) {
+    return result
+  }
+
+  return result[0]
+}
diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
new file mode 100644
index 0000000000..95716b0467
--- /dev/null
+++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
@@ -0,0 +1,62 @@
+import * as backendCore from "@budibase/backend-core"
+import { FieldSubtype, User } from "@budibase/types"
+import { processInputBBReferences } from "../bbReferenceProcessor"
+import { generator, structures } from "@budibase/backend-core/tests"
+
+jest.mock("@budibase/backend-core", (): typeof backendCore => {
+  const actual = jest.requireActual("@budibase/backend-core")
+  return {
+    ...actual,
+    cache: {
+      ...actual.cache,
+      user: {
+        getUser: jest.fn(),
+        invalidateUser: jest.fn(),
+      },
+    },
+  }
+})
+
+describe("bbReferenceProcessor", () => {
+  const mockedCacheGetUser = backendCore.cache.user.getUser as jest.Mock
+
+  beforeEach(() => {
+    jest.resetAllMocks()
+  })
+
+  describe("processInputBBReferences", () => {
+    describe("subtype user", () => {
+      it("fetches by user id", async () => {
+        const input = generator.guid()
+
+        const userFromCache = structures.users.user()
+        mockedCacheGetUser.mockResolvedValueOnce(userFromCache)
+
+        const result = await processInputBBReferences(input, FieldSubtype.USER)
+
+        expect(result).toEqual(userFromCache)
+        expect(mockedCacheGetUser).toBeCalledTimes(1)
+        expect(mockedCacheGetUser).toBeCalledWith(input)
+      })
+
+      it("fetches by user id when send as csv", async () => {
+        const users: Record<string, User> = {}
+        for (let i = 0; i < 5; i++) {
+          const userId = generator.guid()
+          const user = structures.users.user({ _id: userId, userId })
+          mockedCacheGetUser.mockResolvedValueOnce(user)
+          users[userId] = user
+        }
+
+        const input = Object.keys(users).join(" ,  ")
+        const result = await processInputBBReferences(input, FieldSubtype.USER)
+
+        expect(result).toEqual(Object.values(users))
+        expect(mockedCacheGetUser).toBeCalledTimes(5)
+        Object.keys(users).forEach(userId => {
+          expect(mockedCacheGetUser).toBeCalledWith(userId)
+        })
+      })
+    })
+  })
+})

From f1a1cf74bc0486efc528811017f22ee70809c9a8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 12:07:25 +0200
Subject: [PATCH 43/46] Validate and return csv

---
 .../rowProcessor/bbReferenceProcessor.ts      | 33 ++++++++------
 .../src/utilities/rowProcessor/errors.ts      |  7 +++
 .../tests/bbReferenceProcessor.spec.ts        | 45 +++++++++++++++----
 3 files changed, 64 insertions(+), 21 deletions(-)
 create mode 100644 packages/server/src/utilities/rowProcessor/errors.ts

diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
index 2d296cea3f..7e78eb9693 100644
--- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
+++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
@@ -1,27 +1,34 @@
 import { cache } from "@budibase/backend-core"
 import { utils } from "@budibase/shared-core"
-import { FieldSubtype } from "@budibase/types"
+import { Document, FieldSubtype } from "@budibase/types"
+import { InvalidBBRefError } from "./errors"
 
 export async function processInputBBReferences(
-  value: string,
+  value: string | string[] | { _id: string } | { _id: string }[],
   subtype: FieldSubtype
-) {
-  const result = []
-  const ids = value.split(",").map((id: string) => id.trim())
+): Promise<string> {
+  const result: string[] = []
 
   switch (subtype) {
     case FieldSubtype.USER:
-      for (const id of ids) {
-        result.push(await cache.user.getUser(id))
+      if (Array.isArray(value)) {
+        result.push(...value.map(x => (typeof x === "string" ? x : x._id)))
+      } else if (typeof value !== "string") {
+        result.push(value._id)
+      } else {
+        result.push(...value.split(",").map((id: string) => id.trim()))
+      }
+
+      for (const id of result) {
+        const user = await cache.user.getUser(id)
+        if (!user) {
+          throw new InvalidBBRefError(id, FieldSubtype.USER)
+        }
       }
       break
     default:
-      utils.unreachable(subtype)
+      throw utils.unreachable(subtype)
   }
 
-  if (result.length > 1) {
-    return result
-  }
-
-  return result[0]
+  return result.join(",")
 }
diff --git a/packages/server/src/utilities/rowProcessor/errors.ts b/packages/server/src/utilities/rowProcessor/errors.ts
new file mode 100644
index 0000000000..279a528b5f
--- /dev/null
+++ b/packages/server/src/utilities/rowProcessor/errors.ts
@@ -0,0 +1,7 @@
+import { FieldSubtype } from "@budibase/types"
+
+export class InvalidBBRefError extends Error {
+  constructor(id: string, subtype: FieldSubtype) {
+    super(`Id "${id}" is not valid for the subtype "${subtype}"`)
+  }
+}
diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
index 95716b0467..7e4312b398 100644
--- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
@@ -2,6 +2,7 @@ import * as backendCore from "@budibase/backend-core"
 import { FieldSubtype, User } from "@budibase/types"
 import { processInputBBReferences } from "../bbReferenceProcessor"
 import { generator, structures } from "@budibase/backend-core/tests"
+import { InvalidBBRefError } from "../errors"
 
 jest.mock("@budibase/backend-core", (): typeof backendCore => {
   const actual = jest.requireActual("@budibase/backend-core")
@@ -26,7 +27,7 @@ describe("bbReferenceProcessor", () => {
 
   describe("processInputBBReferences", () => {
     describe("subtype user", () => {
-      it("fetches by user id", async () => {
+      it("validate valid string id", async () => {
         const input = generator.guid()
 
         const userFromCache = structures.users.user()
@@ -34,29 +35,57 @@ describe("bbReferenceProcessor", () => {
 
         const result = await processInputBBReferences(input, FieldSubtype.USER)
 
-        expect(result).toEqual(userFromCache)
+        expect(result).toEqual(input)
         expect(mockedCacheGetUser).toBeCalledTimes(1)
         expect(mockedCacheGetUser).toBeCalledWith(input)
       })
 
-      it("fetches by user id when send as csv", async () => {
-        const users: Record<string, User> = {}
+      it("throws an error given an invalid id", async () => {
+        const input = generator.guid()
+
+        await expect(
+          processInputBBReferences(input, FieldSubtype.USER)
+        ).rejects.toThrowError(new InvalidBBRefError(input, FieldSubtype.USER))
+      })
+
+      it("validates valid user ids as csv", async () => {
+        const userIds: string[] = []
         for (let i = 0; i < 5; i++) {
           const userId = generator.guid()
           const user = structures.users.user({ _id: userId, userId })
           mockedCacheGetUser.mockResolvedValueOnce(user)
-          users[userId] = user
+          userIds.push(userId)
         }
 
-        const input = Object.keys(users).join(" ,  ")
+        const input = userIds.join(" ,  ")
         const result = await processInputBBReferences(input, FieldSubtype.USER)
 
-        expect(result).toEqual(Object.values(users))
+        expect(result).toEqual(userIds.join(","))
         expect(mockedCacheGetUser).toBeCalledTimes(5)
-        Object.keys(users).forEach(userId => {
+        userIds.forEach(userId => {
           expect(mockedCacheGetUser).toBeCalledWith(userId)
         })
       })
+
+      it("throws an error given an invalid id in a csv", async () => {
+        const userId1 = generator.guid()
+        const userId2 = generator.guid()
+        const userId3 = generator.guid()
+        mockedCacheGetUser.mockResolvedValueOnce(
+          structures.users.user({ _id: userId1 })
+        )
+        mockedCacheGetUser.mockResolvedValueOnce(
+          structures.users.user({ _id: userId2 })
+        )
+
+        const input = [userId1, userId2, userId3].join(" ,  ")
+
+        await expect(
+          processInputBBReferences(input, FieldSubtype.USER)
+        ).rejects.toThrowError(
+          new InvalidBBRefError(userId3, FieldSubtype.USER)
+        )
+      })
     })
   })
 })

From b808d76734e4aed62acaa6b86fe63d49ef5b711c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 12:10:59 +0200
Subject: [PATCH 44/46] Validate objects

---
 .../tests/bbReferenceProcessor.spec.ts        | 57 +++++++++++++++----
 1 file changed, 46 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
index 7e4312b398..efc9805ef3 100644
--- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
@@ -28,24 +28,24 @@ describe("bbReferenceProcessor", () => {
   describe("processInputBBReferences", () => {
     describe("subtype user", () => {
       it("validate valid string id", async () => {
-        const input = generator.guid()
+        const userId = generator.guid()
 
         const userFromCache = structures.users.user()
         mockedCacheGetUser.mockResolvedValueOnce(userFromCache)
 
-        const result = await processInputBBReferences(input, FieldSubtype.USER)
+        const result = await processInputBBReferences(userId, FieldSubtype.USER)
 
-        expect(result).toEqual(input)
+        expect(result).toEqual(userId)
         expect(mockedCacheGetUser).toBeCalledTimes(1)
-        expect(mockedCacheGetUser).toBeCalledWith(input)
+        expect(mockedCacheGetUser).toBeCalledWith(userId)
       })
 
       it("throws an error given an invalid id", async () => {
-        const input = generator.guid()
+        const userId = generator.guid()
 
         await expect(
-          processInputBBReferences(input, FieldSubtype.USER)
-        ).rejects.toThrowError(new InvalidBBRefError(input, FieldSubtype.USER))
+          processInputBBReferences(userId, FieldSubtype.USER)
+        ).rejects.toThrowError(new InvalidBBRefError(userId, FieldSubtype.USER))
       })
 
       it("validates valid user ids as csv", async () => {
@@ -57,8 +57,11 @@ describe("bbReferenceProcessor", () => {
           userIds.push(userId)
         }
 
-        const input = userIds.join(" ,  ")
-        const result = await processInputBBReferences(input, FieldSubtype.USER)
+        const userIdCsv = userIds.join(" ,  ")
+        const result = await processInputBBReferences(
+          userIdCsv,
+          FieldSubtype.USER
+        )
 
         expect(result).toEqual(userIds.join(","))
         expect(mockedCacheGetUser).toBeCalledTimes(5)
@@ -78,14 +81,46 @@ describe("bbReferenceProcessor", () => {
           structures.users.user({ _id: userId2 })
         )
 
-        const input = [userId1, userId2, userId3].join(" ,  ")
+        const userIdCsv = [userId1, userId2, userId3].join(" ,  ")
 
         await expect(
-          processInputBBReferences(input, FieldSubtype.USER)
+          processInputBBReferences(userIdCsv, FieldSubtype.USER)
         ).rejects.toThrowError(
           new InvalidBBRefError(userId3, FieldSubtype.USER)
         )
       })
+
+      it("validate valid user object", async () => {
+        const userId = generator.guid()
+
+        const userFromCache = structures.users.user()
+        mockedCacheGetUser.mockResolvedValueOnce(userFromCache)
+
+        const result = await processInputBBReferences(
+          { _id: userId },
+          FieldSubtype.USER
+        )
+
+        expect(result).toEqual(userId)
+        expect(mockedCacheGetUser).toBeCalledTimes(1)
+        expect(mockedCacheGetUser).toBeCalledWith(userId)
+      })
+
+      it("validate valid user object array", async () => {
+        const users = Array.from({ length: 3 }, () => ({
+          _id: generator.guid(),
+        }))
+
+        mockedCacheGetUser.mockResolvedValue(structures.users.user())
+
+        const result = await processInputBBReferences(users, FieldSubtype.USER)
+
+        expect(result).toEqual(users.map(x => x._id).join(","))
+        expect(mockedCacheGetUser).toBeCalledTimes(3)
+        for (const user of users) {
+          expect(mockedCacheGetUser).toBeCalledWith(user._id)
+        }
+      })
     })
   })
 })

From 60460da4f92e5d8175eb22c0b91f96c15475e838 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 12:29:57 +0200
Subject: [PATCH 45/46] Error handling

---
 .../utilities/rowProcessor/bbReferenceProcessor.ts  | 13 ++++++++++---
 .../rowProcessor/tests/bbReferenceProcessor.spec.ts |  5 +++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
index 7e78eb9693..a9a4e5fddf 100644
--- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
+++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
@@ -20,9 +20,16 @@ export async function processInputBBReferences(
       }
 
       for (const id of result) {
-        const user = await cache.user.getUser(id)
-        if (!user) {
-          throw new InvalidBBRefError(id, FieldSubtype.USER)
+        try {
+          const user = await cache.user.getUser(id)
+          if (!user) {
+            throw new InvalidBBRefError(id, FieldSubtype.USER)
+          }
+        } catch (err: any) {
+          if (err != null && err.status === 404 && err.error === "not_found") {
+            throw new InvalidBBRefError(id, FieldSubtype.USER)
+          }
+          throw err
         }
       }
       break
diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
index efc9805ef3..748bc7efb6 100644
--- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
@@ -43,6 +43,11 @@ describe("bbReferenceProcessor", () => {
       it("throws an error given an invalid id", async () => {
         const userId = generator.guid()
 
+        mockedCacheGetUser.mockRejectedValue({
+          status: 404,
+          error: "not_found",
+        })
+
         await expect(
           processInputBBReferences(userId, FieldSubtype.USER)
         ).rejects.toThrowError(new InvalidBBRefError(userId, FieldSubtype.USER))

From 9ef710c8c0369f104f017c7dc52a1cac8cbc5a65 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Sep 2023 13:04:45 +0200
Subject: [PATCH 46/46] Handle bb ref only if there is a value

---
 .../src/utilities/rowProcessor/index.ts       |  2 +-
 .../tests/inputProcessing.spec.ts             | 46 ++++++++++++++++++-
 2 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index a69bebd0a2..e30eecf829 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -168,7 +168,7 @@ export async function inputProcessing(
       }
     }
 
-    if (field.type === FieldTypes.BB_REFERENCE) {
+    if (field.type === FieldTypes.BB_REFERENCE && value) {
       clonedRow[key] = await processInputBBReferences(
         value,
         field.subtype as FieldSubtype
diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
index e291b62663..65630da2b9 100644
--- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
@@ -62,7 +62,7 @@ describe("rowProcessor - inputProcessing", () => {
     expect(row).toEqual({ ...newRow, user })
   })
 
-  it("it does not processe BB references if on the schema but it is not populated", async () => {
+  it("it does not process BB references if on the schema but it is not populated", async () => {
     const userId = generator.guid()
 
     const table: Table = {
@@ -100,7 +100,49 @@ describe("rowProcessor - inputProcessing", () => {
     expect(row).toEqual({ ...newRow, user: undefined })
   })
 
-  it("it does not processe BB references if not in the schema", async () => {
+  it.each([undefined, null, ""])(
+    "it does not process BB references the field is $%",
+    async userValue => {
+      const userId = generator.guid()
+
+      const table: Table = {
+        _id: generator.guid(),
+        name: "TestTable",
+        type: "table",
+        schema: {
+          name: {
+            type: FieldType.STRING,
+            name: "name",
+            constraints: {
+              presence: true,
+              type: "string",
+            },
+          },
+          user: {
+            type: FieldType.BB_REFERENCE,
+            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+            name: "user",
+            constraints: {
+              presence: false,
+              type: "string",
+            },
+          },
+        },
+      }
+
+      const newRow = {
+        name: "Jack",
+        user: userValue,
+      }
+
+      const { row } = await inputProcessing(userId, table, newRow)
+
+      expect(bbReferenceProcessor.processInputBBReferences).not.toBeCalled()
+      expect(row).toEqual(newRow)
+    }
+  )
+
+  it("it does not process BB references if not in the schema", async () => {
     const userId = generator.guid()
 
     const table: Table = {