From cb3937b5c10c439ffec2c2c4910434a5927d2a6c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 18 Apr 2024 15:58:40 +0100
Subject: [PATCH 01/92] Auto fill card blocks with bindings and reset when
 changing datasource

---
 .../builder/src/stores/builder/components.js  | 57 ++++++++++++++++---
 packages/client/manifest.json                 |  9 +--
 2 files changed, 53 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index fe5f4e8a05..24789b6b05 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -1,4 +1,4 @@
-import { get, derived } from "svelte/store"
+import { get, derived, readable } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { API } from "api"
 import { Helpers } from "@budibase/bbui"
@@ -20,7 +20,7 @@ import {
   previewStore,
   tables,
   componentTreeNodesStore,
-} from "stores/builder/index"
+} from "stores/builder"
 import { buildFormSchema, getSchemaForDatasource } from "dataBinding"
 import {
   BUDIBASE_INTERNAL_DB_ID,
@@ -30,6 +30,7 @@ import {
 } from "constants/backend"
 import BudiStore from "../BudiStore"
 import { Utils } from "@budibase/frontend-core"
+import { FieldType } from "@budibase/types"
 
 export const INITIAL_COMPONENTS_STATE = {
   components: {},
@@ -295,6 +296,44 @@ export class ComponentStore extends BudiStore {
         }
       }
     })
+
+    // Add default bindings to card blocks
+    if (component._component.endsWith("/cardsblock")) {
+      const { _id, dataSource } = component
+      if (dataSource) {
+        const { schema, table } = getSchemaForDatasource(screen, dataSource)
+        const readableTypes = [
+          FieldType.STRING,
+          FieldType.OPTIONS,
+          FieldType.DATETIME,
+          FieldType.NUMBER,
+        ]
+
+        // Extract good field candidates to prefil our cards with
+        const fields = Object.entries(schema || {})
+          .filter(([name, fieldSchema]) => {
+            return (
+              readableTypes.includes(fieldSchema.type) &&
+              !fieldSchema.autoColumn &&
+              name !== table?.primaryDisplay
+            )
+          })
+          .map(([name]) => name)
+
+        // Use the primary display as the best field, if it exists
+        if (schema?.[table?.primaryDisplay]) {
+          fields.unshift(table.primaryDisplay)
+        }
+
+        // Fill our cards with as many bindings as we can
+        const cardKeys = ["cardTitle", "cardSubtitle", "cardDescription"]
+        cardKeys.forEach(key => {
+          if (!fields[0] || component[key]) return
+          component[key] = `{{ ${safe(`${_id}-repeater`)}.${safe(fields[0])} }}`
+          fields.shift()
+        })
+      }
+    }
   }
 
   /**
@@ -323,21 +362,21 @@ export class ComponentStore extends BudiStore {
       ...presetProps,
     }
 
-    // Enrich empty settings
+    // Standard post processing
     this.enrichEmptySettings(instance, {
       parent,
       screen: get(selectedScreen),
       useDefaultValues: true,
     })
-
-    // Migrate nested component settings
     this.migrateSettings(instance)
 
-    // Add any extra properties the component needs
+    // Custom post processing for creation only
     let extras = {}
     if (definition.hasChildren) {
       extras._children = []
     }
+
+    // Add step name to form steps
     if (componentName.endsWith("/formstep")) {
       const parentForm = findClosestMatchingComponent(
         get(selectedScreen).props,
@@ -350,6 +389,7 @@ export class ComponentStore extends BudiStore {
       extras.step = formSteps.length + 1
       extras._instanceName = `Step ${formSteps.length + 1}`
     }
+
     return {
       ...cloneDeep(instance),
       ...extras,
@@ -460,7 +500,6 @@ export class ComponentStore extends BudiStore {
     if (!componentId || !screenId) {
       const state = get(this.store)
       componentId = componentId || state.selectedComponentId
-
       const screenState = get(screenStore)
       screenId = screenId || screenState.selectedScreenId
     }
@@ -468,7 +507,6 @@ export class ComponentStore extends BudiStore {
       return
     }
     const patchScreen = screen => {
-      // findComponent looks in the tree not comp.settings[0]
       let component = findComponent(screen.props, componentId)
       if (!component) {
         return false
@@ -477,7 +515,8 @@ export class ComponentStore extends BudiStore {
       // Mutates the fetched component with updates
       const patchResult = patchFn(component, screen)
 
-      // Mutates the component with any required settings updates
+      // Post processing
+      this.enrichEmptySettings(component, { screen, useDefaultValues: false })
       const migrated = this.migrateSettings(component)
 
       // Returning an explicit false signifies that we should skip this
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 40abc7a9a0..91fd141704 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5990,27 +5990,28 @@
             "key": "cardTitle",
             "label": "Title",
             "nested": true,
-            "defaultValue": "Title"
+            "resetOn": "dataSource"
           },
           {
             "type": "text",
             "key": "cardSubtitle",
             "label": "Subtitle",
             "nested": true,
-            "defaultValue": "Subtitle"
+            "resetOn": "dataSource"
           },
           {
             "type": "text",
             "key": "cardDescription",
             "label": "Description",
             "nested": true,
-            "defaultValue": "Description"
+            "resetOn": "dataSource"
           },
           {
             "type": "text",
             "key": "cardImageURL",
             "label": "Image URL",
-            "nested": true
+            "nested": true,
+            "resetOn": "dataSource"
           },
           {
             "type": "boolean",

From e98a9f7f80792086cda9f4e1f0b470375673fddb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 19 Apr 2024 10:55:58 +0100
Subject: [PATCH 02/92] Update when empty setting enrichment runs to ensure
 card blocks properly reset when changing datasource

---
 .../builder/src/stores/builder/components.js  | 83 ++++++++++++-------
 1 file changed, 54 insertions(+), 29 deletions(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 24789b6b05..33d21f57cc 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -299,39 +299,64 @@ export class ComponentStore extends BudiStore {
 
     // Add default bindings to card blocks
     if (component._component.endsWith("/cardsblock")) {
-      const { _id, dataSource } = component
-      if (dataSource) {
-        const { schema, table } = getSchemaForDatasource(screen, dataSource)
-        const readableTypes = [
-          FieldType.STRING,
-          FieldType.OPTIONS,
-          FieldType.DATETIME,
-          FieldType.NUMBER,
-        ]
+      // Only proceed if the card is empty, i.e. we just changed datasource or
+      // just created the card
+      const cardKeys = ["cardTitle", "cardSubtitle", "cardDescription"]
+      if (cardKeys.every(key => !component[key]) && !component.cardImageURL) {
+        const { _id, dataSource } = component
+        if (dataSource) {
+          const { schema, table } = getSchemaForDatasource(screen, dataSource)
+          const findFieldTypes = fieldTypes => {
+            if (!Array.isArray(fieldTypes)) {
+              fieldTypes = [fieldTypes]
+            }
+            return Object.entries(schema || {})
+              .filter(([name, fieldSchema]) => {
+                return (
+                  fieldTypes.includes(fieldSchema.type) &&
+                  !fieldSchema.autoColumn &&
+                  name !== table?.primaryDisplay
+                )
+              })
+              .map(([name]) => name)
+          }
 
-        // Extract good field candidates to prefil our cards with
-        const fields = Object.entries(schema || {})
-          .filter(([name, fieldSchema]) => {
-            return (
-              readableTypes.includes(fieldSchema.type) &&
-              !fieldSchema.autoColumn &&
-              name !== table?.primaryDisplay
-            )
+          // Extract good field candidates to prefil our cards with
+          const fields = findFieldTypes([
+            FieldType.STRING,
+            FieldType.OPTIONS,
+            FieldType.DATETIME,
+            FieldType.NUMBER,
+          ])
+
+          // Use the primary display as the best field, if it exists
+          if (schema?.[table?.primaryDisplay]) {
+            fields.unshift(table.primaryDisplay)
+          }
+
+          // Fill our cards with as many bindings as we can
+          const prefix = safe(`${_id}-repeater`)
+          cardKeys.forEach(key => {
+            if (!fields[0]) return
+            component[key] = `{{ ${prefix}.${safe(fields[0])} }}`
+            fields.shift()
           })
-          .map(([name]) => name)
 
-        // Use the primary display as the best field, if it exists
-        if (schema?.[table?.primaryDisplay]) {
-          fields.unshift(table.primaryDisplay)
+          // Attempt to fill the image setting
+          let imgFields = findFieldTypes([FieldType.ATTACHMENT_SINGLE])
+          if (imgFields[0]) {
+            component.cardImageURL = `{{ ${prefix}.${safe(
+              imgFields[0]
+            )}.[url] }}`
+          } else {
+            imgFields = findFieldTypes([FieldType.ATTACHMENTS])
+            if (imgFields[0]) {
+              component.cardImageURL = `{{ ${prefix}.${safe(
+                imgFields[0]
+              )}.[0].[url] }}`
+            }
+          }
         }
-
-        // Fill our cards with as many bindings as we can
-        const cardKeys = ["cardTitle", "cardSubtitle", "cardDescription"]
-        cardKeys.forEach(key => {
-          if (!fields[0] || component[key]) return
-          component[key] = `{{ ${safe(`${_id}-repeater`)}.${safe(fields[0])} }}`
-          fields.shift()
-        })
       }
     }
   }

From 75bf928242386844a000a906e916ba08f8a4bbcd Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 19 Apr 2024 11:08:59 +0100
Subject: [PATCH 03/92] Tidy up card binding logic

---
 .../builder/src/stores/builder/components.js  | 39 ++++++++++---------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 33d21f57cc..c1c16193c2 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -321,39 +321,42 @@ export class ComponentStore extends BudiStore {
               .map(([name]) => name)
           }
 
-          // Extract good field candidates to prefil our cards with
+          // Inserts a card binding for a certain setting
+          const addBinding = (key, ...parts) => {
+            parts.unshift(`${_id}-repeater`)
+            component[key] = `{{ ${parts.map(safe).join(".")} }}`
+          }
+
+          // Extract good field candidates to prefil our cards with.
+          // Use the primary display as the best field, if it exists.
           const fields = findFieldTypes([
             FieldType.STRING,
             FieldType.OPTIONS,
             FieldType.DATETIME,
             FieldType.NUMBER,
           ])
-
-          // Use the primary display as the best field, if it exists
           if (schema?.[table?.primaryDisplay]) {
             fields.unshift(table.primaryDisplay)
           }
 
           // Fill our cards with as many bindings as we can
-          const prefix = safe(`${_id}-repeater`)
           cardKeys.forEach(key => {
-            if (!fields[0]) return
-            component[key] = `{{ ${prefix}.${safe(fields[0])} }}`
-            fields.shift()
+            if (fields[0]) {
+              addBinding(key, fields[0])
+              fields.shift()
+            }
           })
 
-          // Attempt to fill the image setting
-          let imgFields = findFieldTypes([FieldType.ATTACHMENT_SINGLE])
-          if (imgFields[0]) {
-            component.cardImageURL = `{{ ${prefix}.${safe(
-              imgFields[0]
-            )}.[url] }}`
+          // Attempt to fill the image setting.
+          // Check single attachment fields first.
+          let imgField = findFieldTypes(FieldType.ATTACHMENT_SINGLE)[0]
+          if (imgField) {
+            addBinding("cardImageURL", imgField, "url")
           } else {
-            imgFields = findFieldTypes([FieldType.ATTACHMENTS])
-            if (imgFields[0]) {
-              component.cardImageURL = `{{ ${prefix}.${safe(
-                imgFields[0]
-              )}.[0].[url] }}`
+            // Then try multi-attachment fields if no single ones exist
+            imgField = findFieldTypes(FieldType.ATTACHMENTS)[0]
+            if (imgField) {
+              addBinding("cardImageURL", imgField, 0, "url")
             }
           }
         }

From bc29d3515fcbf6acc78263b59838c58195ebead4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 19 Apr 2024 11:35:16 +0100
Subject: [PATCH 04/92] Improve card binding autofill field selection

---
 .../builder/src/stores/builder/components.js  | 63 +++++++++++--------
 1 file changed, 36 insertions(+), 27 deletions(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index c1c16193c2..7eb212d81e 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -30,7 +30,7 @@ import {
 } from "constants/backend"
 import BudiStore from "../BudiStore"
 import { Utils } from "@budibase/frontend-core"
-import { FieldType } from "@budibase/types"
+import { FieldSubtype, FieldType } from "@budibase/types"
 
 export const INITIAL_COMPONENTS_STATE = {
   components: {},
@@ -306,6 +306,8 @@ export class ComponentStore extends BudiStore {
         const { _id, dataSource } = component
         if (dataSource) {
           const { schema, table } = getSchemaForDatasource(screen, dataSource)
+
+          // Finds fields by types from the schema of the configured datasource
           const findFieldTypes = fieldTypes => {
             if (!Array.isArray(fieldTypes)) {
               fieldTypes = [fieldTypes]
@@ -315,48 +317,55 @@ export class ComponentStore extends BudiStore {
                 return (
                   fieldTypes.includes(fieldSchema.type) &&
                   !fieldSchema.autoColumn &&
-                  name !== table?.primaryDisplay
+                  name !== table?.primaryDisplay &&
+                  !name.startsWith("_")
                 )
               })
               .map(([name]) => name)
           }
 
           // Inserts a card binding for a certain setting
-          const addBinding = (key, ...parts) => {
-            parts.unshift(`${_id}-repeater`)
-            component[key] = `{{ ${parts.map(safe).join(".")} }}`
-          }
-
-          // Extract good field candidates to prefil our cards with.
-          // Use the primary display as the best field, if it exists.
-          const fields = findFieldTypes([
-            FieldType.STRING,
-            FieldType.OPTIONS,
-            FieldType.DATETIME,
-            FieldType.NUMBER,
-          ])
-          if (schema?.[table?.primaryDisplay]) {
-            fields.unshift(table.primaryDisplay)
-          }
-
-          // Fill our cards with as many bindings as we can
-          cardKeys.forEach(key => {
-            if (fields[0]) {
-              addBinding(key, fields[0])
-              fields.shift()
+          const addBinding = (key, fallback, ...parts) => {
+            if (parts.some(x => x == null)) {
+              component[key] = fallback
+            } else {
+              parts.unshift(`${_id}-repeater`)
+              component[key] = `{{ ${parts.map(safe).join(".")} }}`
             }
-          })
+          }
+
+          // Extract good field candidates to prefill our cards with.
+          // Use the primary display as the best field, if it exists.
+          const shortFields = [
+            ...findFieldTypes(FieldType.STRING),
+            ...findFieldTypes(FieldType.OPTIONS),
+            ...findFieldTypes(FieldType.ARRAY),
+            ...findFieldTypes(FieldType.DATETIME),
+            ...findFieldTypes(FieldType.NUMBER),
+          ]
+          const longFields = findFieldTypes(FieldType.LONGFORM)
+          if (schema?.[table?.primaryDisplay]) {
+            shortFields.unshift(table.primaryDisplay)
+          }
+
+          // Fill title and subtitle with short fields
+          addBinding("cardTitle", "Title", shortFields[0])
+          addBinding("cardSubtitle", "Subtitle", shortFields[1])
+
+          // Fill description with a long field if possible
+          const longField = longFields[0] ?? shortFields[2]
+          addBinding("cardDescription", "Description", longField)
 
           // Attempt to fill the image setting.
           // Check single attachment fields first.
           let imgField = findFieldTypes(FieldType.ATTACHMENT_SINGLE)[0]
           if (imgField) {
-            addBinding("cardImageURL", imgField, "url")
+            addBinding("cardImageURL", null, imgField, "url")
           } else {
             // Then try multi-attachment fields if no single ones exist
             imgField = findFieldTypes(FieldType.ATTACHMENTS)[0]
             if (imgField) {
-              addBinding("cardImageURL", imgField, 0, "url")
+              addBinding("cardImageURL", null, imgField, 0, "url")
             }
           }
         }

From e670dc1e1b5c6601e5128e70308ca78ae4e716d5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 19 Apr 2024 11:43:25 +0100
Subject: [PATCH 05/92] Lint

---
 packages/builder/src/stores/builder/components.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 7eb212d81e..7f5d7a2022 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -1,4 +1,4 @@
-import { get, derived, readable } from "svelte/store"
+import { get, derived } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { API } from "api"
 import { Helpers } from "@budibase/bbui"
@@ -30,7 +30,7 @@ import {
 } from "constants/backend"
 import BudiStore from "../BudiStore"
 import { Utils } from "@budibase/frontend-core"
-import { FieldSubtype, FieldType } from "@budibase/types"
+import { FieldType } from "@budibase/types"
 
 export const INITIAL_COMPONENTS_STATE = {
   components: {},

From ebbd0a87d4b28c9bbae80b31753f99593fb22029 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 19 Apr 2024 11:48:26 +0100
Subject: [PATCH 06/92] Remove dates from card block autofill because the
 timestamps don't look nice

---
 packages/builder/src/stores/builder/components.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 7f5d7a2022..7ee3db08cf 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -340,7 +340,6 @@ export class ComponentStore extends BudiStore {
             ...findFieldTypes(FieldType.STRING),
             ...findFieldTypes(FieldType.OPTIONS),
             ...findFieldTypes(FieldType.ARRAY),
-            ...findFieldTypes(FieldType.DATETIME),
             ...findFieldTypes(FieldType.NUMBER),
           ]
           const longFields = findFieldTypes(FieldType.LONGFORM)

From b6bcf6719fe866f3f8de26932352349543cd9dd9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 10 May 2024 11:27:49 +0100
Subject: [PATCH 07/92] Fixes an issue with fetch information being passed up
 from DatabaseImpl, making sure errors are fully sanitised.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts | 62 +++++++++++++------
 1 file changed, 44 insertions(+), 18 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index d220d0a8ac..d54e23217b 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -3,11 +3,11 @@ import {
   AllDocsResponse,
   AnyDocument,
   Database,
-  DatabaseOpts,
-  DatabaseQueryOpts,
-  DatabasePutOpts,
   DatabaseCreateIndexOpts,
   DatabaseDeleteIndexOpts,
+  DatabaseOpts,
+  DatabasePutOpts,
+  DatabaseQueryOpts,
   Document,
   isDocument,
   RowResponse,
@@ -17,7 +17,7 @@ import {
 import { getCouchInfo } from "./connections"
 import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
-import { WriteStream, ReadStream } from "fs"
+import { ReadStream, WriteStream } from "fs"
 import { newid } from "../../docIds/newid"
 import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
@@ -38,6 +38,34 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
 
 type DBCall<T> = () => Promise<T>
 
+class CouchDBError extends Error {
+  status: number
+  statusCode: number
+  reason: string
+  name: string
+  errid: string | undefined
+  description: string | undefined
+
+  constructor(
+    message: string,
+    info: {
+      status: number
+      name: string
+      errid: string
+      description: string
+      reason: string
+    }
+  ) {
+    super(message)
+    this.status = info.status
+    this.statusCode = info.status
+    this.reason = info.reason
+    this.name = info.name
+    this.errid = info.errid
+    this.description = info.description
+  }
+}
+
 export function DatabaseWithConnection(
   dbName: string,
   connection: string,
@@ -119,7 +147,7 @@ export class DatabaseImpl implements Database {
       } catch (err: any) {
         // Handling race conditions
         if (err.statusCode !== 412) {
-          throw err
+          throw new CouchDBError(err.message, err)
         }
       }
     }
@@ -138,10 +166,15 @@ export class DatabaseImpl implements Database {
       if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) {
         await this.checkAndCreateDb()
         return await this.performCall(call)
-      } else if (err.statusCode) {
-        err.status = err.statusCode
       }
-      throw err
+      // stripping the error down the props which are safe/useful, drop everything else
+      throw new CouchDBError(`CouchDB error: ${err.message}`, {
+        status: err.status || err.statusCode,
+        name: err.name,
+        errid: err.errid,
+        description: err.description,
+        reason: err.reason,
+      })
     }
   }
 
@@ -281,16 +314,9 @@ export class DatabaseImpl implements Database {
   }
 
   async destroy() {
-    try {
-      return await this.nano().db.destroy(this.name)
-    } catch (err: any) {
-      // didn't exist, don't worry
-      if (err.statusCode === 404) {
-        return
-      } else {
-        throw { ...err, status: err.statusCode }
-      }
-    }
+    return this.performCall(async () => {
+      return () => this.nano().db.destroy(this.name)
+    })
   }
 
   async compact() {

From f036776a907d12204163a7ce3764afa64465013d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 10 May 2024 11:32:57 +0100
Subject: [PATCH 08/92] One small change to keep 404 functionality on destroy
 DB.

---
 packages/backend-core/src/db/couch/DatabaseImpl.ts | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index d54e23217b..ca8a22b54e 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -314,9 +314,16 @@ export class DatabaseImpl implements Database {
   }
 
   async destroy() {
-    return this.performCall(async () => {
-      return () => this.nano().db.destroy(this.name)
-    })
+    try {
+      return await this.nano().db.destroy(this.name)
+    } catch (err: any) {
+      // didn't exist, don't worry
+      if (err.statusCode === 404) {
+        return
+      } else {
+        throw new CouchDBError(err.message, err)
+      }
+    }
   }
 
   async compact() {

From c9ec06b5b1d4e790c29542a9d708aabb1709237d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 10 May 2024 11:51:57 +0100
Subject: [PATCH 09/92] Adding error field.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts     | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index ca8a22b54e..c520f4d81f 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -43,8 +43,9 @@ class CouchDBError extends Error {
   statusCode: number
   reason: string
   name: string
-  errid: string | undefined
-  description: string | undefined
+  errid: string
+  error: string
+  description: string
 
   constructor(
     message: string,
@@ -54,6 +55,7 @@ class CouchDBError extends Error {
       errid: string
       description: string
       reason: string
+      error: string
     }
   ) {
     super(message)
@@ -63,6 +65,7 @@ class CouchDBError extends Error {
     this.name = info.name
     this.errid = info.errid
     this.description = info.description
+    this.error = info.error
   }
 }
 
@@ -168,13 +171,7 @@ export class DatabaseImpl implements Database {
         return await this.performCall(call)
       }
       // stripping the error down the props which are safe/useful, drop everything else
-      throw new CouchDBError(`CouchDB error: ${err.message}`, {
-        status: err.status || err.statusCode,
-        name: err.name,
-        errid: err.errid,
-        description: err.description,
-        reason: err.reason,
-      })
+      throw new CouchDBError(`CouchDB error: ${err.message}`, err)
     }
   }
 

From a1a50de61c611d7bcff22cbded4af61d9ce661f5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 10 May 2024 11:59:11 +0100
Subject: [PATCH 10/92] Final final fix.

---
 packages/backend-core/src/db/couch/DatabaseImpl.ts | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index c520f4d81f..ef351f7d4d 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -50,7 +50,8 @@ class CouchDBError extends Error {
   constructor(
     message: string,
     info: {
-      status: number
+      status: number | undefined
+      statusCode: number | undefined
       name: string
       errid: string
       description: string
@@ -59,8 +60,9 @@ class CouchDBError extends Error {
     }
   ) {
     super(message)
-    this.status = info.status
-    this.statusCode = info.status
+    const statusCode = info.status || info.statusCode || 500
+    this.status = statusCode
+    this.statusCode = statusCode
     this.reason = info.reason
     this.name = info.name
     this.errid = info.errid

From 1365d190488e816f96f49a0568e74e4f05d861cc Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 10 May 2024 12:03:24 +0100
Subject: [PATCH 11/92] Updating pro reference.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 479879246a..ff397e5454 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 479879246aac5dd3073cc695945c62c41fae5b0e
+Subproject commit ff397e5454ad3361b25efdf14746c36dcbd3f409

From bec7b782775df65e298a483ce5f141e60c54320b Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 10 May 2024 11:10:28 +0000
Subject: [PATCH 12/92] Bump version to 2.24.3

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 9c5a6c6bab..7daf0b039b 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.24.2",
+  "version": "2.24.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From ad57776b7fe780296695a93a1c22e190157edc2e Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 10 May 2024 11:13:00 +0000
Subject: [PATCH 13/92] Bump version to 2.25.0

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 7daf0b039b..16dc73aa30 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.24.3",
+  "version": "2.25.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From e8b8e6e8b4385cb5e167b1467e1d91ad6768e377 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Fri, 10 May 2024 13:18:30 +0100
Subject: [PATCH 14/92] Allow Fancy Input validation to be triggered onBlur
 (#13658)

* Add free_trial to deploy camunda script

* Allow for more validation customisation on fancy input
---
 packages/bbui/src/FancyForm/FancyInput.svelte | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/packages/bbui/src/FancyForm/FancyInput.svelte b/packages/bbui/src/FancyForm/FancyInput.svelte
index 0c58b9b045..f665fa5724 100644
--- a/packages/bbui/src/FancyForm/FancyInput.svelte
+++ b/packages/bbui/src/FancyForm/FancyInput.svelte
@@ -11,6 +11,7 @@
   export let error = null
   export let validate = null
   export let suffix = null
+  export let validateOn = "change"
 
   const dispatch = createEventDispatcher()
 
@@ -24,7 +25,16 @@
     const newValue = e.target.value
     dispatch("change", newValue)
     value = newValue
-    if (validate) {
+    if (validate && (error || validateOn === "change")) {
+      error = validate(newValue)
+    }
+  }
+
+  const onBlur = e => {
+    focused = false
+    const newValue = e.target.value
+    dispatch("blur", newValue)
+    if (validate && validateOn === "blur") {
       error = validate(newValue)
     }
   }
@@ -61,7 +71,7 @@
     type={type || "text"}
     on:input={onChange}
     on:focus={() => (focused = true)}
-    on:blur={() => (focused = false)}
+    on:blur={onBlur}
     class:placeholder
     bind:this={ref}
   />

From 16e58a38eaec8f483f6e08b0044bcd5dd13befa2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 May 2024 10:51:40 +0200
Subject: [PATCH 15/92] Support pg time types

---
 packages/server/src/integrations/utils/utils.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts
index 892d8ae034..466f539ef3 100644
--- a/packages/server/src/integrations/utils/utils.ts
+++ b/packages/server/src/integrations/utils/utils.ts
@@ -71,7 +71,11 @@ const SQL_DATE_TYPE_MAP: Record<string, PrimitiveTypes> = {
 }
 
 const SQL_DATE_ONLY_TYPES = ["date"]
-const SQL_TIME_ONLY_TYPES = ["time"]
+const SQL_TIME_ONLY_TYPES = [
+  "time",
+  "time without time zone",
+  "time with time zone",
+]
 
 const SQL_STRING_TYPE_MAP: Record<string, PrimitiveTypes> = {
   varchar: FieldType.STRING,

From a81626005c7b65c1321b462d861ca64dac977561 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 May 2024 15:55:27 +0200
Subject: [PATCH 16/92] Save timeonly on external db

---
 .../api/controllers/row/ExternalRequest.ts    | 120 ++++++++++--------
 1 file changed, 64 insertions(+), 56 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index bd92413851..823330d601 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -1,3 +1,4 @@
+import dayjs from "dayjs"
 import {
   AutoFieldSubType,
   AutoReason,
@@ -285,65 +286,72 @@ export class ExternalRequest<T extends Operation> {
       // parse floats/numbers
       if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) {
         newRow[key] = parseFloat(row[key])
-      }
-      // if its not a link then just copy it over
-      if (field.type !== FieldType.LINK) {
-        newRow[key] = row[key]
-        continue
-      }
-      const { tableName: linkTableName } = breakExternalTableId(field?.tableId)
-      // table has to exist for many to many
-      if (!linkTableName || !this.tables[linkTableName]) {
-        continue
-      }
-      const linkTable = this.tables[linkTableName]
-      // @ts-ignore
-      const linkTablePrimary = linkTable.primary[0]
-      // one to many
-      if (isOneSide(field)) {
-        let id = row[key][0]
-        if (id) {
-          if (typeof row[key] === "string") {
-            id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
-          }
-          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
-        } else {
-          // Removing from both new and row, as we don't know if it has already been processed
-          row[field.foreignKey || linkTablePrimary] = null
-          newRow[field.foreignKey || linkTablePrimary] = null
+      } else if (field.type === FieldType.LINK) {
+        const { tableName: linkTableName } = breakExternalTableId(
+          field?.tableId
+        )
+        // table has to exist for many to many
+        if (!linkTableName || !this.tables[linkTableName]) {
+          continue
         }
-      }
-      // many to many
-      else if (isManyToMany(field)) {
-        // we're not inserting a doc, will be a bunch of update calls
-        const otherKey: string = field.throughFrom || linkTablePrimary
-        const thisKey: string = field.throughTo || tablePrimary
-        for (const relationship of row[key]) {
-          manyRelationships.push({
-            tableId: field.through || field.tableId,
-            isUpdate: false,
-            key: otherKey,
-            [otherKey]: breakRowIdField(relationship)[0],
-            // leave the ID for enrichment later
-            [thisKey]: `{{ literal ${tablePrimary} }}`,
-          })
-        }
-      }
-      // many to one
-      else {
-        const thisKey: string = "id"
+        const linkTable = this.tables[linkTableName]
         // @ts-ignore
-        const otherKey: string = field.fieldName
-        for (const relationship of row[key]) {
-          manyRelationships.push({
-            tableId: field.tableId,
-            isUpdate: true,
-            key: otherKey,
-            [thisKey]: breakRowIdField(relationship)[0],
-            // leave the ID for enrichment later
-            [otherKey]: `{{ literal ${tablePrimary} }}`,
-          })
+        const linkTablePrimary = linkTable.primary[0]
+        // one to many
+        if (isOneSide(field)) {
+          let id = row[key][0]
+          if (id) {
+            if (typeof row[key] === "string") {
+              id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+            }
+            newRow[field.foreignKey || linkTablePrimary] =
+              breakRowIdField(id)[0]
+          } else {
+            // Removing from both new and row, as we don't know if it has already been processed
+            row[field.foreignKey || linkTablePrimary] = null
+            newRow[field.foreignKey || linkTablePrimary] = null
+          }
         }
+        // many to many
+        else if (isManyToMany(field)) {
+          // we're not inserting a doc, will be a bunch of update calls
+          const otherKey: string = field.throughFrom || linkTablePrimary
+          const thisKey: string = field.throughTo || tablePrimary
+          for (const relationship of row[key]) {
+            manyRelationships.push({
+              tableId: field.through || field.tableId,
+              isUpdate: false,
+              key: otherKey,
+              [otherKey]: breakRowIdField(relationship)[0],
+              // leave the ID for enrichment later
+              [thisKey]: `{{ literal ${tablePrimary} }}`,
+            })
+          }
+        }
+        // many to one
+        else {
+          const thisKey: string = "id"
+          // @ts-ignore
+          const otherKey: string = field.fieldName
+          for (const relationship of row[key]) {
+            manyRelationships.push({
+              tableId: field.tableId,
+              isUpdate: true,
+              key: otherKey,
+              [thisKey]: breakRowIdField(relationship)[0],
+              // leave the ID for enrichment later
+              [otherKey]: `{{ literal ${tablePrimary} }}`,
+            })
+          }
+        }
+      } else if (
+        field.type === FieldType.DATETIME &&
+        field.timeOnly &&
+        row[key]
+      ) {
+        newRow[key] = dayjs(row[key]).format("HH:mm")
+      } else {
+        newRow[key] = row[key]
       }
     }
     // we return the relationships that may need to be created in the through table

From 76ac300cf004d947272cc95a9745e4a03c3021ba Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 May 2024 16:53:28 +0200
Subject: [PATCH 17/92] Save only if valid

---
 packages/server/src/api/controllers/row/ExternalRequest.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 823330d601..b30c97e289 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -347,7 +347,8 @@ export class ExternalRequest<T extends Operation> {
       } else if (
         field.type === FieldType.DATETIME &&
         field.timeOnly &&
-        row[key]
+        row[key] &&
+        dayjs(row[key]).isValid()
       ) {
         newRow[key] = dayjs(row[key]).format("HH:mm")
       } else {

From 902613d6007a2b3c49d3cac30af9aa583046365f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 20 May 2024 12:00:08 +0100
Subject: [PATCH 18/92] Working towards user relationship tests passing.

---
 .../src/api/routes/tests/search.spec.ts       | 134 +++++++++++++++++-
 packages/server/src/integrations/base/sql.ts  |   2 +-
 .../server/src/sdk/app/rows/search/sqs.ts     |   3 +-
 3 files changed, 131 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index d036da646e..426f383ad0 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1,6 +1,6 @@
 import { tableForDatasource } from "../../../tests/utilities/structures"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
-import { db as dbCore } from "@budibase/backend-core"
+import { db as dbCore, utils } from "@budibase/backend-core"
 
 import * as setup from "./utilities"
 import {
@@ -25,12 +25,12 @@ const serverTime = new Date("2024-05-06T00:00:00.000Z")
 tk.freeze(serverTime)
 
 describe.each([
-  ["lucene", undefined],
+  //["lucene", undefined],
   ["sqs", undefined],
-  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
 ])("/api/:sourceId/search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
   const isLucene = name === "lucene"
@@ -1152,4 +1152,126 @@ describe.each([
             ]))
         })
     })
+
+  describe("user", () => {
+    let user1: User
+    let user2: User
+
+    beforeAll(async () => {
+      user1 = await config.createUser({ _id: `us_${utils.newid()}` })
+      user2 = await config.createUser({ _id: `us_${utils.newid()}` })
+
+      await createTable({
+        user: {
+          name: "user",
+          type: FieldType.BB_REFERENCE_SINGLE,
+          subtype: BBReferenceFieldSubType.USER,
+        },
+      })
+
+      await createRows([
+        { user: JSON.stringify(user1) },
+        { user: JSON.stringify(user2) },
+      ])
+    })
+
+    describe("equal", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ equal: { user: user1._id } }).toContainExactly([
+          { user: { _id: user1._id } },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ equal: { user: "us_none" } }).toFindNothing())
+    })
+
+    describe("notEqual", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ notEqual: { user: user1._id } }).toContainExactly([
+          { user: { _id: user2._id } },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([
+          { user: { _id: user1._id } },
+          { user: { _id: user2._id } },
+        ]))
+    })
+
+    describe("oneOf", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([
+          { user: { _id: user1._id } },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing())
+    })
+  })
+
+  describe("multi user", () => {
+    let user1: User
+    let user2: User
+
+    beforeAll(async () => {
+      user1 = await config.createUser({ _id: `us_${utils.newid()}` })
+      user2 = await config.createUser({ _id: `us_${utils.newid()}` })
+
+      await createTable({
+        users: {
+          name: "users",
+          type: FieldType.BB_REFERENCE,
+          subtype: BBReferenceFieldSubType.USER,
+        },
+      })
+
+      await createRows([
+        { users: JSON.stringify([user1]) },
+        { users: JSON.stringify([user2]) },
+        { users: JSON.stringify([user1, user2]) },
+        { users: JSON.stringify([]) },
+      ])
+    })
+
+    describe("contains", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ contains: { users: [user1._id] } }).toContainExactly([
+          { users: [{ _id: user1._id }] },
+          { users: [{ _id: user1._id }, { _id: user2._id }] },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ contains: { users: ["us_none"] } }).toFindNothing())
+    })
+
+    describe("notContains", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([
+          { users: [{ _id: user2._id }] },
+          { users: [] },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ notContains: { users: ["us_none"] } }).toContainExactly([
+          { users: [{ _id: user1._id }] },
+          { users: [{ _id: user2._id }] },
+          { users: [{ _id: user1._id }, { _id: user2._id }] },
+          { users: [] },
+        ]))
+    })
+
+    describe("containsAny", () => {
+      it("successfully finds rows", () =>
+        expectQuery({
+          containsAny: { users: [user1._id, user2._id] },
+        }).toContainExactly([
+          { users: [{ _id: user1._id }] },
+          { users: [{ _id: user2._id }] },
+          { users: [{ _id: user1._id }, { _id: user2._id }] },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing())
+    })
+  })
 })
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 7a2b819007..33e276c81b 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -271,7 +271,7 @@ class InternalBuilder {
             }
             statement +=
               (statement ? andOr : "") +
-              `LOWER(${likeKey(this.client, key)}) LIKE ?`
+              `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)`
           }
 
           if (statement === "") {
diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 05b1a3bd96..7aaaa6bd6c 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -172,7 +172,8 @@ export async function search(
       sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
 
       const db = context.getAppDB()
-      return await db.sql<Row>(sql, bindings)
+      const rows = await db.sql<Row>(sql, bindings)
+      return rows
     })
 
     // process from the format of tableId.column to expected format

From 1562e7b1f10ae920ece4c6006a75bb496dde93bc Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 20 May 2024 12:05:01 +0100
Subject: [PATCH 19/92] Working towards user relationship tests passing.

---
 packages/server/src/api/routes/tests/search.spec.ts | 2 +-
 packages/server/src/integrations/base/sql.ts        | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index f20e0534e5..f777eb6db1 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -406,7 +406,7 @@ describe.each([
       ])
     })
 
-    it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => {
+    it.only("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => {
       const jsBinding =
         "const currentTime = new Date()\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();"
       const encodedBinding = encodeJSBinding(jsBinding)
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 1140a1ac54..1c0c252b1c 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -272,7 +272,8 @@ class InternalBuilder {
             }
             statement +=
               (statement ? andOr : "") +
-              `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)`
+              // `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)`
+              `LOWER(${likeKey(this.client, key)}) LIKE ?`
           }
 
           if (statement === "") {

From a74f82a5359a01cd0ba02b67d0dcf8660c9ae79a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 13:06:45 +0200
Subject: [PATCH 20/92] Use native inputs

---
 .../Form/Core/DatePicker/NumberInput.svelte   |  3 +-
 .../Form/Core/DatePicker/TimePicker.svelte    | 37 +++++--------------
 2 files changed, 11 insertions(+), 29 deletions(-)

diff --git a/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte b/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte
index dc4886d28d..7e013341ac 100644
--- a/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte
+++ b/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte
@@ -4,13 +4,14 @@
   export let max
   export let hideArrows = false
   export let width
+  export let type = "number"
 
   $: style = width ? `width:${width}px;` : ""
 </script>
 
 <input
   class:hide-arrows={hideArrows}
-  type="number"
+  {type}
   {style}
   {value}
   {min}
diff --git a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
index 047e5a4f08..af38eaef7a 100644
--- a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
+++ b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
@@ -1,6 +1,4 @@
 <script>
-  import { cleanInput } from "./utils"
-  import dayjs from "dayjs"
   import NumberInput from "./NumberInput.svelte"
   import { createEventDispatcher } from "svelte"
 
@@ -8,39 +6,22 @@
 
   const dispatch = createEventDispatcher()
 
-  $: displayValue = value || dayjs()
+  $: displayValue = value?.format("HH:mm")
 
-  const handleHourChange = e => {
-    dispatch("change", displayValue.hour(parseInt(e.target.value)))
+  const handleChange = e => {
+    const [hour, minute] = e.target.value.split(":").map(x => parseInt(x))
+    dispatch("change", value.hour(hour).minute(minute))
   }
-
-  const handleMinuteChange = e => {
-    dispatch("change", displayValue.minute(parseInt(e.target.value)))
-  }
-
-  const cleanHour = cleanInput({ max: 23, pad: 2, fallback: "00" })
-  const cleanMinute = cleanInput({ max: 59, pad: 2, fallback: "00" })
 </script>
 
 <div class="time-picker">
   <NumberInput
     hideArrows
-    value={displayValue.hour().toString().padStart(2, "0")}
-    min={0}
-    max={23}
-    width={20}
-    on:input={cleanHour}
-    on:change={handleHourChange}
-  />
-  <span>:</span>
-  <NumberInput
-    hideArrows
-    value={displayValue.minute().toString().padStart(2, "0")}
-    min={0}
-    max={59}
-    width={20}
-    on:input={cleanMinute}
-    on:change={handleMinuteChange}
+    type={"time"}
+    value={displayValue}
+    width={60}
+    on:input={handleChange}
+    on:change={handleChange}
   />
 </div>
 

From 5dc75582d9532042dc46db5acc76f08333a1bb1d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 13:09:28 +0200
Subject: [PATCH 21/92] Fix timezone issues

---
 packages/bbui/src/helpers.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/bbui/src/helpers.js b/packages/bbui/src/helpers.js
index 90b447f3c1..66bc6551d8 100644
--- a/packages/bbui/src/helpers.js
+++ b/packages/bbui/src/helpers.js
@@ -166,7 +166,7 @@ export const stringifyDate = (
   const offsetForTimezone = (enableTime && ignoreTimezones) || timeOnly
   if (offsetForTimezone) {
     // Ensure we use the correct offset for the date
-    const referenceDate = timeOnly ? new Date() : value.toDate()
+    const referenceDate = value.toDate()
     const offset = referenceDate.getTimezoneOffset() * 60000
     return new Date(value.valueOf() - offset).toISOString().slice(0, -1)
   }

From 995d2154ca3edf983b5993ee283be62e852126ae Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 20 May 2024 14:09:07 +0100
Subject: [PATCH 22/92] Included blob: in worker csp security header. Required
 for atrament to load

---
 hosting/proxy/nginx.prod.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf
index 79007da311..12b8df049f 100644
--- a/hosting/proxy/nginx.prod.conf
+++ b/hosting/proxy/nginx.prod.conf
@@ -61,7 +61,7 @@ http {
     set $csp_img "img-src http: https: data: blob:";
     set $csp_manifest "manifest-src 'self'";
     set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live"; 
-    set $csp_worker "worker-src 'none'";
+    set $csp_worker "worker-src blob:";
 
     error_page 502 503 504 /error.html;
     location = /error.html {

From 9f759220118d603b7a16259ca81fba4d6d2ce933 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 16:31:11 +0200
Subject: [PATCH 23/92] SQL time only column creation as time

---
 packages/server/src/integrations/base/sqlTable.ts | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts
index a82a9fcea8..9871d57a3a 100644
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@@ -79,9 +79,13 @@ function generateSchema(
         schema.boolean(key)
         break
       case FieldType.DATETIME:
-        schema.datetime(key, {
-          useTz: !column.ignoreTimezones,
-        })
+        if (!column.timeOnly) {
+          schema.datetime(key, {
+            useTz: !column.ignoreTimezones,
+          })
+        } else {
+          schema.time(key)
+        }
         break
       case FieldType.ARRAY:
       case FieldType.BB_REFERENCE:

From 57e73488b572a2834a01d717743acb046e51b7d0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 16:34:00 +0200
Subject: [PATCH 24/92] Add tests

---
 .../src/api/routes/tests/search.spec.ts       | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 87d0aa72c7..9f7209129a 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -962,6 +962,36 @@ describe.each([
     })
   })
 
+  describe("datetime - time only", () => {
+    const T_1000 = "10:00"
+    const T_1045 = "10:45"
+    const T_1200 = "12:00"
+    const T_1530 = "15:30"
+    const T_0000 = "00:00"
+
+    beforeAll(async () => {
+      await createTable({
+        time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
+      })
+
+      await createRows(
+        _.shuffle([T_1000, T_1045, T_1200, T_1530, T_0000]).map(time => ({
+          time,
+        }))
+      )
+    })
+
+    describe("equal", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ equal: { time: T_1000 } }).toContainExactly([
+          { time: "10:00:00" },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ equal: { time: "10:01" } }).toFindNothing())
+    })
+  })
+
   describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
     beforeAll(async () => {
       await createTable({

From 318dd5e6288d8fc70e5850c2805239112e51feab Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 16:34:22 +0200
Subject: [PATCH 25/92] Fix time and mssql tests

---
 packages/server/src/integrations/base/sql.ts | 26 +++++++++++++++-----
 packages/server/src/utilities/schema.ts      |  5 ++--
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 85db642e47..dd663dc918 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -122,11 +122,8 @@ function generateSelectStatement(
     const fieldNames = field.split(/\./g)
     const tableName = fieldNames[0]
     const columnName = fieldNames[1]
-    if (
-      columnName &&
-      schema?.[columnName] &&
-      knex.client.config.client === SqlClient.POSTGRES
-    ) {
+    const columnSchema = schema?.[columnName]
+    if (columnSchema && knex.client.config.client === SqlClient.POSTGRES) {
       const externalType = schema[columnName].externalType
       if (externalType?.includes("money")) {
         return knex.raw(
@@ -134,6 +131,13 @@ function generateSelectStatement(
         )
       }
     }
+    if (
+      knex.client.config.client === SqlClient.MS_SQL &&
+      columnSchema.type === FieldType.DATETIME &&
+      columnSchema.timeOnly
+    ) {
+      return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
+    }
     return `${field} as ${field}`
   })
 }
@@ -634,13 +638,23 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
    */
   _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] {
     const sqlClient = this.getSqlClient()
-    const config: { client: string; useNullAsDefault?: boolean } = {
+    const config: Knex.Config = {
       client: sqlClient,
     }
     if (sqlClient === SqlClient.SQL_LITE) {
       config.useNullAsDefault = true
     }
+
+    if (sqlClient === SqlClient.MS_SQL) {
+      // config.connection ??= {}
+      // config.connection.typeCast = (field: any, next: any): any => {
+      //   if (field.type === "TIME") return field.string()
+      //   return next()
+      // }
+    }
+
     const client = knex(config)
+
     let query: Knex.QueryBuilder
     const builder = new InternalBuilder(sqlClient)
     switch (this._operation(json)) {
diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 4f0feb3c93..a205bf8c11 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -129,11 +129,12 @@ export function parse(rows: Rows, schema: TableSchema): Rows {
         return
       }
 
-      const { type: columnType } = schema[columnName]
+      const columnSchema = schema[columnName]
+      const { type: columnType } = columnSchema
       if (columnType === FieldType.NUMBER) {
         // If provided must be a valid number
         parsedRow[columnName] = columnData ? Number(columnData) : columnData
-      } else if (columnType === FieldType.DATETIME) {
+      } else if (columnType === FieldType.DATETIME && !columnSchema.timeOnly) {
         // If provided must be a valid date
         parsedRow[columnName] = columnData
           ? new Date(columnData).toISOString()

From 684e3607157c576a3ee0cdbb3bae03e43b14ddd5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 16:49:38 +0200
Subject: [PATCH 26/92] Add other tests

---
 .../src/api/routes/tests/search.spec.ts       | 159 +++++++++++++++---
 1 file changed, 136 insertions(+), 23 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 9f7209129a..5e306739c9 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -962,35 +962,148 @@ describe.each([
     })
   })
 
-  describe("datetime - time only", () => {
-    const T_1000 = "10:00"
-    const T_1045 = "10:45"
-    const T_1200 = "12:00"
-    const T_1530 = "15:30"
-    const T_0000 = "00:00"
+  !isInternal &&
+    describe("datetime - time only", () => {
+      const T_1000 = "10:00"
+      const T_1045 = "10:45"
+      const T_1200 = "12:00"
+      const T_1530 = "15:30"
+      const T_0000 = "00:00"
 
-    beforeAll(async () => {
-      await createTable({
-        time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
+      const UNEXISTING_TIME = "10:01"
+
+      beforeAll(async () => {
+        await createTable({
+          time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
+        })
+
+        await createRows(
+          _.shuffle([T_1000, T_1045, T_1200, T_1530, T_0000]).map(time => ({
+            time,
+          }))
+        )
       })
 
-      await createRows(
-        _.shuffle([T_1000, T_1045, T_1200, T_1530, T_0000]).map(time => ({
-          time,
-        }))
-      )
-    })
+      describe("equal", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ equal: { time: T_1000 } }).toContainExactly([
+            { time: "10:00:00" },
+          ]))
 
-    describe("equal", () => {
-      it("successfully finds a row", () =>
-        expectQuery({ equal: { time: T_1000 } }).toContainExactly([
-          { time: "10:00:00" },
-        ]))
+        it("fails to find nonexistent row", () =>
+          expectQuery({ equal: { time: UNEXISTING_TIME } }).toFindNothing())
+      })
 
-      it("fails to find nonexistent row", () =>
-        expectQuery({ equal: { time: "10:01" } }).toFindNothing())
+      describe("notEqual", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([
+            { time: "10:45:00" },
+            { time: "12:00:00" },
+            { time: "15:30:00" },
+            { time: "00:00:00" },
+          ]))
+
+        it("return all when requesting non-existing", () =>
+          expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly(
+            [
+              { time: "10:00:00" },
+              { time: "10:45:00" },
+              { time: "12:00:00" },
+              { time: "15:30:00" },
+              { time: "00:00:00" },
+            ]
+          ))
+      })
+
+      describe("oneOf", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([
+            { time: "10:00:00" },
+          ]))
+
+        it("fails to find nonexistent row", () =>
+          expectQuery({ oneOf: { time: [UNEXISTING_TIME] } }).toFindNothing())
+      })
+
+      describe("range", () => {
+        it("successfully finds a row", () =>
+          expectQuery({
+            range: { time: { low: T_1045, high: T_1045 } },
+          }).toContainExactly([{ time: "10:45:00" }]))
+
+        it("successfully finds multiple rows", () =>
+          expectQuery({
+            range: { time: { low: T_1045, high: T_1530 } },
+          }).toContainExactly([
+            { time: "10:45:00" },
+            { time: "12:00:00" },
+            { time: "15:30:00" },
+          ]))
+
+        it("successfully finds no rows", () =>
+          expectQuery({
+            range: { time: { low: UNEXISTING_TIME, high: UNEXISTING_TIME } },
+          }).toFindNothing())
+      })
+
+      describe("sort", () => {
+        it("sorts ascending", () =>
+          expectSearch({
+            query: {},
+            sort: "time",
+            sortOrder: SortOrder.ASCENDING,
+          }).toMatchExactly([
+            { time: "00:00:00" },
+            { time: "10:00:00" },
+            { time: "10:45:00" },
+            { time: "12:00:00" },
+            { time: "15:30:00" },
+          ]))
+
+        it("sorts descending", () =>
+          expectSearch({
+            query: {},
+            sort: "time",
+            sortOrder: SortOrder.DESCENDING,
+          }).toMatchExactly([
+            { time: "15:30:00" },
+            { time: "12:00:00" },
+            { time: "10:45:00" },
+            { time: "10:00:00" },
+            { time: "00:00:00" },
+          ]))
+
+        describe("sortType STRING", () => {
+          it("sorts ascending", () =>
+            expectSearch({
+              query: {},
+              sort: "time",
+              sortType: SortType.STRING,
+              sortOrder: SortOrder.ASCENDING,
+            }).toMatchExactly([
+              { time: "00:00:00" },
+              { time: "10:00:00" },
+              { time: "10:45:00" },
+              { time: "12:00:00" },
+              { time: "15:30:00" },
+            ]))
+
+          it("sorts descending", () =>
+            expectSearch({
+              query: {},
+              sort: "time",
+              sortType: SortType.STRING,
+              sortOrder: SortOrder.DESCENDING,
+            }).toMatchExactly([
+              { time: "15:30:00" },
+              { time: "12:00:00" },
+              { time: "10:45:00" },
+              { time: "10:00:00" },
+              { time: "00:00:00" },
+            ]))
+        })
+      })
     })
-  })
 
   describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
     beforeAll(async () => {

From 7d709d0d221963234c1c3e97c6f3440d1f4c11d0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 16:57:03 +0200
Subject: [PATCH 27/92] Fix flaky test

---
 packages/server/src/api/routes/tests/search.spec.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 5e306739c9..fec0c4738e 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -407,8 +407,7 @@ describe.each([
     })
 
     it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => {
-      const jsBinding =
-        "const currentTime = new Date()\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();"
+      const jsBinding = `const currentTime = new Date(${Date.now()})\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();`
       const encodedBinding = encodeJSBinding(jsBinding)
 
       await expectQuery({

From 028aaa0bb40d1788bb4539b1c681a82c6bf03234 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 17:00:12 +0200
Subject: [PATCH 28/92] Clean

---
 packages/server/src/integrations/base/sql.ts | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index dd663dc918..1b8d2ea4ae 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -645,14 +645,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
       config.useNullAsDefault = true
     }
 
-    if (sqlClient === SqlClient.MS_SQL) {
-      // config.connection ??= {}
-      // config.connection.typeCast = (field: any, next: any): any => {
-      //   if (field.type === "TIME") return field.string()
-      //   return next()
-      // }
-    }
-
     const client = knex(config)
 
     let query: Knex.QueryBuilder

From 356da44b4b4133370fcd9b89e578d2ee144c9be2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 17:00:51 +0200
Subject: [PATCH 29/92] Clean code

---
 packages/server/src/integrations/base/sql.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 1b8d2ea4ae..59efe9a9ba 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -646,7 +646,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     }
 
     const client = knex(config)
-
     let query: Knex.QueryBuilder
     const builder = new InternalBuilder(sqlClient)
     switch (this._operation(json)) {

From 1d00604674d0c85fc8489f2015c34ef0a6557a8b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 May 2024 17:06:34 +0200
Subject: [PATCH 30/92] Lint

---
 packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
index af38eaef7a..6f1490a573 100644
--- a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
+++ b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
@@ -31,10 +31,4 @@
     flex-direction: row;
     align-items: center;
   }
-  .time-picker span {
-    font-weight: bold;
-    font-size: 18px;
-    z-index: 0;
-    margin-bottom: 1px;
-  }
 </style>

From e2a1ab7eaf40ff7140a0380778fba40b6ca6ce48 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 20 May 2024 17:01:52 +0100
Subject: [PATCH 31/92] All tests passing.

---
 .../src/api/routes/tests/search.spec.ts       | 53 +++++++++++++++----
 packages/server/src/integrations/base/sql.ts  | 50 +++++++++++------
 2 files changed, 78 insertions(+), 25 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 3886822c21..0321cdf49e 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -22,12 +22,12 @@ import tk from "timekeeper"
 import { encodeJSBinding } from "@budibase/string-templates"
 
 describe.each([
-  //["lucene", undefined],
+  ["lucene", undefined],
   ["sqs", undefined],
-  //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
 ])("/api/:sourceId/search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
   const isLucene = name === "lucene"
@@ -1288,6 +1288,7 @@ describe.each([
       await createRows([
         { user: JSON.stringify(user1) },
         { user: JSON.stringify(user2) },
+        { user: null },
       ])
     })
 
@@ -1305,12 +1306,14 @@ describe.each([
       it("successfully finds a row", () =>
         expectQuery({ notEqual: { user: user1._id } }).toContainExactly([
           { user: { _id: user2._id } },
+          {},
         ]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([
           { user: { _id: user1._id } },
           { user: { _id: user2._id } },
+          { user: {} },
         ]))
     })
 
@@ -1323,6 +1326,19 @@ describe.each([
       it("fails to find nonexistent row", () =>
         expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing())
     })
+
+    describe("empty", () => {
+      it("finds empty rows", () =>
+        expectQuery({ empty: { user: null } }).toContainExactly([{}]))
+    })
+
+    describe("notEmpty", () => {
+      it("finds non-empty rows", () =>
+        expectQuery({ notEmpty: { user: null } }).toContainExactly([
+          { user: { _id: user1._id } },
+          { user: { _id: user2._id } },
+        ]))
+    })
   })
 
   describe("multi user", () => {
@@ -1338,14 +1354,19 @@ describe.each([
           name: "users",
           type: FieldType.BB_REFERENCE,
           subtype: BBReferenceFieldSubType.USER,
+          constraints: { type: "array" },
+        },
+        number: {
+          name: "number",
+          type: FieldType.NUMBER,
         },
       })
 
       await createRows([
-        { users: JSON.stringify([user1]) },
-        { users: JSON.stringify([user2]) },
-        { users: JSON.stringify([user1, user2]) },
-        { users: JSON.stringify([]) },
+        { number: 1, users: JSON.stringify([user1]) },
+        { number: 2, users: JSON.stringify([user2]) },
+        { number: 3, users: JSON.stringify([user1, user2]) },
+        { number: 4, users: JSON.stringify([]) },
       ])
     })
 
@@ -1389,5 +1410,19 @@ describe.each([
       it("fails to find nonexistent row", () =>
         expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing())
     })
+
+    describe("multi-column equals", () => {
+      it("successfully finds a row", () =>
+        expectQuery({
+          equal: { number: 1 },
+          contains: { users: [user1._id] },
+        }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({
+          equal: { number: 2 },
+          contains: { users: [user1._id] },
+        }).toFindNothing())
+    })
   })
 })
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 1c0c252b1c..c3292cf424 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -226,8 +226,7 @@ class InternalBuilder {
     }
 
     const contains = (mode: object, any: boolean = false) => {
-      const fnc = allOr ? "orWhere" : "where"
-      const rawFnc = `${fnc}Raw`
+      const rawFnc = allOr ? "orWhereRaw" : "whereRaw"
       const not = mode === filters?.notContains ? "NOT " : ""
       function stringifyArray(value: Array<any>, quoteStyle = '"'): string {
         for (let i in value) {
@@ -240,24 +239,24 @@ class InternalBuilder {
       if (this.client === SqlClient.POSTGRES) {
         iterate(mode, (key: string, value: Array<any>) => {
           const wrap = any ? "" : "'"
-          const containsOp = any ? "\\?| array" : "@>"
+          const op = any ? "\\?| array" : "@>"
           const fieldNames = key.split(/\./g)
-          const tableName = fieldNames[0]
-          const columnName = fieldNames[1]
-          // @ts-ignore
+          const table = fieldNames[0]
+          const col = fieldNames[1]
           query = query[rawFnc](
-            `${not}"${tableName}"."${columnName}"::jsonb ${containsOp} ${wrap}${stringifyArray(
+            `${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray(
               value,
               any ? "'" : '"'
-            )}${wrap}`
+            )}${wrap}, FALSE)`
           )
         })
       } else if (this.client === SqlClient.MY_SQL) {
         const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
         iterate(mode, (key: string, value: Array<any>) => {
-          // @ts-ignore
           query = query[rawFnc](
-            `${not}${jsonFnc}(${key}, '${stringifyArray(value)}')`
+            `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray(
+              value
+            )}'), FALSE)`
           )
         })
       } else {
@@ -272,8 +271,7 @@ class InternalBuilder {
             }
             statement +=
               (statement ? andOr : "") +
-              // `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)`
-              `LOWER(${likeKey(this.client, key)}) LIKE ?`
+              `COALESCE(LOWER(${likeKey(this.client, key)}), '') LIKE ?`
           }
 
           if (statement === "") {
@@ -338,14 +336,34 @@ class InternalBuilder {
     }
     if (filters.equal) {
       iterate(filters.equal, (key, value) => {
-        const fnc = allOr ? "orWhere" : "where"
-        query = query[fnc]({ [key]: value })
+        const fnc = allOr ? "orWhereRaw" : "whereRaw"
+        if (this.client === SqlClient.MS_SQL) {
+          query = query[fnc](
+            `CASE WHEN ${likeKey(this.client, key)} = ? THEN 1 ELSE 0 END = 1`,
+            [value]
+          )
+        } else {
+          query = query[fnc](
+            `COALESCE(${likeKey(this.client, key)} = ?, FALSE)`,
+            [value]
+          )
+        }
       })
     }
     if (filters.notEqual) {
       iterate(filters.notEqual, (key, value) => {
-        const fnc = allOr ? "orWhereNot" : "whereNot"
-        query = query[fnc]({ [key]: value })
+        const fnc = allOr ? "orWhereRaw" : "whereRaw"
+        if (this.client === SqlClient.MS_SQL) {
+          query = query[fnc](
+            `CASE WHEN ${likeKey(this.client, key)} = ? THEN 1 ELSE 0 END = 0`,
+            [value]
+          )
+        } else {
+          query = query[fnc](
+            `COALESCE(${likeKey(this.client, key)} != ?, TRUE)`,
+            [value]
+          )
+        }
       })
     }
     if (filters.empty) {

From 2131cc689cd01eaaaa5ae7eb3d76de5bd1bf0fe5 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 20 May 2024 17:13:12 +0100
Subject: [PATCH 32/92] Put pro back in line with master.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index ff397e5454..d3c3077011 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ff397e5454ad3361b25efdf14746c36dcbd3f409
+Subproject commit d3c3077011a8e20ed3c48dcd6301caca4120b6ac

From 2b1df81649dde09fd9d8974720818133e9a669c7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 11:24:04 +0200
Subject: [PATCH 33/92] Fix null references

---
 packages/server/src/integrations/base/sql.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 59efe9a9ba..25f2d13a35 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -133,7 +133,7 @@ function generateSelectStatement(
     }
     if (
       knex.client.config.client === SqlClient.MS_SQL &&
-      columnSchema.type === FieldType.DATETIME &&
+      columnSchema?.type === FieldType.DATETIME &&
       columnSchema.timeOnly
     ) {
       return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)

From d97f3b03781e796c2eff7288de823bb6bea9baa4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 11:54:42 +0200
Subject: [PATCH 34/92] Handle undefineds

---
 packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
index 6f1490a573..14f3559e9a 100644
--- a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
+++ b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
@@ -9,6 +9,11 @@
   $: displayValue = value?.format("HH:mm")
 
   const handleChange = e => {
+    if (!e.target.value) {
+      dispatch("change", undefined)
+      return
+    }
+
     const [hour, minute] = e.target.value.split(":").map(x => parseInt(x))
     dispatch("change", value.hour(hour).minute(minute))
   }

From fc99fad3d2e85a0090724150ea9cb541ddd65bcb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 11:54:55 +0200
Subject: [PATCH 35/92] Fix display in chromium

---
 packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte | 3 +++
 packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte  | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte b/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte
index 7e013341ac..6c06ce4e79 100644
--- a/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte
+++ b/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte
@@ -52,4 +52,7 @@
   input.hide-arrows {
     -moz-appearance: textfield;
   }
+  input[type="time"]::-webkit-calendar-picker-indicator {
+    display: none;
+  }
 </style>
diff --git a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
index 14f3559e9a..e1ea4d625b 100644
--- a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
+++ b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
@@ -24,7 +24,6 @@
     hideArrows
     type={"time"}
     value={displayValue}
-    width={60}
     on:input={handleChange}
     on:change={handleChange}
   />

From 221c8a3f0a7734bd53446855a23e0226cc5eb666 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 13:31:15 +0200
Subject: [PATCH 36/92] Shuffle all test createRows

---
 .../server/src/api/routes/tests/search.spec.ts     | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 2a5890decf..62d55e7e32 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -78,7 +78,7 @@ describe.each([
   }
 
   async function createRows(rows: Record<string, any>[]) {
-    await config.api.row.bulkImport(table._id!, { rows })
+    await config.api.row.bulkImport(table._id!, { rows: _.shuffle(rows) })
   }
 
   class SearchAssertion {
@@ -981,11 +981,13 @@ describe.each([
           time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
         })
 
-        await createRows(
-          _.shuffle([T_1000, T_1045, T_1200, T_1530, T_0000]).map(time => ({
-            time,
-          }))
-        )
+        await createRows([
+          { time: T_1000 },
+          { time: T_1045 },
+          { time: T_1200 },
+          { time: T_1530 },
+          { time: T_0000 },
+        ])
       })
 
       describe("equal", () => {

From 7c7f88bd47a3b9b776a3efcef40ad946e54fb8a8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 13:41:46 +0200
Subject: [PATCH 37/92] Pop on asserts

---
 .../server/src/api/routes/tests/search.spec.ts  | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 62d55e7e32..74eca789cb 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -84,7 +84,7 @@ describe.each([
   class SearchAssertion {
     constructor(private readonly query: RowSearchParams) {}
 
-    private findRow(expectedRow: any, foundRows: any[]) {
+    private popRow(expectedRow: any, foundRows: any[]) {
       const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
       if (!row) {
         const fields = Object.keys(expectedRow)
@@ -97,6 +97,9 @@ describe.each([
           )} in ${JSON.stringify(searchedObjects)}`
         )
       }
+
+      // Ensuring the same row is not matched twice
+      foundRows.splice(foundRows.indexOf(row), 1)
       return row
     }
 
@@ -113,9 +116,9 @@ describe.each([
       // eslint-disable-next-line jest/no-standalone-expect
       expect(foundRows).toHaveLength(expectedRows.length)
       // eslint-disable-next-line jest/no-standalone-expect
-      expect(foundRows).toEqual(
+      expect([...foundRows]).toEqual(
         expectedRows.map((expectedRow: any) =>
-          expect.objectContaining(this.findRow(expectedRow, foundRows))
+          expect.objectContaining(this.popRow(expectedRow, foundRows))
         )
       )
     }
@@ -132,10 +135,10 @@ describe.each([
       // eslint-disable-next-line jest/no-standalone-expect
       expect(foundRows).toHaveLength(expectedRows.length)
       // eslint-disable-next-line jest/no-standalone-expect
-      expect(foundRows).toEqual(
+      expect([...foundRows]).toEqual(
         expect.arrayContaining(
           expectedRows.map((expectedRow: any) =>
-            expect.objectContaining(this.findRow(expectedRow, foundRows))
+            expect.objectContaining(this.popRow(expectedRow, foundRows))
           )
         )
       )
@@ -151,10 +154,10 @@ describe.each([
       })
 
       // eslint-disable-next-line jest/no-standalone-expect
-      expect(foundRows).toEqual(
+      expect([...foundRows]).toEqual(
         expect.arrayContaining(
           expectedRows.map((expectedRow: any) =>
-            expect.objectContaining(this.findRow(expectedRow, foundRows))
+            expect.objectContaining(this.popRow(expectedRow, foundRows))
           )
         )
       )

From 6eefa1afe81d4ebd503dffdd25d00909f3cb7aa6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 14:30:03 +0200
Subject: [PATCH 38/92] Add comment

---
 packages/server/src/api/routes/tests/search.spec.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 74eca789cb..00864e7454 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -78,6 +78,7 @@ describe.each([
   }
 
   async function createRows(rows: Record<string, any>[]) {
+    // Shuffling to avoid false positives given a fixed order
     await config.api.row.bulkImport(table._id!, { rows: _.shuffle(rows) })
   }
 

From 8cf021f254f8144b0ffa14624871ecd3fb3c4505 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 14:30:37 +0200
Subject: [PATCH 39/92] Add null row

---
 packages/server/src/api/routes/tests/search.spec.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 00864e7454..15220a9c1a 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -980,12 +980,16 @@ describe.each([
 
       const UNEXISTING_TIME = "10:01"
 
+      const NULL_TIME__ID = `null_time__id`
+
       beforeAll(async () => {
         await createTable({
+          timeid: { name: "timeid", type: FieldType.STRING },
           time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
         })
 
         await createRows([
+          { timeid: NULL_TIME__ID, time: null },
           { time: T_1000 },
           { time: T_1045 },
           { time: T_1200 },

From 9c9b2ff48da936deea204681142bfc4e64dc386c Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 21 May 2024 14:15:17 +0100
Subject: [PATCH 40/92] Move empty object range tests out of sql.spec.ts.

---
 .../src/api/routes/tests/search.spec.ts       | 14 +++++++
 .../server/src/integrations/tests/sql.spec.ts | 38 -------------------
 2 files changed, 14 insertions(+), 38 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 2ddc6ef1ff..5b5f05bc58 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -712,6 +712,20 @@ describe.each([
         expectQuery({
           range: { name: { low: "g", high: "h" } },
         }).toFindNothing())
+
+      !isLucene &&
+        it("ignores low if it's an empty object", () =>
+          expectQuery({
+            // @ts-ignore
+            range: { name: { low: {}, high: "z" } },
+          }).toContainExactly([{ name: "foo" }, { name: "bar" }]))
+
+      !isLucene &&
+        it("ignores high if it's an empty object", () =>
+          expectQuery({
+            // @ts-ignore
+            range: { name: { low: "a", high: {} } },
+          }).toContainExactly([{ name: "foo" }, { name: "bar" }]))
     })
 
     describe("empty", () => {
diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index cf433725ce..377c1d65e0 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -190,44 +190,6 @@ describe("SQL query builder", () => {
     })
   })
 
-  it("should ignore high range value if it is an empty object", () => {
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          range: {
-            dob: {
-              low: "2000-01-01 00:00:00",
-              high: {},
-            },
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: ["2000-01-01 00:00:00", 500],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" >= $1 limit $2) as "${TABLE_NAME}"`,
-    })
-  })
-
-  it("should ignore low range value if it is an empty object", () => {
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          range: {
-            dob: {
-              low: {},
-              high: "2010-01-01 00:00:00",
-            },
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: ["2010-01-01 00:00:00", 500],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" <= $1 limit $2) as "${TABLE_NAME}"`,
-    })
-  })
-
   it("should lowercase the values for Oracle LIKE statements", () => {
     let query = new Sql(SqlClient.ORACLE, limit)._query(
       generateReadJson({

From a074cb6befe600bd56bf6c1ce9cd0b0261f818cb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 21 May 2024 14:17:35 +0100
Subject: [PATCH 41/92] Remove enrichEmptySettings from component patch
 function as any screen update already invokes empty setting enrichment on
 every single component

---
 packages/builder/src/stores/builder/components.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 1311c26d5d..c281c73dfe 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -555,7 +555,6 @@ export class ComponentStore extends BudiStore {
       const patchResult = patchFn(component, screen)
 
       // Post processing
-      this.enrichEmptySettings(component, { screen, useDefaultValues: false })
       const migrated = this.migrateSettings(component)
 
       // Returning an explicit false signifies that we should skip this

From bc63a119798d03e92e2c39e072483b6f36c9976a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 21 May 2024 14:20:05 +0100
Subject: [PATCH 42/92] Move sort stability check to search.spec.ts.

---
 .../src/api/routes/tests/search.spec.ts       | 19 +++++++++++++++++++
 .../server/src/integrations/tests/sql.spec.ts | 15 ---------------
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 5b5f05bc58..876b52c0a6 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1309,6 +1309,25 @@ describe.each([
               { auto: 2 },
               { auto: 1 },
             ]))
+
+          // This is important for pagination. The order of results must always
+          // be stable or pagination will break. We don't want the user to need
+          // to specify an order for pagination to work.
+          it("is stable without a sort specified", async () => {
+            let { rows } = await config.api.row.search(table._id!, {
+              tableId: table._id!,
+              query: {},
+            })
+
+            for (let i = 0; i < 10; i++) {
+              const response = await config.api.row.search(table._id!, {
+                tableId: table._id!,
+                limit: 1,
+                query: {},
+              })
+              expect(response.rows).toEqual(rows)
+            }
+          })
         })
 
       // TODO(samwho): fix for SQS
diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index 377c1d65e0..4fc964b320 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -235,21 +235,6 @@ describe("SQL query builder", () => {
     })
   })
 
-  it("should sort SQL Server tables by the primary key if no sort data is provided", () => {
-    let query = new Sql(SqlClient.MS_SQL, limit)._query(
-      generateReadJson({
-        sort: {},
-        paginate: {
-          limit: 10,
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [10],
-      sql: `select * from (select top (@p0) * from [test] order by [test].[id] asc) as [test]`,
-    })
-  })
-
   it("should not parse JSON string as Date", () => {
     let query = new Sql(SqlClient.POSTGRES, limit)._query(
       generateCreateJson(TABLE_NAME, {

From 53605ec8ed68c06c5baab65e98706e2642bec2ef Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 16:08:22 +0200
Subject: [PATCH 43/92] Fix sorting in pg

---
 packages/server/src/integrations/base/sql.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 25f2d13a35..1d0d909829 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -387,7 +387,13 @@ class InternalBuilder {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
           value.direction === SortDirection.ASCENDING ? "asc" : "desc"
-        query = query.orderBy(`${aliased}.${key}`, direction)
+        let nulls
+        if (this.client === SqlClient.POSTGRES) {
+          // All other clients already sort this as expected by default, and adding this to the rest of the clients is causing issues
+          nulls = value.direction === SortDirection.ASCENDING ? "first" : "last"
+        }
+
+        query = query.orderBy(`${aliased}.${key}`, direction, nulls)
       }
     } else if (this.client === SqlClient.MS_SQL && paginate?.limit) {
       // @ts-ignore

From a55f975489abebad2c020c497aac32f1f85f862b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 16:11:36 +0200
Subject: [PATCH 44/92] Fix tests

---
 packages/server/src/api/routes/tests/search.spec.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 15220a9c1a..709f8a7597 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1067,6 +1067,7 @@ describe.each([
             sort: "time",
             sortOrder: SortOrder.ASCENDING,
           }).toMatchExactly([
+            { timeid: NULL_TIME__ID },
             { time: "00:00:00" },
             { time: "10:00:00" },
             { time: "10:45:00" },
@@ -1085,6 +1086,7 @@ describe.each([
             { time: "10:45:00" },
             { time: "10:00:00" },
             { time: "00:00:00" },
+            { timeid: NULL_TIME__ID },
           ]))
 
         describe("sortType STRING", () => {
@@ -1095,6 +1097,7 @@ describe.each([
               sortType: SortType.STRING,
               sortOrder: SortOrder.ASCENDING,
             }).toMatchExactly([
+              { timeid: NULL_TIME__ID },
               { time: "00:00:00" },
               { time: "10:00:00" },
               { time: "10:45:00" },
@@ -1114,6 +1117,7 @@ describe.each([
               { time: "10:45:00" },
               { time: "10:00:00" },
               { time: "00:00:00" },
+              { timeid: NULL_TIME__ID },
             ]))
         })
       })

From 4bf0a43c64ed73d9d213d24aa91ed9c6be2adbd5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 16:35:11 +0200
Subject: [PATCH 45/92] Add comment

---
 packages/server/src/integrations/base/sql.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 1d0d909829..c3dca2a39c 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -136,6 +136,7 @@ function generateSelectStatement(
       columnSchema?.type === FieldType.DATETIME &&
       columnSchema.timeOnly
     ) {
+      // Time gets returned as timestamp from mssql, not matching the expected HH:mm format
       return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
     }
     return `${field} as ${field}`

From 7bbb920aa7fc17ccd91d8c0c8da555042b4be934 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 21 May 2024 15:49:48 +0100
Subject: [PATCH 46/92] Optimise the `reverse` method in sqlAlias.ts.

---
 packages/server/src/sdk/app/rows/sqlAlias.ts | 23 ++++++++++++++------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts
index 0fc338ecbe..79d1ff485d 100644
--- a/packages/server/src/sdk/app/rows/sqlAlias.ts
+++ b/packages/server/src/sdk/app/rows/sqlAlias.ts
@@ -126,16 +126,25 @@ export default class AliasTables {
   }
 
   reverse<T extends Row | Row[]>(rows: T): T {
+    const mapping = new Map()
+
     const process = (row: Row) => {
       const final: Row = {}
-      for (let [key, value] of Object.entries(row)) {
-        if (!key.includes(".")) {
-          final[key] = value
-        } else {
-          const [alias, column] = key.split(".")
-          const tableName = this.tableAliases[alias] || alias
-          final[`${tableName}.${column}`] = value
+      for (const key of Object.keys(row)) {
+        let mappedKey = mapping.get(key)
+        if (!mappedKey) {
+          const dotLocation = key.indexOf(".")
+          if (dotLocation === -1) {
+            mappedKey = key
+          } else {
+            const alias = key.slice(0, dotLocation)
+            const column = key.slice(dotLocation + 1)
+            const tableName = this.tableAliases[alias] || alias
+            mappedKey = `${tableName}.${column}`
+          }
+          mapping.set(key, mappedKey)
         }
+        final[mappedKey] = row[key]
       }
       return final
     }

From d93a9e2c4f6d7cb531a6dfd84582d1dc776ef682 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 16:52:40 +0200
Subject: [PATCH 47/92] Fix value from null

---
 packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
index e1ea4d625b..4f070bdcfb 100644
--- a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
+++ b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte
@@ -1,4 +1,5 @@
 <script>
+  import dayjs from "dayjs"
   import NumberInput from "./NumberInput.svelte"
   import { createEventDispatcher } from "svelte"
 
@@ -15,7 +16,7 @@
     }
 
     const [hour, minute] = e.target.value.split(":").map(x => parseInt(x))
-    dispatch("change", value.hour(hour).minute(minute))
+    dispatch("change", (value || dayjs()).hour(hour).minute(minute))
   }
 </script>
 

From 229af2e695cc3decd037f1980a05d3a3c5a15f22 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 21 May 2024 16:12:16 +0000
Subject: [PATCH 48/92] Bump version to 2.27.0

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index cba15492eb..d4e12438ee 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.26.4",
+  "version": "2.27.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From 35c52203ce4b432fc811212f55f4043cdf0c35cc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 21 May 2024 17:29:32 +0100
Subject: [PATCH 49/92] Add tests for enriching empty card block settings

---
 .../stores/builder/tests/component.test.js    | 23 +++++++
 .../stores/builder/tests/fixtures/index.js    | 60 ++++++++++++++++++-
 2 files changed, 81 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/stores/builder/tests/component.test.js b/packages/builder/src/stores/builder/tests/component.test.js
index b8baefc5e6..80a0c8077d 100644
--- a/packages/builder/src/stores/builder/tests/component.test.js
+++ b/packages/builder/src/stores/builder/tests/component.test.js
@@ -23,6 +23,7 @@ import {
   DB_TYPE_EXTERNAL,
   DEFAULT_BB_DATASOURCE_ID,
 } from "constants/backend"
+import { makePropSafe as safe } from "@budibase/string-templates"
 
 // Could move to fixtures
 const COMP_PREFIX = "@budibase/standard-components"
@@ -360,8 +361,30 @@ describe("Component store", () => {
       resourceId: internalTableDoc._id,
       type: "table",
     })
+
+    return comp
   }
 
+  it("enrichEmptySettings - initialise cards blocks with correct fields", async ctx => {
+    const comp = enrichSettingsDS("cardsblock", ctx)
+    const expectBinding = (setting, ...parts) => {
+      expect(comp[setting]).toStrictEqual(
+        `{{ ${safe(`${comp._id}-repeater`)}.${parts.map(safe).join(".")} }}`
+      )
+    }
+    expectBinding("cardTitle", internalTableDoc.schema.MediaTitle.name)
+    expectBinding("cardSubtitle", internalTableDoc.schema.MediaVersion.name)
+    expectBinding(
+      "cardDescription",
+      internalTableDoc.schema.MediaDescription.name
+    )
+    expectBinding(
+      "cardImageURL",
+      internalTableDoc.schema.MediaImage.name,
+      "url"
+    )
+  })
+
   it("enrichEmptySettings - set default datasource for 'table' setting type", async ctx => {
     enrichSettingsDS("formblock", ctx)
   })
diff --git a/packages/builder/src/stores/builder/tests/fixtures/index.js b/packages/builder/src/stores/builder/tests/fixtures/index.js
index f636790f53..fbad17e374 100644
--- a/packages/builder/src/stores/builder/tests/fixtures/index.js
+++ b/packages/builder/src/stores/builder/tests/fixtures/index.js
@@ -8,6 +8,7 @@ import {
   DB_TYPE_EXTERNAL,
   DEFAULT_BB_DATASOURCE_ID,
 } from "constants/backend"
+import { FieldType } from "@budibase/types"
 
 const getDocId = () => {
   return v4().replace(/-/g, "")
@@ -45,6 +46,52 @@ export const COMPONENT_DEFINITIONS = {
       },
     ],
   },
+  cardsblock: {
+    block: true,
+    name: "Cards Block",
+    settings: [
+      {
+        type: "dataSource",
+        label: "Data",
+        key: "dataSource",
+        required: true,
+      },
+      {
+        section: true,
+        name: "Cards",
+        settings: [
+          {
+            type: "text",
+            key: "cardTitle",
+            label: "Title",
+            nested: true,
+            resetOn: "dataSource",
+          },
+          {
+            type: "text",
+            key: "cardSubtitle",
+            label: "Subtitle",
+            nested: true,
+            resetOn: "dataSource",
+          },
+          {
+            type: "text",
+            key: "cardDescription",
+            label: "Description",
+            nested: true,
+            resetOn: "dataSource",
+          },
+          {
+            type: "text",
+            key: "cardImageURL",
+            label: "Image URL",
+            nested: true,
+            resetOn: "dataSource",
+          },
+        ],
+      },
+    ],
+  },
   container: {
     name: "Container",
   },
@@ -262,14 +309,23 @@ export const internalTableDoc = {
   name: "Media",
   sourceId: BUDIBASE_INTERNAL_DB_ID,
   sourceType: DB_TYPE_INTERNAL,
+  primaryDisplay: "MediaTitle",
   schema: {
     MediaTitle: {
       name: "MediaTitle",
-      type: "string",
+      type: FieldType.STRING,
     },
     MediaVersion: {
       name: "MediaVersion",
-      type: "string",
+      type: FieldType.STRING,
+    },
+    MediaDescription: {
+      name: "MediaDescription",
+      type: FieldType.LONGFORM,
+    },
+    MediaImage: {
+      name: "MediaImage",
+      type: FieldType.ATTACHMENT_SINGLE,
     },
   },
 }

From 7f7ed9f0cb25353c9dd28109b364af42525dc12b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 21 May 2024 17:38:38 +0100
Subject: [PATCH 50/92] Move more tests out of sql.spec.ts.

---
 .../server/src/api/routes/tests/row.spec.ts   | 19 +++++
 .../src/api/routes/tests/search.spec.ts       | 69 ++++++++++++++++---
 .../server/src/integrations/tests/sql.spec.ts | 53 --------------
 .../server/src/tests/utilities/structures.ts  |  2 +-
 .../src/utilities/rowProcessor/index.ts       |  6 ++
 5 files changed, 85 insertions(+), 64 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 934a838e6a..4801ac4c55 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -485,6 +485,25 @@ describe.each([
         )
         expect(response.message).toBe("Cannot create new user entry.")
       })
+
+    it("should not mis-parse date string out of JSON", async () => {
+      const table = await config.api.table.save(
+        saveTableRequest({
+          schema: {
+            name: {
+              type: FieldType.STRING,
+              name: "name",
+            },
+          },
+        })
+      )
+
+      const row = await config.api.row.save(table._id!, {
+        name: `{ "foo": "2023-01-26T11:48:57.000Z" }`,
+      })
+
+      expect(row.name).toEqual(`{ "foo": "2023-01-26T11:48:57.000Z" }`)
+    })
   })
 
   describe("get", () => {
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 876b52c0a6..d65980a7cb 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -17,6 +17,7 @@ import {
   TableSchema,
   User,
   Row,
+  RelationshipType,
 } from "@budibase/types"
 import _ from "lodash"
 import tk from "timekeeper"
@@ -73,7 +74,7 @@ describe.each([
   })
 
   async function createTable(schema: TableSchema) {
-    table = await config.api.table.save(
+    return await config.api.table.save(
       tableForDatasource(datasource, { schema })
     )
   }
@@ -186,7 +187,7 @@ describe.each([
 
   describe("boolean", () => {
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
       })
       await createRows([{ isTrue: true }, { isTrue: false }])
@@ -316,7 +317,7 @@ describe.each([
           })
       )
 
-      await createTable({
+      table = await createTable({
         name: { name: "name", type: FieldType.STRING },
         appointment: { name: "appointment", type: FieldType.DATETIME },
         single_user: {
@@ -592,7 +593,7 @@ describe.each([
 
   describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         name: { name: "name", type: FieldType.STRING },
       })
       await createRows([{ name: "foo" }, { name: "bar" }])
@@ -790,7 +791,7 @@ describe.each([
 
   describe("numbers", () => {
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         age: { name: "age", type: FieldType.NUMBER },
       })
       await createRows([{ age: 1 }, { age: 10 }])
@@ -899,7 +900,7 @@ describe.each([
     const JAN_10TH = "2020-01-10T00:00:00.000Z"
 
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         dob: { name: "dob", type: FieldType.DATETIME },
       })
 
@@ -1011,7 +1012,7 @@ describe.each([
 
   describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         numbers: {
           name: "numbers",
           type: FieldType.ARRAY,
@@ -1091,7 +1092,7 @@ describe.each([
     const BIG = "9223372036854775807"
 
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         num: { name: "num", type: FieldType.BIGINT },
       })
       await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
@@ -1182,7 +1183,7 @@ describe.each([
   isInternal &&
     describe("auto", () => {
       beforeAll(async () => {
-        await createTable({
+        table = await createTable({
           auto: {
             name: "auto",
             type: FieldType.AUTO,
@@ -1366,7 +1367,7 @@ describe.each([
 
   describe("field name 1:name", () => {
     beforeAll(async () => {
-      await createTable({
+      table = await createTable({
         "1:name": { name: "1:name", type: FieldType.STRING },
       })
       await createRows([{ "1:name": "bar" }, { "1:name": "foo" }])
@@ -1382,4 +1383,52 @@ describe.each([
         expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing())
     })
   })
+
+  // This will never work for Lucene.
+  // TODO(samwho): fix for SQS
+  !isInternal &&
+    describe("relations", () => {
+      let otherTable: Table
+      let rows: Row[]
+
+      beforeAll(async () => {
+        otherTable = await createTable({
+          one: { name: "one", type: FieldType.STRING },
+        })
+        table = await createTable({
+          two: { name: "two", type: FieldType.STRING },
+          other: {
+            type: FieldType.LINK,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            name: "other",
+            fieldName: "other",
+            tableId: otherTable._id!,
+            constraints: {
+              type: "array",
+            },
+          },
+        })
+
+        rows = await Promise.all([
+          config.api.row.save(otherTable._id!, { one: "foo" }),
+          config.api.row.save(otherTable._id!, { one: "bar" }),
+        ])
+
+        await Promise.all([
+          config.api.row.save(table._id!, {
+            two: "foo",
+            other: [rows[0]._id],
+          }),
+          config.api.row.save(table._id!, {
+            two: "bar",
+            other: [rows[1]._id],
+          }),
+        ])
+      })
+
+      it("can search through relations", () =>
+        expectQuery({
+          equal: { [`${otherTable.name}.one`]: "foo" },
+        }).toContainExactly([{ two: "foo", other: [{ _id: rows[0]._id }] }]))
+    })
 })
diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index 4fc964b320..9b84409e92 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -56,16 +56,6 @@ function generateReadJson({
   }
 }
 
-function generateCreateJson(table = TABLE_NAME, body = {}): QueryJson {
-  return {
-    endpoint: endpoint(table, "CREATE"),
-    meta: {
-      table: TABLE,
-    },
-    body,
-  }
-}
-
 function generateRelationshipJson(config: { schema?: string } = {}): QueryJson {
   return {
     endpoint: {
@@ -146,24 +136,6 @@ describe("SQL query builder", () => {
     sql = new Sql(client, limit)
   })
 
-  it("should allow filtering on a related field", () => {
-    const query = sql._query(
-      generateReadJson({
-        filters: {
-          equal: {
-            age: 10,
-            "task.name": "task 1",
-          },
-        },
-      })
-    )
-    // order of bindings changes because relationship filters occur outside inner query
-    expect(query).toEqual({
-      bindings: [10, limit, "task 1"],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" = $1 limit $2) as "${TABLE_NAME}" where "task"."name" = $3`,
-    })
-  })
-
   it("should add the schema to the LEFT JOIN", () => {
     const query = sql._query(generateRelationshipJson({ schema: "production" }))
     expect(query).toEqual({
@@ -234,29 +206,4 @@ describe("SQL query builder", () => {
       sql: `select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1) where rownum <= :2) "test"`,
     })
   })
-
-  it("should not parse JSON string as Date", () => {
-    let query = new Sql(SqlClient.POSTGRES, limit)._query(
-      generateCreateJson(TABLE_NAME, {
-        name: '{ "created_at":"2023-09-09T03:21:06.024Z" }',
-      })
-    )
-    expect(query).toEqual({
-      bindings: ['{ "created_at":"2023-09-09T03:21:06.024Z" }'],
-      sql: `insert into "test" ("name") values ($1) returning *`,
-    })
-  })
-
-  it("should parse and trim valid string as Date", () => {
-    const dateObj = new Date("2023-09-09T03:21:06.024Z")
-    let query = new Sql(SqlClient.POSTGRES, limit)._query(
-      generateCreateJson(TABLE_NAME, {
-        name: " 2023-09-09T03:21:06.024Z ",
-      })
-    )
-    expect(query).toEqual({
-      bindings: [dateObj],
-      sql: `insert into "test" ("name") values ($1) returning *`,
-    })
-  })
 })
diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts
index 77a6431335..7213cc66f1 100644
--- a/packages/server/src/tests/utilities/structures.ts
+++ b/packages/server/src/tests/utilities/structures.ts
@@ -37,7 +37,7 @@ export function tableForDatasource(
 ): Table {
   return merge(
     {
-      name: generator.guid(),
+      name: generator.guid().substring(0, 10),
       type: "table",
       sourceType: datasource
         ? TableSourceType.EXTERNAL
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 73176af6d8..4d91759be5 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -150,6 +150,12 @@ export async function inputProcessing(
       clonedRow[key] = coerce(value, field.type)
     }
 
+    if (field.type === FieldType.DATETIME) {
+      if (typeof clonedRow[key] === "string") {
+        clonedRow[key] = clonedRow[key].trim()
+      }
+    }
+
     // remove any attachment urls, they are generated on read
     if (field.type === FieldType.ATTACHMENTS) {
       const attachments = clonedRow[key]

From 0561ca6e1b367729921328af5c6e4a8ad20c4f41 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 21 May 2024 17:39:33 +0100
Subject: [PATCH 51/92] Remove date leniency.

---
 packages/server/src/utilities/rowProcessor/index.ts | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 4d91759be5..73176af6d8 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -150,12 +150,6 @@ export async function inputProcessing(
       clonedRow[key] = coerce(value, field.type)
     }
 
-    if (field.type === FieldType.DATETIME) {
-      if (typeof clonedRow[key] === "string") {
-        clonedRow[key] = clonedRow[key].trim()
-      }
-    }
-
     // remove any attachment urls, they are generated on read
     if (field.type === FieldType.ATTACHMENTS) {
       const attachments = clonedRow[key]

From a5c5e2ffcdfcd9ecc13f15e0df1ef5b986ed662e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 21 May 2024 18:39:46 +0100
Subject: [PATCH 52/92] Fixing issue discovered by test with 1: syntax being
 required for relationship based filters.

---
 packages/server/src/sdk/app/rows/search/sqs.ts | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index c4dc408cac..ec3ac08c2e 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -55,8 +55,8 @@ function buildInternalFieldList(
   return fieldList
 }
 
-function tableInFilter(name: string) {
-  return `:${name}.`
+function tableNameInFieldRegex(tableName: string) {
+  return new RegExp(`^${tableName}.|:${tableName}.`, "g")
 }
 
 function cleanupFilters(filters: SearchFilters, tables: Table[]) {
@@ -72,15 +72,13 @@ function cleanupFilters(filters: SearchFilters, tables: Table[]) {
       // relationship, switch to table ID
       const tableRelated = tables.find(
         table =>
-          table.originalName && key.includes(tableInFilter(table.originalName))
+          table.originalName &&
+          key.match(tableNameInFieldRegex(table.originalName))
       )
       if (tableRelated && tableRelated.originalName) {
-        filter[
-          key.replace(
-            tableInFilter(tableRelated.originalName),
-            tableInFilter(tableRelated._id!)
-          )
-        ] = filter[key]
+        // only replace the first, not replaceAll
+        filter[key.replace(tableRelated.originalName, tableRelated._id!)] =
+          filter[key]
         delete filter[key]
       }
     }

From 9efffdf9d6929ab7a81d2f474e8506bd4b5110dd Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 21 May 2024 18:47:10 +0000
Subject: [PATCH 53/92] Bump version to 2.27.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index d4e12438ee..f0b3f51d47 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.27.0",
+  "version": "2.27.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From a920161e92e5aaf283733c6e17681d77bb952d64 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 21 May 2024 22:02:47 +0200
Subject: [PATCH 54/92] Fix tests

---
 .../src/integrations/tests/sqlAlias.spec.ts      | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index fda2a091fa..0de4d0a151 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -61,9 +61,9 @@ describe("Captures of real examples", () => {
               "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", 
               "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
-              from (select * from "persons" as "a" order by "a"."firstname" asc limit $1) as "a" 
+              from (select * from "persons" as "a" order by "a"."firstname" asc nulls first limit $1) as "a" 
               left join "tasks" as "b" on "a"."personid" = "b"."qaid" or "a"."personid" = "b"."executorid" 
-              order by "a"."firstname" asc limit $2`),
+              order by "a"."firstname" asc nulls first limit $2`),
       })
     })
 
@@ -75,10 +75,10 @@ describe("Captures of real examples", () => {
         sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", 
               "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
-              from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" 
+              from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" 
               left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
               left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 
-              order by "a"."productname" asc limit $3`),
+              order by "a"."productname" asc nulls first limit $3`),
       })
     })
 
@@ -90,10 +90,10 @@ describe("Captures of real examples", () => {
         sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", 
               "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
-              from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" 
+              from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" 
               left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
               left join "tasks" as "b" on "b"."taskid" = "c"."taskid" 
-              order by "a"."productname" asc limit $2`),
+              order by "a"."productname" asc nulls first limit $2`),
       })
     })
 
@@ -138,11 +138,11 @@ describe("Captures of real examples", () => {
              "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", 
              "c"."city" as "c.city", "c"."lastname" as "c.lastname" 
              from (select * from "tasks" as "a" where not "a"."completed" = $1 
-             order by "a"."taskname" asc limit $2) as "a" 
+             order by "a"."taskname" asc nulls first limit $2) as "a" 
              left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" 
              left join "products" as "b" on "b"."productid" = "d"."productid" 
              left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" 
-             where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc limit $6`),
+             where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc nulls first limit $6`),
       })
     })
   })

From c726dfbedbd27a4ae71c5f3f9472612315264fcb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 22 May 2024 09:22:06 +0100
Subject: [PATCH 55/92] Add link in navigation when duplicating screens

---
 .../_components/ScreenList/DropdownMenu.svelte         | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/DropdownMenu.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/DropdownMenu.svelte
index 0d96dcda93..7421bfa68a 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/DropdownMenu.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/DropdownMenu.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { screenStore, componentStore } from "stores/builder"
+  import { screenStore, componentStore, navigationStore } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import {
     ActionMenu,
@@ -12,6 +12,7 @@
   import ScreenDetailsModal from "components/design/ScreenDetailsModal.svelte"
   import sanitizeUrl from "helpers/sanitizeUrl"
   import { makeComponentUnique } from "helpers/components"
+  import { capitalise } from "helpers"
 
   export let screenId
 
@@ -48,6 +49,13 @@
     try {
       // Create the screen
       await screenStore.save(duplicateScreen)
+
+      // Add new screen to navigation
+      await navigationStore.saveLink(
+        duplicateScreen.routing.route,
+        capitalise(duplicateScreen.routing.route.split("/")[1]),
+        duplicateScreen.routing.roleId
+      )
     } catch (error) {
       notifications.error("Error duplicating screen")
     }

From c89a7a261fede85b746dc7ad3936cf813bc9d8e6 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 22 May 2024 09:35:27 +0100
Subject: [PATCH 56/92] Ensure in-app navigation via navigate to action
 automatically prefixes URLs with slashes

---
 packages/client/src/utils/buttonActions.js | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js
index 0c3866768e..25e0892fc0 100644
--- a/packages/client/src/utils/buttonActions.js
+++ b/packages/client/src/utils/buttonActions.js
@@ -238,7 +238,13 @@ const triggerAutomationHandler = async action => {
   }
 }
 const navigationHandler = action => {
-  const { url, peek, externalNewTab } = action.parameters
+  let { url, peek, externalNewTab, type } = action.parameters
+
+  // Ensure in-app navigation starts with a slash
+  if (type === "screen" && url && !url.startsWith("/")) {
+    url = `/${url}`
+  }
+
   routeStore.actions.navigate(url, peek, externalNewTab)
   closeSidePanelHandler()
 }

From 7a6863763a0a5b7903a04c076c27a364f2211dab Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 22 May 2024 09:53:44 +0100
Subject: [PATCH 57/92] Allow opening app nav links in new tabs

---
 packages/client/src/components/app/NavItem.svelte | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/packages/client/src/components/app/NavItem.svelte b/packages/client/src/components/app/NavItem.svelte
index fcdda57ace..a5ac61a39f 100644
--- a/packages/client/src/components/app/NavItem.svelte
+++ b/packages/client/src/components/app/NavItem.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { createEventDispatcher, getContext } from "svelte"
+  import { createEventDispatcher } from "svelte"
   import active from "svelte-spa-router/active"
   import { Icon } from "@budibase/bbui"
 
@@ -13,8 +13,6 @@
   export let navStateStore
 
   const dispatch = createEventDispatcher()
-  const sdk = getContext("sdk")
-  const { linkable } = sdk
 
   let renderKey
 
@@ -46,10 +44,9 @@
       styled
     -->
     <a
-      href={url}
+      href="#{url}"
       on:click={onClickLink}
       use:active={url}
-      use:linkable
       class:active={false}
     >
       {text}
@@ -73,10 +70,9 @@
           {#each subLinks || [] as subLink}
             {#if subLink.internalLink}
               <a
-                href={subLink.url}
+                href="#{subLink.url}"
                 on:click={onClickLink}
                 use:active={subLink.url}
-                use:linkable
               >
                 {subLink.text}
               </a>

From 1a47365d39529881e2a8f6646cf290c90ac03e6c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 22 May 2024 10:04:54 +0100
Subject: [PATCH 58/92] Don't save empty JS expressions as JS

---
 .../common/bindings/DrawerBindableInput.svelte         | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte
index d11ebcf87a..0792866581 100644
--- a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte
+++ b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte
@@ -4,10 +4,9 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "dataBinding"
-
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher, setContext } from "svelte"
-  import { isJSBinding } from "@budibase/string-templates"
+  import { isJSBinding, decodeJSBinding } from "@budibase/string-templates"
   import { builderStore } from "stores/builder"
 
   export let panel = ClientBindingPanel
@@ -35,7 +34,12 @@
   $: isJS = isJSBinding(value)
 
   const saveBinding = () => {
-    onChange(tempValue)
+    // Don't bother saving empty JS expressions as JS
+    let val = tempValue
+    if (decodeJSBinding(tempValue)?.trim() === "") {
+      val = null
+    }
+    onChange(val)
     onBlur()
     builderStore.propertyFocus()
     bindingDrawer.hide()

From 96aa83d4548d57a076830a7ad031862ed264af85 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 22 May 2024 10:15:09 +0100
Subject: [PATCH 59/92] Change how empty JS values are handled to avoid weird
 lockup

---
 .../src/components/common/bindings/BindingPanel.svelte | 10 ++++++++--
 .../common/bindings/DrawerBindableInput.svelte         |  9 ++-------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte
index 8c94455c58..0046725a72 100644
--- a/packages/builder/src/components/common/bindings/BindingPanel.svelte
+++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte
@@ -186,6 +186,8 @@
   }
 
   const updateValue = val => {
+    console.log("val", val)
+
     const runtimeExpression = readableToRuntimeBinding(enrichedBindings, val)
     dispatch("change", val)
     requestEval(runtimeExpression, context, snippets)
@@ -236,8 +238,12 @@
   }
 
   const onChangeJSValue = e => {
-    jsValue = encodeJSBinding(e.detail)
-    updateValue(jsValue)
+    if (!e.detail?.trim()) {
+      // Don't bother saving empty values as JS
+      updateValue("")
+    } else {
+      updateValue(encodeJSBinding(e.detail))
+    }
   }
 
   onMount(() => {
diff --git a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte
index 0792866581..0cb10d1aa5 100644
--- a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte
+++ b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte
@@ -6,7 +6,7 @@
   } from "dataBinding"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher, setContext } from "svelte"
-  import { isJSBinding, decodeJSBinding } from "@budibase/string-templates"
+  import { isJSBinding } from "@budibase/string-templates"
   import { builderStore } from "stores/builder"
 
   export let panel = ClientBindingPanel
@@ -34,12 +34,7 @@
   $: isJS = isJSBinding(value)
 
   const saveBinding = () => {
-    // Don't bother saving empty JS expressions as JS
-    let val = tempValue
-    if (decodeJSBinding(tempValue)?.trim() === "") {
-      val = null
-    }
-    onChange(val)
+    onChange(tempValue)
     onBlur()
     builderStore.propertyFocus()
     bindingDrawer.hide()

From 41ab3685bebab8732c280427b0e7526c9b7a4d1a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 22 May 2024 10:16:19 +0100
Subject: [PATCH 60/92] Remove log

---
 .../builder/src/components/common/bindings/BindingPanel.svelte  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte
index 0046725a72..c08509b0e4 100644
--- a/packages/builder/src/components/common/bindings/BindingPanel.svelte
+++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte
@@ -186,8 +186,6 @@
   }
 
   const updateValue = val => {
-    console.log("val", val)
-
     const runtimeExpression = readableToRuntimeBinding(enrichedBindings, val)
     dispatch("change", val)
     requestEval(runtimeExpression, context, snippets)

From 3cf3dd3afac5893cac3c288e100e49437c59a76f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 22 May 2024 10:45:03 +0100
Subject: [PATCH 61/92] Small tweak to JS bindings

---
 .../src/components/common/bindings/BindingPanel.svelte       | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte
index c08509b0e4..5114ad2d7a 100644
--- a/packages/builder/src/components/common/bindings/BindingPanel.svelte
+++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte
@@ -236,11 +236,12 @@
   }
 
   const onChangeJSValue = e => {
+    jsValue = encodeJSBinding(e.detail)
     if (!e.detail?.trim()) {
       // Don't bother saving empty values as JS
-      updateValue("")
+      updateValue(null)
     } else {
-      updateValue(encodeJSBinding(e.detail))
+      updateValue(jsValue)
     }
   }
 

From eac426f3c024af1c43c0e8c7dc94f9a907027595 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 22 May 2024 11:58:57 +0200
Subject: [PATCH 62/92] Return date only as dates

---
 packages/server/src/utilities/rowProcessor/utils.ts | 3 +++
 packages/server/src/utilities/schema.ts             | 6 +++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 8201680f13..0fa6f62807 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -105,6 +105,9 @@ export function processDates<T extends Row | Row[]>(
     if (schema.type !== FieldType.DATETIME) {
       continue
     }
+    if (schema.dateOnly) {
+      continue
+    }
     if (!schema.timeOnly && !schema.ignoreTimezones) {
       datesWithTZ.push(column)
     }
diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 5652391d7a..f73701fdfd 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -134,7 +134,11 @@ export function parse(rows: Rows, schema: TableSchema): Rows {
       if (columnType === FieldType.NUMBER) {
         // If provided must be a valid number
         parsedRow[columnName] = columnData ? Number(columnData) : columnData
-      } else if (columnType === FieldType.DATETIME && !columnSchema.timeOnly) {
+      } else if (
+        columnType === FieldType.DATETIME &&
+        !columnSchema.timeOnly &&
+        !columnSchema.dateOnly
+      ) {
         // If provided must be a valid date
         parsedRow[columnName] = columnData
           ? new Date(columnData).toISOString()

From 6dc463610a8533008ebbf0cb62f884654001624e Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 22 May 2024 10:22:02 +0000
Subject: [PATCH 63/92] Bump version to 2.27.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index f0b3f51d47..a943000cea 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.27.1",
+  "version": "2.27.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From 4e67e05ad1d73645e90dd6320e9cc742731e9bee Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 22 May 2024 14:35:19 +0200
Subject: [PATCH 64/92] Save date only as date

---
 packages/bbui/src/helpers.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/bbui/src/helpers.js b/packages/bbui/src/helpers.js
index 66bc6551d8..1db4a773ba 100644
--- a/packages/bbui/src/helpers.js
+++ b/packages/bbui/src/helpers.js
@@ -177,7 +177,7 @@ export const stringifyDate = (
     const year = value.year()
     const month = `${value.month() + 1}`.padStart(2, "0")
     const day = `${value.date()}`.padStart(2, "0")
-    return `${year}-${month}-${day}T00:00:00.000`
+    return `${year}-${month}-${day}`
   }
 
   // Otherwise use a normal ISO string with time and timezone

From b6e3e7659ccdf0eff0a01f6e7302add04ddb41d9 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 22 May 2024 14:01:59 +0100
Subject: [PATCH 65/92] Remove extraneous comment.

---
 packages/server/src/api/routes/tests/search.spec.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index d65980a7cb..45462f6848 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1385,8 +1385,7 @@ describe.each([
   })
 
   // This will never work for Lucene.
-  // TODO(samwho): fix for SQS
-  !isInternal &&
+  !isLucene &&
     describe("relations", () => {
       let otherTable: Table
       let rows: Row[]

From 46e310018ef125a96c6705a989740612d45f2616 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 22 May 2024 16:57:27 +0100
Subject: [PATCH 66/92] Fix tests.

---
 packages/server/src/api/routes/tests/search.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 40f61e17b7..5de788dc9c 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1027,7 +1027,7 @@ describe.each([
       const NULL_TIME__ID = `null_time__id`
 
       beforeAll(async () => {
-        await createTable({
+        table = await createTable({
           timeid: { name: "timeid", type: FieldType.STRING },
           time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
         })

From a9f8a72ebd33e341f1d10511a7cc0f0c52428444 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 22 May 2024 18:00:32 +0100
Subject: [PATCH 67/92] Attempting to fix tests.

---
 .../server/src/api/routes/tests/search.spec.ts    | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index e4fe461999..26bf58dbf3 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -87,8 +87,11 @@ describe.each([
   class SearchAssertion {
     constructor(private readonly query: RowSearchParams) {}
 
-    private popRow(expectedRow: any, foundRows: any[]) {
-      const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
+    private popRow<T extends object>(
+      expectedRow: T,
+      foundRows: T[]
+    ): NonNullable<T> {
+      const row = foundRows.find(row => _.isMatch(row, expectedRow))
       if (!row) {
         const fields = Object.keys(expectedRow)
         // To make the error message more readable, we only include the fields
@@ -130,6 +133,8 @@ describe.each([
     // passed in. The order of the rows is not important, but extra rows will
     // cause the assertion to fail.
     async toContainExactly(expectedRows: any[]) {
+      expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length)
+
       const { rows: foundRows } = await config.api.row.search(table._id!, {
         ...this.query,
         tableId: table._id!,
@@ -151,6 +156,8 @@ describe.each([
     // The order of the rows is not important. Extra rows will not cause the
     // assertion to fail.
     async toContain(expectedRows: any[]) {
+      expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length)
+
       const { rows: foundRows } = await config.api.row.search(table._id!, {
         ...this.query,
         tableId: table._id!,
@@ -1055,6 +1062,7 @@ describe.each([
       describe("notEqual", () => {
         it("successfully finds a row", () =>
           expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([
+            { timeid: NULL_TIME__ID },
             { time: "10:45:00" },
             { time: "12:00:00" },
             { time: "15:30:00" },
@@ -1064,6 +1072,7 @@ describe.each([
         it("return all when requesting non-existing", () =>
           expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly(
             [
+              { timeid: NULL_TIME__ID },
               { time: "10:00:00" },
               { time: "10:45:00" },
               { time: "12:00:00" },
@@ -1668,7 +1677,7 @@ describe.each([
     })
 
     describe("containsAny", () => {
-      it("successfully finds rows", () =>
+      it.only("successfully finds rows", () =>
         expectQuery({
           containsAny: { users: [user1._id, user2._id] },
         }).toContainExactly([

From 2cc329994a16ff3010259837782bdcf49aace95c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 22 May 2024 17:18:39 +0200
Subject: [PATCH 68/92] Don't allow chaning time only to datetime

---
 .../DataTable/modals/CreateEditColumn.svelte  | 40 ++++++++++---------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index e88c28a9d9..26d0357b5f 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -604,27 +604,29 @@
         <DatePicker bind:value={editableColumn.constraints.datetime.latest} />
       </div>
     </div>
-    {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
-      <div>
-        <div class="row">
-          <Label>Time zones</Label>
-          <AbsTooltip
-            position="top"
-            type="info"
-            text={isCreating
-              ? null
-              : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
-          >
-            <Icon size="XS" name="InfoOutline" />
-          </AbsTooltip>
+    {#if !editableColumn.timeOnly}
+      {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
+        <div>
+          <div class="row">
+            <Label>Time zones</Label>
+            <AbsTooltip
+              position="top"
+              type="info"
+              text={isCreating
+                ? null
+                : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
+            >
+              <Icon size="XS" name="InfoOutline" />
+            </AbsTooltip>
+          </div>
+          <Toggle
+            bind:value={editableColumn.ignoreTimezones}
+            text="Ignore time zones"
+          />
         </div>
-        <Toggle
-          bind:value={editableColumn.ignoreTimezones}
-          text="Ignore time zones"
-        />
-      </div>
+      {/if}
+      <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
     {/if}
-    <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
   {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">

From b3bea1e839b091efbe4beac0e50faa981c5e1716 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 22 May 2024 17:18:58 +0200
Subject: [PATCH 69/92] Block timeOnly changes in the api

---
 packages/server/src/sdk/app/tables/external/index.ts | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts
index 98e6e561c8..53302085fc 100644
--- a/packages/server/src/sdk/app/tables/external/index.ts
+++ b/packages/server/src/sdk/app/tables/external/index.ts
@@ -73,6 +73,16 @@ function validate(table: Table, oldTable?: Table) {
         `Column "${key}" has subtype "${column.subtype}" - this is not supported.`
       )
     }
+
+    if (column.type === FieldType.DATETIME) {
+      const oldColumn = oldTable?.schema[key] as typeof column
+
+      if (column.timeOnly !== oldColumn.timeOnly) {
+        throw new Error(
+          `Column "${key}" can not change from time to datetime or viceversa.`
+        )
+      }
+    }
   }
 }
 

From 028afd9cca04808615b672f9979f62af469b13d5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 11:34:58 +0200
Subject: [PATCH 70/92] Fix checks

---
 packages/server/src/sdk/app/tables/external/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts
index 53302085fc..842b6b5648 100644
--- a/packages/server/src/sdk/app/tables/external/index.ts
+++ b/packages/server/src/sdk/app/tables/external/index.ts
@@ -77,7 +77,7 @@ function validate(table: Table, oldTable?: Table) {
     if (column.type === FieldType.DATETIME) {
       const oldColumn = oldTable?.schema[key] as typeof column
 
-      if (column.timeOnly !== oldColumn.timeOnly) {
+      if (oldColumn && column.timeOnly !== oldColumn.timeOnly) {
         throw new Error(
           `Column "${key}" can not change from time to datetime or viceversa.`
         )

From f6146c4974b82e96d1ab3242d4ddb202de76015f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 11:33:41 +0200
Subject: [PATCH 71/92] Validate time only fields

---
 packages/server/src/sdk/app/rows/utils.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index 777ebff655..5456b81858 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -205,6 +205,10 @@ export async function validate({
       } catch (err) {
         errors[fieldName] = [`Contains invalid JSON`]
       }
+    } else if (type === FieldType.DATETIME && column.timeOnly) {
+      if (row[fieldName] && !row[fieldName].match(/^(\d+)(:[0-5]\d){1,2}$/)) {
+        errors[fieldName] = [`${fieldName} is not a valid time`]
+      }
     } else {
       res = validateJs.single(row[fieldName], constraints)
     }

From 2626c1a721ad40f3ae2bc35745e02b80b8b887ab Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 11:33:50 +0200
Subject: [PATCH 72/92] Send right time

---
 packages/bbui/src/helpers.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/bbui/src/helpers.js b/packages/bbui/src/helpers.js
index 1db4a773ba..dd94d12f7f 100644
--- a/packages/bbui/src/helpers.js
+++ b/packages/bbui/src/helpers.js
@@ -168,7 +168,11 @@ export const stringifyDate = (
     // Ensure we use the correct offset for the date
     const referenceDate = value.toDate()
     const offset = referenceDate.getTimezoneOffset() * 60000
-    return new Date(value.valueOf() - offset).toISOString().slice(0, -1)
+    const date = new Date(value.valueOf() - offset)
+    if (timeOnly) {
+      return date.toISOString().slice(11, 19)
+    }
+    return date.toISOString().slice(0, -1)
   }
 
   // For date-only fields, construct a manual timestamp string without a time

From 120f240f01e0178ddae12cb228b2058b9bb367a4 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 23 May 2024 10:43:44 +0100
Subject: [PATCH 73/92] Fix tests.

---
 .../src/api/routes/tests/search.spec.ts       | 67 +++++++++++++++----
 1 file changed, 53 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 26bf58dbf3..aac43874a0 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -87,24 +87,67 @@ describe.each([
   class SearchAssertion {
     constructor(private readonly query: RowSearchParams) {}
 
-    private popRow<T extends object>(
+    // We originally used _.isMatch to compare rows, but found that when
+    // comparing arrays it would return true if the source array was a subset of
+    // the target array. This would sometimes create false matches. This
+    // function is a more strict version of _.isMatch that only returns true if
+    // the source array is an exact match of the target.
+    //
+    // _.isMatch("100", "1") also returns true which is not what we want.
+    private isMatch<T extends Record<string, any>>(expected: T, found: T) {
+      if (!expected) {
+        throw new Error("Expected is undefined")
+      }
+      if (!found) {
+        return false
+      }
+
+      for (const key of Object.keys(expected)) {
+        if (Array.isArray(expected[key])) {
+          if (!Array.isArray(found[key])) {
+            return false
+          }
+          if (expected[key].length !== found[key].length) {
+            return false
+          }
+          if (!_.isMatch(found[key], expected[key])) {
+            return false
+          }
+        } else if (typeof expected[key] === "object") {
+          if (!this.isMatch(expected[key], found[key])) {
+            return false
+          }
+        } else {
+          if (expected[key] !== found[key]) {
+            return false
+          }
+        }
+      }
+      return true
+    }
+
+    // This function exists to ensure that the same row is not matched twice.
+    // When a row gets matched, we make sure to remove it from the list of rows
+    // we're matching against.
+    private popRow<T extends { [key: string]: any }>(
       expectedRow: T,
       foundRows: T[]
     ): NonNullable<T> {
-      const row = foundRows.find(row => _.isMatch(row, expectedRow))
+      const row = foundRows.find(row => this.isMatch(expectedRow, row))
       if (!row) {
         const fields = Object.keys(expectedRow)
         // To make the error message more readable, we only include the fields
         // that are present in the expected row.
         const searchedObjects = foundRows.map(row => _.pick(row, fields))
         throw new Error(
-          `Failed to find row: ${JSON.stringify(
-            expectedRow
-          )} in ${JSON.stringify(searchedObjects)}`
+          `Failed to find row:\n\n${JSON.stringify(
+            expectedRow,
+            null,
+            2
+          )}\n\nin\n\n${JSON.stringify(searchedObjects, null, 2)}`
         )
       }
 
-      // Ensuring the same row is not matched twice
       foundRows.splice(foundRows.indexOf(row), 1)
       return row
     }
@@ -133,8 +176,6 @@ describe.each([
     // passed in. The order of the rows is not important, but extra rows will
     // cause the assertion to fail.
     async toContainExactly(expectedRows: any[]) {
-      expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length)
-
       const { rows: foundRows } = await config.api.row.search(table._id!, {
         ...this.query,
         tableId: table._id!,
@@ -156,8 +197,6 @@ describe.each([
     // The order of the rows is not important. Extra rows will not cause the
     // assertion to fail.
     async toContain(expectedRows: any[]) {
-      expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length)
-
       const { rows: foundRows } = await config.api.row.search(table._id!, {
         ...this.query,
         tableId: table._id!,
@@ -1592,7 +1631,7 @@ describe.each([
         expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([
           { user: { _id: user1._id } },
           { user: { _id: user2._id } },
-          { user: {} },
+          {},
         ]))
     })
 
@@ -1664,7 +1703,7 @@ describe.each([
       it("successfully finds a row", () =>
         expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([
           { users: [{ _id: user2._id }] },
-          { users: [] },
+          {},
         ]))
 
       it("fails to find nonexistent row", () =>
@@ -1672,12 +1711,12 @@ describe.each([
           { users: [{ _id: user1._id }] },
           { users: [{ _id: user2._id }] },
           { users: [{ _id: user1._id }, { _id: user2._id }] },
-          { users: [] },
+          {},
         ]))
     })
 
     describe("containsAny", () => {
-      it.only("successfully finds rows", () =>
+      it("successfully finds rows", () =>
         expectQuery({
           containsAny: { users: [user1._id, user2._id] },
         }).toContainExactly([

From 9b3f467a52f917c320a25c53e7cde3038fae5f63 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 11:51:02 +0200
Subject: [PATCH 74/92] Time-date only on constraint settings

---
 .../DataTable/modals/CreateEditColumn.svelte       | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index e88c28a9d9..b16e57e866 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -586,13 +586,17 @@
       bind:constraints={editableColumn.constraints}
       bind:optionColors={editableColumn.optionColors}
     />
-  {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn}
+  {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
         <Label size="M">Earliest</Label>
       </div>
       <div class="input-length">
-        <DatePicker bind:value={editableColumn.constraints.datetime.earliest} />
+        <DatePicker
+          bind:value={editableColumn.constraints.datetime.earliest}
+          enableTime={!editableColumn.dateOnly}
+          timeOnly={editableColumn.timeOnly}
+        />
       </div>
     </div>
 
@@ -601,7 +605,11 @@
         <Label size="M">Latest</Label>
       </div>
       <div class="input-length">
-        <DatePicker bind:value={editableColumn.constraints.datetime.latest} />
+        <DatePicker
+          bind:value={editableColumn.constraints.datetime.latest}
+          enableTime={!editableColumn.dateOnly}
+          timeOnly={editableColumn.timeOnly}
+        />
       </div>
     </div>
     {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}

From ac92aaeab3f8f5ad8a78f38abd9ef4cf8f2b03aa Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 13:00:50 +0200
Subject: [PATCH 75/92] Extract HH:mm

---
 packages/bbui/src/helpers.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/bbui/src/helpers.js b/packages/bbui/src/helpers.js
index dd94d12f7f..0f912a7161 100644
--- a/packages/bbui/src/helpers.js
+++ b/packages/bbui/src/helpers.js
@@ -170,7 +170,8 @@ export const stringifyDate = (
     const offset = referenceDate.getTimezoneOffset() * 60000
     const date = new Date(value.valueOf() - offset)
     if (timeOnly) {
-      return date.toISOString().slice(11, 19)
+      // Extract HH:mm
+      return date.toISOString().slice(11, 16)
     }
     return date.toISOString().slice(0, -1)
   }

From 402426a5f3b06ceb7cc78948135fc0fc0767351d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 13:07:45 +0200
Subject: [PATCH 76/92] Validate time only field constrains

---
 packages/server/src/sdk/app/rows/utils.ts | 60 +++++++++++++++++++++--
 1 file changed, 56 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index 5456b81858..78f499f895 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -1,8 +1,10 @@
-import cloneDeep from "lodash/cloneDeep"
 import validateJs from "validate.js"
+import dayjs from "dayjs"
+import cloneDeep from "lodash/fp/cloneDeep"
 import {
   Datasource,
   DatasourcePlusQueryResponse,
+  FieldConstraints,
   FieldType,
   QueryJson,
   Row,
@@ -206,9 +208,7 @@ export async function validate({
         errors[fieldName] = [`Contains invalid JSON`]
       }
     } else if (type === FieldType.DATETIME && column.timeOnly) {
-      if (row[fieldName] && !row[fieldName].match(/^(\d+)(:[0-5]\d){1,2}$/)) {
-        errors[fieldName] = [`${fieldName} is not a valid time`]
-      }
+      res = validateTimeOnlyField(fieldName, row[fieldName], constraints)
     } else {
       res = validateJs.single(row[fieldName], constraints)
     }
@@ -216,3 +216,55 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
+
+function validateTimeOnlyField(
+  fieldName: string,
+  value: any,
+  constraints: FieldConstraints | undefined
+) {
+  let res
+  if (value && !value.match(/^(\d+)(:[0-5]\d){1,2}$/)) {
+    res = [`${fieldName} is not a valid time`]
+  }
+  if (constraints) {
+    let castedValue = value
+    const stringTimeToDateISOString = (value: string) => {
+      const [hour, minute] = value.split(":").map((x: string) => +x)
+      return dayjs().hour(hour).minute(minute).toISOString()
+    }
+
+    if (castedValue) {
+      castedValue = stringTimeToDateISOString(castedValue)
+    }
+    let castedConstraints = cloneDeep(constraints)
+    if (castedConstraints.datetime?.earliest) {
+      castedConstraints.datetime.earliest = stringTimeToDateISOString(
+        castedConstraints.datetime?.earliest
+      )
+    }
+    if (castedConstraints.datetime?.latest) {
+      castedConstraints.datetime.latest = stringTimeToDateISOString(
+        castedConstraints.datetime?.latest
+      )
+    }
+
+    let jsValidation = validateJs.single(castedValue, castedConstraints)
+    jsValidation = jsValidation?.map((m: string) =>
+      m
+        ?.replace(
+          castedConstraints.datetime?.earliest || "",
+          constraints.datetime?.earliest || ""
+        )
+        .replace(
+          castedConstraints.datetime?.latest || "",
+          constraints.datetime?.latest || ""
+        )
+    )
+    if (jsValidation) {
+      res ??= []
+      res.push(...jsValidation)
+    }
+  }
+
+  return res
+}

From e169454490ba580302fdad8b75470dc1ad8bb0bb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 13:51:49 +0200
Subject: [PATCH 77/92] Move utils to backend-core

---
 packages/backend-core/src/docIds/ids.ts | 8 ++++++++
 packages/server/src/db/utils.ts         | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/docIds/ids.ts b/packages/backend-core/src/docIds/ids.ts
index 9627b2b94c..a828c1b91e 100644
--- a/packages/backend-core/src/docIds/ids.ts
+++ b/packages/backend-core/src/docIds/ids.ts
@@ -18,6 +18,14 @@ export const generateAppID = (tenantId?: string | null) => {
   return `${id}${newid()}`
 }
 
+/**
+ * Generates a new table ID.
+ * @returns The new table ID which the table doc can be stored under.
+ */
+export function generateTableID() {
+  return `${DocumentType.TABLE}${SEPARATOR}${newid()}`
+}
+
 /**
  * Gets a new row ID for the specified table.
  * @param tableId The table which the row is being created for.
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index ce8d0accbb..e42f30c723 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -77,7 +77,7 @@ export function getTableParams(tableId?: Optional, otherProps = {}) {
  * @returns The new table ID which the table doc can be stored under.
  */
 export function generateTableID() {
-  return `${DocumentType.TABLE}${SEPARATOR}${newid()}`
+  return dbCore.generateTableID()
 }
 
 /**

From bed18615b54df076bbae18cbef147621a97c7d06 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 13:57:42 +0200
Subject: [PATCH 78/92] Add basic tests

---
 .../src/sdk/app/rows/tests/utils.spec.ts      | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 packages/server/src/sdk/app/rows/tests/utils.spec.ts

diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
new file mode 100644
index 0000000000..38c2c64d6b
--- /dev/null
+++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
@@ -0,0 +1,48 @@
+import {
+  FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
+  Table,
+  TableSourceType,
+} from "@budibase/types"
+import { generateTableID } from "../../../../db/utils"
+import { validate } from "../utils"
+
+describe("validate", () => {
+  describe("time only", () => {
+    const getTable = (): Table => ({
+      type: "table",
+      _id: generateTableID(),
+      name: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+      schema: {
+        time: {
+          name: "time",
+          type: FieldType.DATETIME,
+          timeOnly: true,
+        },
+      },
+    })
+
+    it("should accept empty fields", async () => {
+      const row = {}
+      const table = getTable()
+      const output = await validate({ table, tableId: table._id!, row })
+      expect(output.valid).toBe(true)
+      expect(output.errors).toEqual({})
+    })
+
+    describe("required", () => {
+      it("should reject empty fields", async () => {
+        const row = {}
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+        }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({ time: ["can't be blank"] })
+      })
+    })
+  })
+})

From fb06254964acf4c1a67a034a0afe8e456998a4b6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 14:00:15 +0200
Subject: [PATCH 79/92] Extra tests

---
 .../src/sdk/app/rows/tests/utils.spec.ts      | 34 +++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
index 38c2c64d6b..e7a9fa10eb 100644
--- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
@@ -6,6 +6,7 @@ import {
 } from "@budibase/types"
 import { generateTableID } from "../../../../db/utils"
 import { validate } from "../utils"
+import { generator } from "@budibase/backend-core/tests"
 
 describe("validate", () => {
   describe("time only", () => {
@@ -24,7 +25,7 @@ describe("validate", () => {
       },
     })
 
-    it("should accept empty fields", async () => {
+    it("should accept empty values", async () => {
       const row = {}
       const table = getTable()
       const output = await validate({ table, tableId: table._id!, row })
@@ -32,8 +33,26 @@ describe("validate", () => {
       expect(output.errors).toEqual({})
     })
 
+    it("should accept valid times with HH:mm format", async () => {
+      const row = {
+        time: `${generator.hour()}:${generator.minute()}`,
+      }
+      const table = getTable()
+      const output = await validate({ table, tableId: table._id!, row })
+      expect(output.valid).toBe(true)
+    })
+
+    it("should accept valid times with HH:mm:ss format", async () => {
+      const row = {
+        time: `${generator.hour()}:${generator.minute()}:${generator.second()}`,
+      }
+      const table = getTable()
+      const output = await validate({ table, tableId: table._id!, row })
+      expect(output.valid).toBe(true)
+    })
+
     describe("required", () => {
-      it("should reject empty fields", async () => {
+      it("should reject empty values", async () => {
         const row = {}
         const table = getTable()
         table.schema.time.constraints = {
@@ -43,6 +62,17 @@ describe("validate", () => {
         expect(output.valid).toBe(false)
         expect(output.errors).toEqual({ time: ["can't be blank"] })
       })
+
+      it.each([undefined, null])("should reject %s values", async time => {
+        const row = { time }
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+        }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({ time: ["can't be blank"] })
+      })
     })
   })
 })

From d58c144dce7d76215e6f0edbeea7bae945ea5569 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 14:27:31 +0200
Subject: [PATCH 80/92] Add extra tests

---
 .../src/sdk/app/rows/tests/utils.spec.ts      | 81 ++++++++++++++++++-
 packages/server/src/sdk/app/rows/utils.ts     | 13 +--
 2 files changed, 87 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
index e7a9fa10eb..bed2a7fbfb 100644
--- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
@@ -9,6 +9,10 @@ import { validate } from "../utils"
 import { generator } from "@budibase/backend-core/tests"
 
 describe("validate", () => {
+  const hour = () => generator.hour().toString().padStart(2, "0")
+  const minute = () => generator.minute().toString().padStart(2, "0")
+  const second = minute
+
   describe("time only", () => {
     const getTable = (): Table => ({
       type: "table",
@@ -35,7 +39,7 @@ describe("validate", () => {
 
     it("should accept valid times with HH:mm format", async () => {
       const row = {
-        time: `${generator.hour()}:${generator.minute()}`,
+        time: `${hour()}:${minute()}`,
       }
       const table = getTable()
       const output = await validate({ table, tableId: table._id!, row })
@@ -44,13 +48,86 @@ describe("validate", () => {
 
     it("should accept valid times with HH:mm:ss format", async () => {
       const row = {
-        time: `${generator.hour()}:${generator.minute()}:${generator.second()}`,
+        time: `${hour()}:${minute()}:${second()}`,
       }
       const table = getTable()
       const output = await validate({ table, tableId: table._id!, row })
       expect(output.valid).toBe(true)
     })
 
+    it.each([
+      ["ISO datetimes", generator.date().toISOString()],
+      ["random values", generator.word()],
+    ])("should reject %s", async (_, time) => {
+      const row = {
+        time,
+      }
+      const table = getTable()
+      table.schema.time.constraints = {
+        presence: true,
+      }
+      const output = await validate({ table, tableId: table._id!, row })
+      expect(output.valid).toBe(false)
+      expect(output.errors).toEqual({ time: ['"time" is not a valid time'] })
+    })
+
+    describe("time constraints", () => {
+      it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])(
+        "should accept values in range (%s)",
+        async time => {
+          const row = { time }
+          const table = getTable()
+          table.schema.time.constraints = {
+            presence: true,
+            datetime: {
+              earliest: "10:00",
+              latest: "15:00",
+            },
+          }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(true)
+        }
+      )
+
+      it.each([
+        "9:59:50",
+        `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
+      ])("should reject values before range (%s)", async time => {
+        const row = { time }
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+          datetime: {
+            earliest: "10:00",
+            latest: "15:00",
+          },
+        }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({
+          time: ["must be no earlier than 10:00"],
+        })
+      })
+
+      it.each([
+        "15:00:01",
+        `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
+      ])("should reject values after range (%s)", async time => {
+        const row = { time }
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+          datetime: {
+            earliest: "10:00",
+            latest: "15:00",
+          },
+        }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({ time: ["must be no later than 15:00"] })
+      })
+    })
+
     describe("required", () => {
       it("should reject empty values", async () => {
         const row = {}
diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index 78f499f895..708bd6f503 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -224,13 +224,16 @@ function validateTimeOnlyField(
 ) {
   let res
   if (value && !value.match(/^(\d+)(:[0-5]\d){1,2}$/)) {
-    res = [`${fieldName} is not a valid time`]
-  }
-  if (constraints) {
+    res = [`"${fieldName}" is not a valid time`]
+  } else if (constraints) {
     let castedValue = value
     const stringTimeToDateISOString = (value: string) => {
-      const [hour, minute] = value.split(":").map((x: string) => +x)
-      return dayjs().hour(hour).minute(minute).toISOString()
+      const [hour, minute, second] = value.split(":").map((x: string) => +x)
+      let date = dayjs("2000-01-01T00:00:00.000Z").hour(hour).minute(minute)
+      if (!isNaN(second)) {
+        date = date.second(second)
+      }
+      return date.toISOString()
     }
 
     if (castedValue) {

From 5b80e4fb6ecbb6ff51001ff739cd7d726788730b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 14:35:33 +0200
Subject: [PATCH 81/92] Add more tests

---
 .../src/sdk/app/rows/tests/utils.spec.ts      | 128 +++++++++++++-----
 1 file changed, 93 insertions(+), 35 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
index bed2a7fbfb..7361f7b13c 100644
--- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
@@ -72,48 +72,76 @@ describe("validate", () => {
     })
 
     describe("time constraints", () => {
-      it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])(
-        "should accept values in range (%s)",
-        async time => {
-          const row = { time }
-          const table = getTable()
-          table.schema.time.constraints = {
-            presence: true,
-            datetime: {
-              earliest: "10:00",
-              latest: "15:00",
-            },
-          }
-          const output = await validate({ table, tableId: table._id!, row })
-          expect(output.valid).toBe(true)
-        }
-      )
-
-      it.each([
-        "9:59:50",
-        `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
-      ])("should reject values before range (%s)", async time => {
-        const row = { time }
+      describe("earliest only", () => {
         const table = getTable()
         table.schema.time.constraints = {
           presence: true,
           datetime: {
             earliest: "10:00",
-            latest: "15:00",
+            latest: "",
           },
         }
-        const output = await validate({ table, tableId: table._id!, row })
-        expect(output.valid).toBe(false)
-        expect(output.errors).toEqual({
-          time: ["must be no earlier than 10:00"],
+
+        it.each([
+          "10:00",
+          "15:00",
+          `10:${minute()}`,
+          "12:34",
+          `${generator.integer({ min: 11, max: 23 })}:${minute()}`,
+        ])("should accept values after config value (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(true)
+        })
+
+        it.each([
+          "09:59:59",
+          `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
+        ])("should reject values before config value (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no earlier than 10:00"],
+          })
         })
       })
 
-      it.each([
-        "15:00:01",
-        `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
-      ])("should reject values after range (%s)", async time => {
-        const row = { time }
+      describe("latest only", () => {
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+          datetime: {
+            earliest: "",
+            latest: "15:16:17",
+          },
+        }
+
+        it.each([
+          "15:16:17",
+          "15:16",
+          "15:00",
+          `${generator.integer({ min: 0, max: 12 })}:${minute()}`,
+        ])("should accept values before config value (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(true)
+        })
+
+        it.each([
+          "15:16:18",
+          `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
+        ])("should reject values after config value (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no later than 15:16:17"],
+          })
+        })
+      })
+
+      describe("range", () => {
         const table = getTable()
         table.schema.time.constraints = {
           presence: true,
@@ -122,9 +150,39 @@ describe("validate", () => {
             latest: "15:00",
           },
         }
-        const output = await validate({ table, tableId: table._id!, row })
-        expect(output.valid).toBe(false)
-        expect(output.errors).toEqual({ time: ["must be no later than 15:00"] })
+
+        it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])(
+          "should accept values in range (%s)",
+          async time => {
+            const row = { time }
+            const output = await validate({ table, tableId: table._id!, row })
+            expect(output.valid).toBe(true)
+          }
+        )
+
+        it.each([
+          "9:59:50",
+          `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
+        ])("should reject values before range (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no earlier than 10:00"],
+          })
+        })
+
+        it.each([
+          "15:00:01",
+          `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
+        ])("should reject values after range (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no later than 15:00"],
+          })
+        })
       })
     })
 

From d1ef9067dcbaaa345c975bb55fba46107818e0b6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 14:47:54 +0200
Subject: [PATCH 82/92] Allow range crossing midnight

---
 .../src/sdk/app/rows/tests/utils.spec.ts      | 20 +++++++++++
 packages/server/src/sdk/app/rows/utils.ts     | 35 +++++++++++++------
 2 files changed, 45 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
index 7361f7b13c..acd4281256 100644
--- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
@@ -183,6 +183,26 @@ describe("validate", () => {
             time: ["must be no later than 15:00"],
           })
         })
+
+        describe("range crossing midnight", () => {
+          const table = getTable()
+          table.schema.time.constraints = {
+            presence: true,
+            datetime: {
+              earliest: "15:00",
+              latest: "10:00",
+            },
+          }
+
+          it.each(["10:00", "15:00", `9:${minute()}`, "16:34"])(
+            "should accept values in range (%s)",
+            async time => {
+              const row = { time }
+              const output = await validate({ table, tableId: table._id!, row })
+              expect(output.valid).toBe(true)
+            }
+          )
+        })
       })
     })
 
diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index 708bd6f503..19e8869494 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -227,31 +227,46 @@ function validateTimeOnlyField(
     res = [`"${fieldName}" is not a valid time`]
   } else if (constraints) {
     let castedValue = value
-    const stringTimeToDateISOString = (value: string) => {
+    const stringTimeToDate = (value: string) => {
       const [hour, minute, second] = value.split(":").map((x: string) => +x)
       let date = dayjs("2000-01-01T00:00:00.000Z").hour(hour).minute(minute)
       if (!isNaN(second)) {
         date = date.second(second)
       }
-      return date.toISOString()
+      return date
     }
 
     if (castedValue) {
-      castedValue = stringTimeToDateISOString(castedValue)
+      castedValue = stringTimeToDate(castedValue)
     }
     let castedConstraints = cloneDeep(constraints)
+
+    let earliest, latest
     if (castedConstraints.datetime?.earliest) {
-      castedConstraints.datetime.earliest = stringTimeToDateISOString(
-        castedConstraints.datetime?.earliest
-      )
+      earliest = stringTimeToDate(castedConstraints.datetime?.earliest)
     }
     if (castedConstraints.datetime?.latest) {
-      castedConstraints.datetime.latest = stringTimeToDateISOString(
-        castedConstraints.datetime?.latest
-      )
+      latest = stringTimeToDate(castedConstraints.datetime?.latest)
     }
 
-    let jsValidation = validateJs.single(castedValue, castedConstraints)
+    if (earliest && latest && earliest.isAfter(latest)) {
+      latest = latest.add(1, "day")
+      if (earliest.isAfter(castedValue)) {
+        castedValue = castedValue.add(1, "day")
+      }
+    }
+
+    if (earliest || latest) {
+      castedConstraints.datetime = {
+        earliest: earliest?.toISOString() || "",
+        latest: latest?.toISOString() || "",
+      }
+    }
+
+    let jsValidation = validateJs.single(
+      castedValue?.toISOString(),
+      castedConstraints
+    )
     jsValidation = jsValidation?.map((m: string) =>
       m
         ?.replace(

From b8400294d54b1baef60de5e95a835918f21a8656 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 14:53:17 +0200
Subject: [PATCH 83/92] Add extra tests

---
 .../server/src/sdk/app/rows/tests/utils.spec.ts    | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
index acd4281256..4c44791135 100644
--- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
@@ -194,7 +194,7 @@ describe("validate", () => {
             },
           }
 
-          it.each(["10:00", "15:00", `9:${minute()}`, "16:34"])(
+          it.each(["10:00", "15:00", `9:${minute()}`, "16:34", "00:00"])(
             "should accept values in range (%s)",
             async time => {
               const row = { time }
@@ -202,6 +202,18 @@ describe("validate", () => {
               expect(output.valid).toBe(true)
             }
           )
+
+          it.each(["10:01", "14:59:59", `12:${minute()}`])(
+            "should reject values out range (%s)",
+            async time => {
+              const row = { time }
+              const output = await validate({ table, tableId: table._id!, row })
+              expect(output.valid).toBe(false)
+              expect(output.errors).toEqual({
+                time: ["must be no later than 10:00"],
+              })
+            }
+          )
         })
       })
     })

From 43acea931ac4b3846604a40569d125b2ad7155af Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 23 May 2024 15:23:02 +0200
Subject: [PATCH 84/92] Ensure iso time config still work

---
 .../src/sdk/app/rows/tests/utils.spec.ts      | 90 +++++++++++++++++++
 packages/server/src/sdk/app/rows/utils.ts     | 21 ++++-
 2 files changed, 107 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
index 4c44791135..55cdf9ea20 100644
--- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts
@@ -1,3 +1,4 @@
+import dayjs from "dayjs"
 import {
   FieldType,
   INTERNAL_TABLE_SOURCE_ID,
@@ -241,5 +242,94 @@ describe("validate", () => {
         expect(output.errors).toEqual({ time: ["can't be blank"] })
       })
     })
+
+    describe("range", () => {
+      const table = getTable()
+      table.schema.time.constraints = {
+        presence: true,
+        datetime: {
+          earliest: "10:00",
+          latest: "15:00",
+        },
+      }
+
+      it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])(
+        "should accept values in range (%s)",
+        async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(true)
+        }
+      )
+
+      it.each([
+        "9:59:50",
+        `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
+      ])("should reject values before range (%s)", async time => {
+        const row = { time }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({
+          time: ["must be no earlier than 10:00"],
+        })
+      })
+
+      it.each([
+        "15:00:01",
+        `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
+      ])("should reject values after range (%s)", async time => {
+        const row = { time }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({
+          time: ["must be no later than 15:00"],
+        })
+      })
+
+      describe("datetime ISO configs", () => {
+        const table = getTable()
+
+        table.schema.time.constraints = {
+          presence: true,
+          datetime: {
+            earliest: dayjs().hour(10).minute(0).second(0).toISOString(),
+            latest: dayjs().hour(15).minute(0).second(0).toISOString(),
+          },
+        }
+
+        it.each(["10:00", "15:00", `12:${minute()}`])(
+          "should accept values in range (%s)",
+          async time => {
+            const row = { time }
+            const output = await validate({ table, tableId: table._id!, row })
+            expect(output.valid).toBe(true)
+          }
+        )
+
+        it.each([
+          "09:59:50",
+          `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
+        ])("should reject values before range (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no earlier than 10:00"],
+          })
+        })
+
+        it.each([
+          "15:00:01",
+          `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
+        ])("should reject values after range (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no later than 15:00"],
+          })
+        })
+      })
+    })
   })
 })
diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index 19e8869494..23e1ab3e6b 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -242,11 +242,24 @@ function validateTimeOnlyField(
     let castedConstraints = cloneDeep(constraints)
 
     let earliest, latest
+    let easliestTimeString: string, latestTimeString: string
     if (castedConstraints.datetime?.earliest) {
-      earliest = stringTimeToDate(castedConstraints.datetime?.earliest)
+      easliestTimeString = castedConstraints.datetime.earliest
+      if (dayjs(castedConstraints.datetime.earliest).isValid()) {
+        easliestTimeString = dayjs(castedConstraints.datetime.earliest).format(
+          "HH:mm"
+        )
+      }
+      earliest = stringTimeToDate(easliestTimeString)
     }
     if (castedConstraints.datetime?.latest) {
-      latest = stringTimeToDate(castedConstraints.datetime?.latest)
+      latestTimeString = castedConstraints.datetime.latest
+      if (dayjs(castedConstraints.datetime.latest).isValid()) {
+        latestTimeString = dayjs(castedConstraints.datetime.latest).format(
+          "HH:mm"
+        )
+      }
+      latest = stringTimeToDate(latestTimeString)
     }
 
     if (earliest && latest && earliest.isAfter(latest)) {
@@ -271,11 +284,11 @@ function validateTimeOnlyField(
       m
         ?.replace(
           castedConstraints.datetime?.earliest || "",
-          constraints.datetime?.earliest || ""
+          easliestTimeString || ""
         )
         .replace(
           castedConstraints.datetime?.latest || "",
-          constraints.datetime?.latest || ""
+          latestTimeString || ""
         )
     )
     if (jsValidation) {

From a6d2f82e7b185656e661d7a7821b3965d00b118f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 23 May 2024 15:16:52 +0100
Subject: [PATCH 85/92] Fix tests.

---
 packages/server/src/api/controllers/view/exporters.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts
index 3b5f951dca..9cf114f4e5 100644
--- a/packages/server/src/api/controllers/view/exporters.ts
+++ b/packages/server/src/api/controllers/view/exporters.ts
@@ -57,5 +57,5 @@ export function isFormat(format: any): format is Format {
 }
 
 export function parseCsvExport<T>(value: string) {
-  return JSON.parse(value?.replace(/'/g, '"')) as T
+  return JSON.parse(value) as T
 }

From 7d256d235afe75c179056616d142392a96101637 Mon Sep 17 00:00:00 2001
From: Conor Webb <126772285+ConorWebb96@users.noreply.github.com>
Date: Thu, 23 May 2024 15:31:11 +0100
Subject: [PATCH 86/92] Enum columns unexpectedly breaking fix (#13756)

* Added enum to SQL_MISC_TYPE_MAP to correctly map to FieldType.OPTIONS

* improve enum values extraction for multiple single-select support

* Tests to ensure enums are typed correctly MySQL and Postgres

* Fixed linting issue

* Ran prettier
---
 .../server/src/integration-test/mysql.spec.ts | 36 +++++++++++++++++++
 .../src/integration-test/postgres.spec.ts     | 31 ++++++++++++++++
 packages/server/src/integrations/postgres.ts  | 10 +++---
 .../server/src/integrations/utils/utils.ts    |  1 +
 4 files changed, 72 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts
index b4eb1035d6..8cf4fb8212 100644
--- a/packages/server/src/integration-test/mysql.spec.ts
+++ b/packages/server/src/integration-test/mysql.spec.ts
@@ -281,4 +281,40 @@ describe("mysql integrations", () => {
       ])
     })
   })
+
+  describe("POST /api/datasources/:datasourceId/schema", () => {
+    let tableName: string
+
+    beforeEach(async () => {
+      tableName = uniqueTableName()
+    })
+
+    afterEach(async () => {
+      await rawQuery(rawDatasource, `DROP TABLE IF EXISTS \`${tableName}\``)
+    })
+
+    it("recognises enum columns as options", async () => {
+      const enumColumnName = "status"
+
+      const createTableQuery = `
+        CREATE TABLE \`${tableName}\` (
+          \`order_id\` INT AUTO_INCREMENT PRIMARY KEY,
+          \`customer_name\` VARCHAR(100) NOT NULL,
+          \`${enumColumnName}\` ENUM('pending', 'processing', 'shipped', 'delivered', 'cancelled')
+        );
+      `
+
+      await rawQuery(rawDatasource, createTableQuery)
+
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${datasource._id}/schema`
+      )
+
+      const table = response.body.datasource.entities[tableName]
+
+      expect(table).toBeDefined()
+      expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS)
+    })
+  })
 })
diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index ec4cb90a86..ccf63d0820 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -1122,6 +1122,37 @@ describe("postgres integrations", () => {
         [tableName]: "Table contains invalid columns.",
       })
     })
+
+    it("recognises enum columns as options", async () => {
+      const tableName = `orders_${generator
+        .guid()
+        .replaceAll("-", "")
+        .substring(0, 6)}`
+      const enumColumnName = "status"
+
+      await rawQuery(
+        rawDatasource,
+        `
+        CREATE TYPE order_status AS ENUM ('pending', 'processing', 'shipped', 'delivered', 'cancelled');
+        
+        CREATE TABLE ${tableName} (
+          order_id SERIAL PRIMARY KEY,
+          customer_name VARCHAR(100) NOT NULL,
+          ${enumColumnName} order_status
+        );
+      `
+      )
+
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${datasource._id}/schema`
+      )
+
+      const table = response.body.datasource.entities[tableName]
+
+      expect(table).toBeDefined()
+      expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS)
+    })
   })
 
   describe("Integration compatibility with postgres search_path", () => {
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index e810986757..584f0ef15d 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -329,14 +329,12 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
 
       // Fetch enum values
       const enumsResponse = await this.client.query(this.ENUM_VALUES())
+      // output array, allows for more than 1 single-select to be used at a time
       const enumValues = enumsResponse.rows?.reduce((acc, row) => {
-        if (!acc[row.typname]) {
-          return {
-            [row.typname]: [row.enumlabel],
-          }
+        return {
+          ...acc,
+          [row.typname]: [...(acc[row.typname] || []), row.enumlabel],
         }
-        acc[row.typname].push(row.enumlabel)
-        return acc
       }, {})
 
       for (let column of columnsResponse.rows) {
diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts
index 8f3aba8907..5bc90bc295 100644
--- a/packages/server/src/integrations/utils/utils.ts
+++ b/packages/server/src/integrations/utils/utils.ts
@@ -102,6 +102,7 @@ const SQL_OPTIONS_TYPE_MAP: Record<string, PrimitiveTypes> = {
 const SQL_MISC_TYPE_MAP: Record<string, PrimitiveTypes> = {
   json: FieldType.JSON,
   bigint: FieldType.BIGINT,
+  enum: FieldType.OPTIONS,
 }
 
 const SQL_TYPE_MAP: Record<string, PrimitiveTypes> = {

From 1004b052baae179513d32dfab57e1a3a64ea1ed3 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 23 May 2024 14:39:52 +0000
Subject: [PATCH 87/92] Bump version to 2.27.3

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index a943000cea..db0a1d59fa 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.27.2",
+  "version": "2.27.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From 325819ebae9fea081f2bd46a94952d7b51fab58a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 23 May 2024 16:13:07 +0100
Subject: [PATCH 88/92] Fix tests (take 3).

---
 .../src/integrations/tests/sqlAlias.spec.ts     | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 0de4d0a151..f907e1ab8e 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -77,7 +77,7 @@ describe("Captures of real examples", () => {
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
               from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" 
               left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
-              left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 
+              left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where COALESCE("b"."taskname" = $2, FALSE)
               order by "a"."productname" asc nulls first limit $3`),
       })
     })
@@ -137,12 +137,12 @@ describe("Captures of real examples", () => {
              "c"."city" as "c.city", "c"."lastname" as "c.lastname", "c"."year" as "c.year", "c"."firstname" as "c.firstname", 
              "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", 
              "c"."city" as "c.city", "c"."lastname" as "c.lastname" 
-             from (select * from "tasks" as "a" where not "a"."completed" = $1 
+             from (select * from "tasks" as "a" where COALESCE("a"."completed" != $1, TRUE)
              order by "a"."taskname" asc nulls first limit $2) as "a" 
              left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" 
              left join "products" as "b" on "b"."productid" = "d"."productid" 
              left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" 
-             where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc nulls first limit $6`),
+             where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE) order by "a"."taskname" asc nulls first limit $6`),
       })
     })
   })
@@ -154,7 +154,7 @@ describe("Captures of real examples", () => {
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
         sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
-             "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`),
+             "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`),
       })
     })
 
@@ -164,7 +164,7 @@ describe("Captures of real examples", () => {
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
         sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
-             "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`),
+             "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`),
       })
     })
   })
@@ -175,8 +175,9 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: ["ddd", ""],
-        sql: multiline(`delete from "compositetable" as "a" where "a"."keypartone" = $1 and "a"."keyparttwo" = $2 
-             returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`),
+        sql: multiline(`delete from "compositetable" as "a" 
+          where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
+          returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`),
       })
     })
   })
@@ -197,7 +198,7 @@ describe("Captures of real examples", () => {
         returningQuery = input
       }, queryJson)
       expect(returningQuery).toEqual({
-        sql: "select * from (select top (@p0) * from [people] where [people].[name] = @p1 and [people].[age] = @p2 order by [people].[name] asc) as [people]",
+        sql: "select * from (select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]",
         bindings: [1, "Test", 22],
       })
     })

From ef60893df1677d58c450f70dcf7532f45ee305be Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 23 May 2024 17:02:08 +0100
Subject: [PATCH 89/92] Fix tests (take 4).

---
 packages/server/src/integrations/tests/sql.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index 9b84409e92..302b61fc74 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -189,7 +189,7 @@ describe("SQL query builder", () => {
     )
     expect(query).toEqual({
       bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit],
-      sql: `select * from (select * from (select * from "test" where (LOWER("test"."age") LIKE :1 AND LOWER("test"."age") LIKE :2) and (LOWER("test"."name") LIKE :3 AND LOWER("test"."name") LIKE :4)) where rownum <= :5) "test"`,
+      sql: `select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4)) where rownum <= :5) "test"`,
     })
 
     query = new Sql(SqlClient.ORACLE, limit)._query(

From 0db08498d9c1a38b02485ec987561dfdd53ea3dc Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 23 May 2024 17:44:23 +0100
Subject: [PATCH 90/92] Fix for grid arrow event handling when creating a
 column

---
 .../DataTable/modals/CreateEditColumn.svelte  | 479 +++++++++---------
 .../grid/overlays/KeyboardManager.svelte      |   1 +
 2 files changed, 242 insertions(+), 238 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 8583dbcab7..477ca1caa9 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -517,258 +517,261 @@
   })
 </script>
 
-<Layout noPadding gap="S">
-  {#if mounted}
-    <Input
-      value={editableColumn.name}
-      autofocus
-      on:input={e => {
-        if (
-          !uneditable &&
-          !(linkEditDisabled && editableColumn.type === LINK_TYPE)
-        ) {
-          editableColumn.name = e.target.value
-        }
-      }}
-      disabled={uneditable ||
-        (linkEditDisabled && editableColumn.type === LINK_TYPE)}
-      error={errors?.name}
-    />
-  {/if}
-  <Select
-    placeholder={null}
-    disabled={!typeEnabled}
-    bind:value={editableColumn.fieldId}
-    on:change={onHandleTypeChange}
-    options={allowedTypes}
-    getOptionLabel={field => field.name}
-    getOptionValue={field => field.fieldId}
-    getOptionIcon={field => field.icon}
-    isOptionEnabled={option => {
-      if (option.type === AUTO_TYPE) {
-        return availableAutoColumnKeys?.length > 0
-      }
-      return true
-    }}
-  />
-
-  {#if editableColumn.type === FieldType.STRING}
-    <Input
-      type="number"
-      label="Max Length"
-      bind:value={editableColumn.constraints.length.maximum}
-    />
-  {:else if editableColumn.type === FieldType.OPTIONS}
-    <OptionSelectDnD
-      bind:constraints={editableColumn.constraints}
-      bind:optionColors={editableColumn.optionColors}
-    />
-  {:else if editableColumn.type === FieldType.LONGFORM}
-    <div>
-      <div class="tooltip-alignment">
-        <Label size="M">Formatting</Label>
-        <AbsTooltip
-          position="top"
-          type="info"
-          text={"Rich text includes support for images, link"}
-        >
-          <Icon size="XS" name="InfoOutline" />
-        </AbsTooltip>
-      </div>
-
-      <Toggle
-        bind:value={editableColumn.useRichText}
-        text="Enable rich text support (markdown)"
+<span class="create-edit-column">
+  <Layout noPadding gap="S">
+    {#if mounted}
+      <Input
+        value={editableColumn.name}
+        autofocus
+        on:input={e => {
+          if (
+            !uneditable &&
+            !(linkEditDisabled && editableColumn.type === LINK_TYPE)
+          ) {
+            editableColumn.name = e.target.value
+          }
+        }}
+        disabled={uneditable ||
+          (linkEditDisabled && editableColumn.type === LINK_TYPE)}
+        error={errors?.name}
       />
-    </div>
-  {:else if editableColumn.type === FieldType.ARRAY}
-    <OptionSelectDnD
-      bind:constraints={editableColumn.constraints}
-      bind:optionColors={editableColumn.optionColors}
-    />
-  {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
-    <div class="split-label">
-      <div class="label-length">
-        <Label size="M">Earliest</Label>
-      </div>
-      <div class="input-length">
-        <DatePicker
-          bind:value={editableColumn.constraints.datetime.earliest}
-          enableTime={!editableColumn.dateOnly}
-          timeOnly={editableColumn.timeOnly}
-        />
-      </div>
-    </div>
-
-    <div class="split-label">
-      <div class="label-length">
-        <Label size="M">Latest</Label>
-      </div>
-      <div class="input-length">
-        <DatePicker
-          bind:value={editableColumn.constraints.datetime.latest}
-          enableTime={!editableColumn.dateOnly}
-          timeOnly={editableColumn.timeOnly}
-        />
-      </div>
-    </div>
-    {#if !editableColumn.timeOnly}
-      {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
-        <div>
-          <div class="row">
-            <Label>Time zones</Label>
-            <AbsTooltip
-              position="top"
-              type="info"
-              text={isCreating
-                ? null
-                : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
-            >
-              <Icon size="XS" name="InfoOutline" />
-            </AbsTooltip>
-          </div>
-          <Toggle
-            bind:value={editableColumn.ignoreTimezones}
-            text="Ignore time zones"
-          />
-        </div>
-      {/if}
-      <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
     {/if}
-  {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn}
-    <div class="split-label">
-      <div class="label-length">
-        <Label size="M">Min Value</Label>
-      </div>
-      <div class="input-length">
-        <Input
-          type="number"
-          bind:value={editableColumn.constraints.numericality
-            .greaterThanOrEqualTo}
-        />
-      </div>
-    </div>
-
-    <div class="split-label">
-      <div class="label-length">
-        <Label size="M">Max Value</Label>
-      </div>
-      <div class="input-length">
-        <Input
-          type="number"
-          bind:value={editableColumn.constraints.numericality.lessThanOrEqualTo}
-        />
-      </div>
-    </div>
-  {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn}
-    <RelationshipSelector
-      bind:relationshipPart1
-      bind:relationshipPart2
-      bind:relationshipTableIdPrimary
-      bind:relationshipTableIdSecondary
-      bind:editableColumn
-      {relationshipOpts1}
-      {relationshipOpts2}
-      {linkEditDisabled}
-      {tableOptions}
-      {errors}
+    <Select
+      placeholder={null}
+      disabled={!typeEnabled}
+      bind:value={editableColumn.fieldId}
+      on:change={onHandleTypeChange}
+      options={allowedTypes}
+      getOptionLabel={field => field.name}
+      getOptionValue={field => field.fieldId}
+      getOptionIcon={field => field.icon}
+      isOptionEnabled={option => {
+        if (option.type === AUTO_TYPE) {
+          return availableAutoColumnKeys?.length > 0
+        }
+        return true
+      }}
     />
-  {:else if editableColumn.type === FORMULA_TYPE}
-    {#if !table.sql}
+
+    {#if editableColumn.type === FieldType.STRING}
+      <Input
+        type="number"
+        label="Max Length"
+        bind:value={editableColumn.constraints.length.maximum}
+      />
+    {:else if editableColumn.type === FieldType.OPTIONS}
+      <OptionSelectDnD
+        bind:constraints={editableColumn.constraints}
+        bind:optionColors={editableColumn.optionColors}
+      />
+    {:else if editableColumn.type === FieldType.LONGFORM}
+      <div>
+        <div class="tooltip-alignment">
+          <Label size="M">Formatting</Label>
+          <AbsTooltip
+            position="top"
+            type="info"
+            text={"Rich text includes support for images, link"}
+          >
+            <Icon size="XS" name="InfoOutline" />
+          </AbsTooltip>
+        </div>
+
+        <Toggle
+          bind:value={editableColumn.useRichText}
+          text="Enable rich text support (markdown)"
+        />
+      </div>
+    {:else if editableColumn.type === FieldType.ARRAY}
+      <OptionSelectDnD
+        bind:constraints={editableColumn.constraints}
+        bind:optionColors={editableColumn.optionColors}
+      />
+    {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
       <div class="split-label">
         <div class="label-length">
-          <Label size="M">Formula Type</Label>
+          <Label size="M">Earliest</Label>
         </div>
         <div class="input-length">
-          <Select
-            bind:value={editableColumn.formulaType}
-            options={[
-              { label: "Dynamic", value: "dynamic" },
-              { label: "Static", value: "static" },
-            ]}
-            disabled={!isCreating}
-            getOptionLabel={option => option.label}
-            getOptionValue={option => option.value}
-            tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by,
-         while static formula are calculated when the row is saved."
+          <DatePicker
+            bind:value={editableColumn.constraints.datetime.earliest}
+            enableTime={!editableColumn.dateOnly}
+            timeOnly={editableColumn.timeOnly}
           />
         </div>
       </div>
-    {/if}
-    <div class="split-label">
-      <div class="label-length">
-        <Label size="M">Formula</Label>
-      </div>
-      <div class="input-length">
-        <ModalBindableInput
-          panel={ServerBindingPanel}
-          title="Formula"
-          value={editableColumn.formula}
-          on:change={e => {
-            editableColumn = {
-              ...editableColumn,
-              formula: e.detail,
-            }
-          }}
-          bindings={getBindings({ table })}
-          allowJS
-          context={rowGoldenSample}
-        />
-      </div>
-    </div>
-  {:else if editableColumn.type === JSON_TYPE}
-    <Button primary text on:click={openJsonSchemaEditor}
-      >Open schema editor</Button
-    >
-  {/if}
-  {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
-    <Select
-      label="Auto column type"
-      value={editableColumn.subtype}
-      on:change={e => (editableColumn.subtype = e.detail)}
-      options={Object.entries(autoColumnOptions)}
-      getOptionLabel={option => option[1].name}
-      getOptionValue={option => option[0]}
-      disabled={!availableAutoColumnKeys?.length || editableColumn.autocolumn}
-      error={errors?.subtype}
-    />
-  {/if}
 
-  {#if canBeRequired || canBeDisplay}
-    <div>
-      {#if canBeRequired}
-        <Toggle
-          value={required}
-          on:change={onChangeRequired}
-          disabled={primaryDisplay}
-          thin
-          text="Required"
-        />
+      <div class="split-label">
+        <div class="label-length">
+          <Label size="M">Latest</Label>
+        </div>
+        <div class="input-length">
+          <DatePicker
+            bind:value={editableColumn.constraints.datetime.latest}
+            enableTime={!editableColumn.dateOnly}
+            timeOnly={editableColumn.timeOnly}
+          />
+        </div>
+      </div>
+      {#if !editableColumn.timeOnly}
+        {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
+          <div>
+            <div class="row">
+              <Label>Time zones</Label>
+              <AbsTooltip
+                position="top"
+                type="info"
+                text={isCreating
+                  ? null
+                  : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
+              >
+                <Icon size="XS" name="InfoOutline" />
+              </AbsTooltip>
+            </div>
+            <Toggle
+              bind:value={editableColumn.ignoreTimezones}
+              text="Ignore time zones"
+            />
+          </div>
+        {/if}
+        <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
       {/if}
-    </div>
-  {/if}
-</Layout>
-
-<div class="action-buttons">
-  {#if !uneditable && originalName != null}
-    <Button quiet warning text on:click={confirmDelete}>Delete</Button>
-  {/if}
-  <Button secondary newStyles on:click={cancelEdit}>Cancel</Button>
-  <Button
-    disabled={invalid || savingColumn}
-    newStyles
-    cta
-    on:click={saveColumn}
-  >
-    {#if savingColumn}
-      <div class="save-loading">
-        <ProgressCircle overBackground={true} size="S" />
+    {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn}
+      <div class="split-label">
+        <div class="label-length">
+          <Label size="M">Min Value</Label>
+        </div>
+        <div class="input-length">
+          <Input
+            type="number"
+            bind:value={editableColumn.constraints.numericality
+              .greaterThanOrEqualTo}
+          />
+        </div>
       </div>
-    {:else}
-      Save
+
+      <div class="split-label">
+        <div class="label-length">
+          <Label size="M">Max Value</Label>
+        </div>
+        <div class="input-length">
+          <Input
+            type="number"
+            bind:value={editableColumn.constraints.numericality
+              .lessThanOrEqualTo}
+          />
+        </div>
+      </div>
+    {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn}
+      <RelationshipSelector
+        bind:relationshipPart1
+        bind:relationshipPart2
+        bind:relationshipTableIdPrimary
+        bind:relationshipTableIdSecondary
+        bind:editableColumn
+        {relationshipOpts1}
+        {relationshipOpts2}
+        {linkEditDisabled}
+        {tableOptions}
+        {errors}
+      />
+    {:else if editableColumn.type === FORMULA_TYPE}
+      {#if !table.sql}
+        <div class="split-label">
+          <div class="label-length">
+            <Label size="M">Formula Type</Label>
+          </div>
+          <div class="input-length">
+            <Select
+              bind:value={editableColumn.formulaType}
+              options={[
+                { label: "Dynamic", value: "dynamic" },
+                { label: "Static", value: "static" },
+              ]}
+              disabled={!isCreating}
+              getOptionLabel={option => option.label}
+              getOptionValue={option => option.value}
+              tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by,
+         while static formula are calculated when the row is saved."
+            />
+          </div>
+        </div>
+      {/if}
+      <div class="split-label">
+        <div class="label-length">
+          <Label size="M">Formula</Label>
+        </div>
+        <div class="input-length">
+          <ModalBindableInput
+            panel={ServerBindingPanel}
+            title="Formula"
+            value={editableColumn.formula}
+            on:change={e => {
+              editableColumn = {
+                ...editableColumn,
+                formula: e.detail,
+              }
+            }}
+            bindings={getBindings({ table })}
+            allowJS
+            context={rowGoldenSample}
+          />
+        </div>
+      </div>
+    {:else if editableColumn.type === JSON_TYPE}
+      <Button primary text on:click={openJsonSchemaEditor}
+        >Open schema editor</Button
+      >
     {/if}
-  </Button>
-</div>
+    {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
+      <Select
+        label="Auto column type"
+        value={editableColumn.subtype}
+        on:change={e => (editableColumn.subtype = e.detail)}
+        options={Object.entries(autoColumnOptions)}
+        getOptionLabel={option => option[1].name}
+        getOptionValue={option => option[0]}
+        disabled={!availableAutoColumnKeys?.length || editableColumn.autocolumn}
+        error={errors?.subtype}
+      />
+    {/if}
+
+    {#if canBeRequired || canBeDisplay}
+      <div>
+        {#if canBeRequired}
+          <Toggle
+            value={required}
+            on:change={onChangeRequired}
+            disabled={primaryDisplay}
+            thin
+            text="Required"
+          />
+        {/if}
+      </div>
+    {/if}
+  </Layout>
+
+  <div class="action-buttons">
+    {#if !uneditable && originalName != null}
+      <Button quiet warning text on:click={confirmDelete}>Delete</Button>
+    {/if}
+    <Button secondary newStyles on:click={cancelEdit}>Cancel</Button>
+    <Button
+      disabled={invalid || savingColumn}
+      newStyles
+      cta
+      on:click={saveColumn}
+    >
+      {#if savingColumn}
+        <div class="save-loading">
+          <ProgressCircle overBackground={true} size="S" />
+        </div>
+      {:else}
+        Save
+      {/if}
+    </Button>
+  </div>
+</span>
 <Modal bind:this={jsonSchemaModal}>
   <JSONSchemaModal
     schema={editableColumn.schema}
diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
index 78cb186f29..62a2b2f830 100644
--- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
@@ -24,6 +24,7 @@
     ".date-time-popover",
     "#builder-side-panel-container",
     "[data-grid-ignore]",
+    ".create-edit-column",
   ]
 
   // Global key listener which intercepts all key events

From 9edc0b7c3c8b5166463c690c086c2888f7cc342e Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 23 May 2024 17:45:01 +0100
Subject: [PATCH 91/92] Fix for tooltips in the Component tree in the builder

---
 .../builder/src/components/common/NavItem.svelte    | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/components/common/NavItem.svelte b/packages/builder/src/components/common/NavItem.svelte
index 5cc6db65a0..5a04907570 100644
--- a/packages/builder/src/components/common/NavItem.svelte
+++ b/packages/builder/src/components/common/NavItem.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { AbsTooltip, Icon } from "@budibase/bbui"
+  import { Icon, TooltipType, TooltipPosition } from "@budibase/bbui"
   import { createEventDispatcher, getContext } from "svelte"
   import { helpers } from "@budibase/shared-core"
   import { UserAvatars } from "@budibase/frontend-core"
@@ -114,9 +114,14 @@
       </div>
     {:else if icon}
       <div class="icon" class:right={rightAlignIcon}>
-        <AbsTooltip type="info" position="right" text={iconTooltip}>
-          <Icon color={iconColor} size="S" name={icon} />
-        </AbsTooltip>
+        <Icon
+          color={iconColor}
+          size="S"
+          name={icon}
+          tooltip={iconTooltip}
+          tooltipType={TooltipType.Info}
+          tooltipPosition={TooltipPosition.Right}
+        />
       </div>
     {/if}
     <div class="text" title={showTooltip ? text : null}>

From 5a289bd027e37b60780a543d4b91e26cd540aa8f Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 23 May 2024 18:06:08 +0100
Subject: [PATCH 92/92] Removed fix for grid keyboard events

---
 .../DataTable/modals/CreateEditColumn.svelte  | 475 +++++++++---------
 .../grid/overlays/KeyboardManager.svelte      |   1 -
 2 files changed, 236 insertions(+), 240 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 477ca1caa9..8583dbcab7 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -517,261 +517,258 @@
   })
 </script>
 
-<span class="create-edit-column">
-  <Layout noPadding gap="S">
-    {#if mounted}
-      <Input
-        value={editableColumn.name}
-        autofocus
-        on:input={e => {
-          if (
-            !uneditable &&
-            !(linkEditDisabled && editableColumn.type === LINK_TYPE)
-          ) {
-            editableColumn.name = e.target.value
-          }
-        }}
-        disabled={uneditable ||
-          (linkEditDisabled && editableColumn.type === LINK_TYPE)}
-        error={errors?.name}
-      />
-    {/if}
-    <Select
-      placeholder={null}
-      disabled={!typeEnabled}
-      bind:value={editableColumn.fieldId}
-      on:change={onHandleTypeChange}
-      options={allowedTypes}
-      getOptionLabel={field => field.name}
-      getOptionValue={field => field.fieldId}
-      getOptionIcon={field => field.icon}
-      isOptionEnabled={option => {
-        if (option.type === AUTO_TYPE) {
-          return availableAutoColumnKeys?.length > 0
+<Layout noPadding gap="S">
+  {#if mounted}
+    <Input
+      value={editableColumn.name}
+      autofocus
+      on:input={e => {
+        if (
+          !uneditable &&
+          !(linkEditDisabled && editableColumn.type === LINK_TYPE)
+        ) {
+          editableColumn.name = e.target.value
         }
-        return true
       }}
+      disabled={uneditable ||
+        (linkEditDisabled && editableColumn.type === LINK_TYPE)}
+      error={errors?.name}
     />
+  {/if}
+  <Select
+    placeholder={null}
+    disabled={!typeEnabled}
+    bind:value={editableColumn.fieldId}
+    on:change={onHandleTypeChange}
+    options={allowedTypes}
+    getOptionLabel={field => field.name}
+    getOptionValue={field => field.fieldId}
+    getOptionIcon={field => field.icon}
+    isOptionEnabled={option => {
+      if (option.type === AUTO_TYPE) {
+        return availableAutoColumnKeys?.length > 0
+      }
+      return true
+    }}
+  />
 
-    {#if editableColumn.type === FieldType.STRING}
-      <Input
-        type="number"
-        label="Max Length"
-        bind:value={editableColumn.constraints.length.maximum}
-      />
-    {:else if editableColumn.type === FieldType.OPTIONS}
-      <OptionSelectDnD
-        bind:constraints={editableColumn.constraints}
-        bind:optionColors={editableColumn.optionColors}
-      />
-    {:else if editableColumn.type === FieldType.LONGFORM}
-      <div>
-        <div class="tooltip-alignment">
-          <Label size="M">Formatting</Label>
-          <AbsTooltip
-            position="top"
-            type="info"
-            text={"Rich text includes support for images, link"}
-          >
-            <Icon size="XS" name="InfoOutline" />
-          </AbsTooltip>
-        </div>
+  {#if editableColumn.type === FieldType.STRING}
+    <Input
+      type="number"
+      label="Max Length"
+      bind:value={editableColumn.constraints.length.maximum}
+    />
+  {:else if editableColumn.type === FieldType.OPTIONS}
+    <OptionSelectDnD
+      bind:constraints={editableColumn.constraints}
+      bind:optionColors={editableColumn.optionColors}
+    />
+  {:else if editableColumn.type === FieldType.LONGFORM}
+    <div>
+      <div class="tooltip-alignment">
+        <Label size="M">Formatting</Label>
+        <AbsTooltip
+          position="top"
+          type="info"
+          text={"Rich text includes support for images, link"}
+        >
+          <Icon size="XS" name="InfoOutline" />
+        </AbsTooltip>
+      </div>
 
-        <Toggle
-          bind:value={editableColumn.useRichText}
-          text="Enable rich text support (markdown)"
+      <Toggle
+        bind:value={editableColumn.useRichText}
+        text="Enable rich text support (markdown)"
+      />
+    </div>
+  {:else if editableColumn.type === FieldType.ARRAY}
+    <OptionSelectDnD
+      bind:constraints={editableColumn.constraints}
+      bind:optionColors={editableColumn.optionColors}
+    />
+  {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Earliest</Label>
+      </div>
+      <div class="input-length">
+        <DatePicker
+          bind:value={editableColumn.constraints.datetime.earliest}
+          enableTime={!editableColumn.dateOnly}
+          timeOnly={editableColumn.timeOnly}
         />
       </div>
-    {:else if editableColumn.type === FieldType.ARRAY}
-      <OptionSelectDnD
-        bind:constraints={editableColumn.constraints}
-        bind:optionColors={editableColumn.optionColors}
-      />
-    {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
-      <div class="split-label">
-        <div class="label-length">
-          <Label size="M">Earliest</Label>
-        </div>
-        <div class="input-length">
-          <DatePicker
-            bind:value={editableColumn.constraints.datetime.earliest}
-            enableTime={!editableColumn.dateOnly}
-            timeOnly={editableColumn.timeOnly}
-          />
-        </div>
-      </div>
+    </div>
 
-      <div class="split-label">
-        <div class="label-length">
-          <Label size="M">Latest</Label>
-        </div>
-        <div class="input-length">
-          <DatePicker
-            bind:value={editableColumn.constraints.datetime.latest}
-            enableTime={!editableColumn.dateOnly}
-            timeOnly={editableColumn.timeOnly}
-          />
-        </div>
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Latest</Label>
       </div>
-      {#if !editableColumn.timeOnly}
-        {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
-          <div>
-            <div class="row">
-              <Label>Time zones</Label>
-              <AbsTooltip
-                position="top"
-                type="info"
-                text={isCreating
-                  ? null
-                  : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
-              >
-                <Icon size="XS" name="InfoOutline" />
-              </AbsTooltip>
-            </div>
-            <Toggle
-              bind:value={editableColumn.ignoreTimezones}
-              text="Ignore time zones"
-            />
+      <div class="input-length">
+        <DatePicker
+          bind:value={editableColumn.constraints.datetime.latest}
+          enableTime={!editableColumn.dateOnly}
+          timeOnly={editableColumn.timeOnly}
+        />
+      </div>
+    </div>
+    {#if !editableColumn.timeOnly}
+      {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
+        <div>
+          <div class="row">
+            <Label>Time zones</Label>
+            <AbsTooltip
+              position="top"
+              type="info"
+              text={isCreating
+                ? null
+                : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
+            >
+              <Icon size="XS" name="InfoOutline" />
+            </AbsTooltip>
           </div>
-        {/if}
-        <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
-      {/if}
-    {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn}
-      <div class="split-label">
-        <div class="label-length">
-          <Label size="M">Min Value</Label>
-        </div>
-        <div class="input-length">
-          <Input
-            type="number"
-            bind:value={editableColumn.constraints.numericality
-              .greaterThanOrEqualTo}
-          />
-        </div>
-      </div>
-
-      <div class="split-label">
-        <div class="label-length">
-          <Label size="M">Max Value</Label>
-        </div>
-        <div class="input-length">
-          <Input
-            type="number"
-            bind:value={editableColumn.constraints.numericality
-              .lessThanOrEqualTo}
-          />
-        </div>
-      </div>
-    {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn}
-      <RelationshipSelector
-        bind:relationshipPart1
-        bind:relationshipPart2
-        bind:relationshipTableIdPrimary
-        bind:relationshipTableIdSecondary
-        bind:editableColumn
-        {relationshipOpts1}
-        {relationshipOpts2}
-        {linkEditDisabled}
-        {tableOptions}
-        {errors}
-      />
-    {:else if editableColumn.type === FORMULA_TYPE}
-      {#if !table.sql}
-        <div class="split-label">
-          <div class="label-length">
-            <Label size="M">Formula Type</Label>
-          </div>
-          <div class="input-length">
-            <Select
-              bind:value={editableColumn.formulaType}
-              options={[
-                { label: "Dynamic", value: "dynamic" },
-                { label: "Static", value: "static" },
-              ]}
-              disabled={!isCreating}
-              getOptionLabel={option => option.label}
-              getOptionValue={option => option.value}
-              tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by,
-         while static formula are calculated when the row is saved."
-            />
-          </div>
-        </div>
-      {/if}
-      <div class="split-label">
-        <div class="label-length">
-          <Label size="M">Formula</Label>
-        </div>
-        <div class="input-length">
-          <ModalBindableInput
-            panel={ServerBindingPanel}
-            title="Formula"
-            value={editableColumn.formula}
-            on:change={e => {
-              editableColumn = {
-                ...editableColumn,
-                formula: e.detail,
-              }
-            }}
-            bindings={getBindings({ table })}
-            allowJS
-            context={rowGoldenSample}
-          />
-        </div>
-      </div>
-    {:else if editableColumn.type === JSON_TYPE}
-      <Button primary text on:click={openJsonSchemaEditor}
-        >Open schema editor</Button
-      >
-    {/if}
-    {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
-      <Select
-        label="Auto column type"
-        value={editableColumn.subtype}
-        on:change={e => (editableColumn.subtype = e.detail)}
-        options={Object.entries(autoColumnOptions)}
-        getOptionLabel={option => option[1].name}
-        getOptionValue={option => option[0]}
-        disabled={!availableAutoColumnKeys?.length || editableColumn.autocolumn}
-        error={errors?.subtype}
-      />
-    {/if}
-
-    {#if canBeRequired || canBeDisplay}
-      <div>
-        {#if canBeRequired}
           <Toggle
-            value={required}
-            on:change={onChangeRequired}
-            disabled={primaryDisplay}
-            thin
-            text="Required"
+            bind:value={editableColumn.ignoreTimezones}
+            text="Ignore time zones"
           />
-        {/if}
+        </div>
+      {/if}
+      <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
+    {/if}
+  {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn}
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Min Value</Label>
+      </div>
+      <div class="input-length">
+        <Input
+          type="number"
+          bind:value={editableColumn.constraints.numericality
+            .greaterThanOrEqualTo}
+        />
+      </div>
+    </div>
+
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Max Value</Label>
+      </div>
+      <div class="input-length">
+        <Input
+          type="number"
+          bind:value={editableColumn.constraints.numericality.lessThanOrEqualTo}
+        />
+      </div>
+    </div>
+  {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn}
+    <RelationshipSelector
+      bind:relationshipPart1
+      bind:relationshipPart2
+      bind:relationshipTableIdPrimary
+      bind:relationshipTableIdSecondary
+      bind:editableColumn
+      {relationshipOpts1}
+      {relationshipOpts2}
+      {linkEditDisabled}
+      {tableOptions}
+      {errors}
+    />
+  {:else if editableColumn.type === FORMULA_TYPE}
+    {#if !table.sql}
+      <div class="split-label">
+        <div class="label-length">
+          <Label size="M">Formula Type</Label>
+        </div>
+        <div class="input-length">
+          <Select
+            bind:value={editableColumn.formulaType}
+            options={[
+              { label: "Dynamic", value: "dynamic" },
+              { label: "Static", value: "static" },
+            ]}
+            disabled={!isCreating}
+            getOptionLabel={option => option.label}
+            getOptionValue={option => option.value}
+            tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by,
+         while static formula are calculated when the row is saved."
+          />
+        </div>
       </div>
     {/if}
-  </Layout>
-
-  <div class="action-buttons">
-    {#if !uneditable && originalName != null}
-      <Button quiet warning text on:click={confirmDelete}>Delete</Button>
-    {/if}
-    <Button secondary newStyles on:click={cancelEdit}>Cancel</Button>
-    <Button
-      disabled={invalid || savingColumn}
-      newStyles
-      cta
-      on:click={saveColumn}
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Formula</Label>
+      </div>
+      <div class="input-length">
+        <ModalBindableInput
+          panel={ServerBindingPanel}
+          title="Formula"
+          value={editableColumn.formula}
+          on:change={e => {
+            editableColumn = {
+              ...editableColumn,
+              formula: e.detail,
+            }
+          }}
+          bindings={getBindings({ table })}
+          allowJS
+          context={rowGoldenSample}
+        />
+      </div>
+    </div>
+  {:else if editableColumn.type === JSON_TYPE}
+    <Button primary text on:click={openJsonSchemaEditor}
+      >Open schema editor</Button
     >
-      {#if savingColumn}
-        <div class="save-loading">
-          <ProgressCircle overBackground={true} size="S" />
-        </div>
-      {:else}
-        Save
+  {/if}
+  {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
+    <Select
+      label="Auto column type"
+      value={editableColumn.subtype}
+      on:change={e => (editableColumn.subtype = e.detail)}
+      options={Object.entries(autoColumnOptions)}
+      getOptionLabel={option => option[1].name}
+      getOptionValue={option => option[0]}
+      disabled={!availableAutoColumnKeys?.length || editableColumn.autocolumn}
+      error={errors?.subtype}
+    />
+  {/if}
+
+  {#if canBeRequired || canBeDisplay}
+    <div>
+      {#if canBeRequired}
+        <Toggle
+          value={required}
+          on:change={onChangeRequired}
+          disabled={primaryDisplay}
+          thin
+          text="Required"
+        />
       {/if}
-    </Button>
-  </div>
-</span>
+    </div>
+  {/if}
+</Layout>
+
+<div class="action-buttons">
+  {#if !uneditable && originalName != null}
+    <Button quiet warning text on:click={confirmDelete}>Delete</Button>
+  {/if}
+  <Button secondary newStyles on:click={cancelEdit}>Cancel</Button>
+  <Button
+    disabled={invalid || savingColumn}
+    newStyles
+    cta
+    on:click={saveColumn}
+  >
+    {#if savingColumn}
+      <div class="save-loading">
+        <ProgressCircle overBackground={true} size="S" />
+      </div>
+    {:else}
+      Save
+    {/if}
+  </Button>
+</div>
 <Modal bind:this={jsonSchemaModal}>
   <JSONSchemaModal
     schema={editableColumn.schema}
diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
index 62a2b2f830..78cb186f29 100644
--- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
@@ -24,7 +24,6 @@
     ".date-time-popover",
     "#builder-side-panel-container",
     "[data-grid-ignore]",
-    ".create-edit-column",
   ]
 
   // Global key listener which intercepts all key events