Merge branch 'master' into grid-ux-improvements

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.27.2",
+  "version": "2.27.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",
@@ -22,4 +22,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}

packages/backend-core/src/docIds/ids.ts

@@ -18,6 +18,14 @@ export const generateAppID = (tenantId?: string | null) => {
   return `${id}${newid()}`
 }
 
+/**
+ * Generates a new table ID.
+ * @returns The new table ID which the table doc can be stored under.
+ */
+export function generateTableID() {
+  return `${DocumentType.TABLE}${SEPARATOR}${newid()}`
+}
+
 /**
  * Gets a new row ID for the specified table.
  * @param tableId The table which the row is being created for.

packages/bbui/src/helpers.js

@@ -170,7 +170,8 @@ export const stringifyDate = (
     const offset = referenceDate.getTimezoneOffset() * 60000
     const date = new Date(value.valueOf() - offset)
     if (timeOnly) {
-      return date.toISOString().slice(11, 19)
+      // Extract HH:mm
+      return date.toISOString().slice(11, 16)
     }
     return date.toISOString().slice(0, -1)
   }

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -586,13 +586,17 @@
       bind:constraints={editableColumn.constraints}
       bind:optionColors={editableColumn.optionColors}
     />
-  {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn}
+  {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
         <Label size="M">Earliest</Label>
       </div>
       <div class="input-length">
-        <DatePicker bind:value={editableColumn.constraints.datetime.earliest} />
+        <DatePicker
+          bind:value={editableColumn.constraints.datetime.earliest}
+          enableTime={!editableColumn.dateOnly}
+          timeOnly={editableColumn.timeOnly}
+        />
       </div>
     </div>
 
@@ -601,30 +605,36 @@
         <Label size="M">Latest</Label>
       </div>
       <div class="input-length">
-        <DatePicker bind:value={editableColumn.constraints.datetime.latest} />
-      </div>
-    </div>
-    {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
-      <div>
-        <div class="row">
-          <Label>Time zones</Label>
-          <AbsTooltip
-            position="top"
-            type="info"
-            text={isCreating
-              ? null
-              : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
-          >
-            <Icon size="XS" name="InfoOutline" />
-          </AbsTooltip>
-        </div>
-        <Toggle
-          bind:value={editableColumn.ignoreTimezones}
-          text="Ignore time zones"
+        <DatePicker
+          bind:value={editableColumn.constraints.datetime.latest}
+          enableTime={!editableColumn.dateOnly}
+          timeOnly={editableColumn.timeOnly}
         />
       </div>
+    </div>
+    {#if !editableColumn.timeOnly}
+      {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
+        <div>
+          <div class="row">
+            <Label>Time zones</Label>
+            <AbsTooltip
+              position="top"
+              type="info"
+              text={isCreating
+                ? null
+                : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
+            >
+              <Icon size="XS" name="InfoOutline" />
+            </AbsTooltip>
+          </div>
+          <Toggle
+            bind:value={editableColumn.ignoreTimezones}
+            text="Ignore time zones"
+          />
+        </div>
+      {/if}
+      <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
     {/if}
-    <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
   {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">

packages/builder/src/components/common/NavItem.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { AbsTooltip, Icon } from "@budibase/bbui"
+  import { Icon, TooltipType, TooltipPosition } from "@budibase/bbui"
   import { createEventDispatcher, getContext } from "svelte"
   import { helpers } from "@budibase/shared-core"
   import { UserAvatars } from "@budibase/frontend-core"
@@ -114,9 +114,14 @@
       </div>
     {:else if icon}
       <div class="icon" class:right={rightAlignIcon}>
-        <AbsTooltip type="info" position="right" text={iconTooltip}>
-          <Icon color={iconColor} size="S" name={icon} />
-        </AbsTooltip>
+        <Icon
+          color={iconColor}
+          size="S"
+          name={icon}
+          tooltip={iconTooltip}
+          tooltipType={TooltipType.Info}
+          tooltipPosition={TooltipPosition.Right}
+        />
       </div>
     {/if}
     <div class="text" title={showTooltip ? text : null}>

packages/server/src/api/controllers/view/exporters.ts

@@ -57,5 +57,5 @@ export function isFormat(format: any): format is Format {
 }
 
 export function parseCsvExport<T>(value: string) {
-  return JSON.parse(value?.replace(/'/g, '"')) as T
+  return JSON.parse(value) as T
 }

packages/server/src/api/routes/tests/search.spec.ts

@@ -1,6 +1,6 @@
 import { tableForDatasource } from "../../../tests/utilities/structures"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
-import { db as dbCore } from "@budibase/backend-core"
+import { db as dbCore, utils } from "@budibase/backend-core"
 
 import * as setup from "./utilities"
 import {
@@ -87,21 +87,67 @@ describe.each([
   class SearchAssertion {
     constructor(private readonly query: RowSearchParams) {}
 
-    private popRow(expectedRow: any, foundRows: any[]) {
-      const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
+    // We originally used _.isMatch to compare rows, but found that when
+    // comparing arrays it would return true if the source array was a subset of
+    // the target array. This would sometimes create false matches. This
+    // function is a more strict version of _.isMatch that only returns true if
+    // the source array is an exact match of the target.
+    //
+    // _.isMatch("100", "1") also returns true which is not what we want.
+    private isMatch<T extends Record<string, any>>(expected: T, found: T) {
+      if (!expected) {
+        throw new Error("Expected is undefined")
+      }
+      if (!found) {
+        return false
+      }
+
+      for (const key of Object.keys(expected)) {
+        if (Array.isArray(expected[key])) {
+          if (!Array.isArray(found[key])) {
+            return false
+          }
+          if (expected[key].length !== found[key].length) {
+            return false
+          }
+          if (!_.isMatch(found[key], expected[key])) {
+            return false
+          }
+        } else if (typeof expected[key] === "object") {
+          if (!this.isMatch(expected[key], found[key])) {
+            return false
+          }
+        } else {
+          if (expected[key] !== found[key]) {
+            return false
+          }
+        }
+      }
+      return true
+    }
+
+    // This function exists to ensure that the same row is not matched twice.
+    // When a row gets matched, we make sure to remove it from the list of rows
+    // we're matching against.
+    private popRow<T extends { [key: string]: any }>(
+      expectedRow: T,
+      foundRows: T[]
+    ): NonNullable<T> {
+      const row = foundRows.find(row => this.isMatch(expectedRow, row))
       if (!row) {
         const fields = Object.keys(expectedRow)
         // To make the error message more readable, we only include the fields
         // that are present in the expected row.
         const searchedObjects = foundRows.map(row => _.pick(row, fields))
         throw new Error(
-          `Failed to find row: ${JSON.stringify(
-            expectedRow
-          )} in ${JSON.stringify(searchedObjects)}`
+          `Failed to find row:\n\n${JSON.stringify(
+            expectedRow,
+            null,
+            2
+          )}\n\nin\n\n${JSON.stringify(searchedObjects, null, 2)}`
         )
       }
 
-      // Ensuring the same row is not matched twice
       foundRows.splice(foundRows.indexOf(row), 1)
       return row
     }
@@ -1055,6 +1101,7 @@ describe.each([
       describe("notEqual", () => {
         it("successfully finds a row", () =>
           expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([
+            { timeid: NULL_TIME__ID },
             { time: "10:45:00" },
             { time: "12:00:00" },
             { time: "15:30:00" },
@@ -1064,6 +1111,7 @@ describe.each([
         it("return all when requesting non-existing", () =>
           expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly(
             [
+              { timeid: NULL_TIME__ID },
               { time: "10:00:00" },
               { time: "10:45:00" },
               { time: "12:00:00" },
@@ -1530,14 +1578,169 @@ describe.each([
       await createRows([{ "1:name": "bar" }, { "1:name": "foo" }])
     })
 
+    it("successfully finds a row", () =>
+      expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([
+        { "1:name": "bar" },
+      ]))
+
+    it("fails to find nonexistent row", () =>
+      expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing())
+  })
+
+  describe("user", () => {
+    let user1: User
+    let user2: User
+
+    beforeAll(async () => {
+      user1 = await config.createUser({ _id: `us_${utils.newid()}` })
+      user2 = await config.createUser({ _id: `us_${utils.newid()}` })
+
+      table = await createTable({
+        user: {
+          name: "user",
+          type: FieldType.BB_REFERENCE_SINGLE,
+          subtype: BBReferenceFieldSubType.USER,
+        },
+      })
+
+      await createRows([
+        { user: JSON.stringify(user1) },
+        { user: JSON.stringify(user2) },
+        { user: null },
+      ])
+    })
+
     describe("equal", () => {
       it("successfully finds a row", () =>
-        expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([
-          { "1:name": "bar" },
+        expectQuery({ equal: { user: user1._id } }).toContainExactly([
+          { user: { _id: user1._id } },
         ]))
 
       it("fails to find nonexistent row", () =>
-        expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing())
+        expectQuery({ equal: { user: "us_none" } }).toFindNothing())
+    })
+
+    describe("notEqual", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ notEqual: { user: user1._id } }).toContainExactly([
+          { user: { _id: user2._id } },
+          {},
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([
+          { user: { _id: user1._id } },
+          { user: { _id: user2._id } },
+          {},
+        ]))
+    })
+
+    describe("oneOf", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([
+          { user: { _id: user1._id } },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing())
+    })
+
+    describe("empty", () => {
+      it("finds empty rows", () =>
+        expectQuery({ empty: { user: null } }).toContainExactly([{}]))
+    })
+
+    describe("notEmpty", () => {
+      it("finds non-empty rows", () =>
+        expectQuery({ notEmpty: { user: null } }).toContainExactly([
+          { user: { _id: user1._id } },
+          { user: { _id: user2._id } },
+        ]))
+    })
+  })
+
+  describe("multi user", () => {
+    let user1: User
+    let user2: User
+
+    beforeAll(async () => {
+      user1 = await config.createUser({ _id: `us_${utils.newid()}` })
+      user2 = await config.createUser({ _id: `us_${utils.newid()}` })
+
+      table = await createTable({
+        users: {
+          name: "users",
+          type: FieldType.BB_REFERENCE,
+          subtype: BBReferenceFieldSubType.USER,
+          constraints: { type: "array" },
+        },
+        number: {
+          name: "number",
+          type: FieldType.NUMBER,
+        },
+      })
+
+      await createRows([
+        { number: 1, users: JSON.stringify([user1]) },
+        { number: 2, users: JSON.stringify([user2]) },
+        { number: 3, users: JSON.stringify([user1, user2]) },
+        { number: 4, users: JSON.stringify([]) },
+      ])
+    })
+
+    describe("contains", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ contains: { users: [user1._id] } }).toContainExactly([
+          { users: [{ _id: user1._id }] },
+          { users: [{ _id: user1._id }, { _id: user2._id }] },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ contains: { users: ["us_none"] } }).toFindNothing())
+    })
+
+    describe("notContains", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([
+          { users: [{ _id: user2._id }] },
+          {},
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ notContains: { users: ["us_none"] } }).toContainExactly([
+          { users: [{ _id: user1._id }] },
+          { users: [{ _id: user2._id }] },
+          { users: [{ _id: user1._id }, { _id: user2._id }] },
+          {},
+        ]))
+    })
+
+    describe("containsAny", () => {
+      it("successfully finds rows", () =>
+        expectQuery({
+          containsAny: { users: [user1._id, user2._id] },
+        }).toContainExactly([
+          { users: [{ _id: user1._id }] },
+          { users: [{ _id: user2._id }] },
+          { users: [{ _id: user1._id }, { _id: user2._id }] },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing())
+    })
+
+    describe("multi-column equals", () => {
+      it("successfully finds a row", () =>
+        expectQuery({
+          equal: { number: 1 },
+          contains: { users: [user1._id] },
+        }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({
+          equal: { number: 2 },
+          contains: { users: [user1._id] },
+        }).toFindNothing())
     })
   })
 

packages/server/src/db/utils.ts

@@ -77,7 +77,7 @@ export function getTableParams(tableId?: Optional, otherProps = {}) {
  * @returns The new table ID which the table doc can be stored under.
  */
 export function generateTableID() {
-  return `${DocumentType.TABLE}${SEPARATOR}${newid()}`
+  return dbCore.generateTableID()
 }
 
 /**

packages/server/src/integration-test/mysql.spec.ts

@@ -281,4 +281,40 @@ describe("mysql integrations", () => {
       ])
     })
   })
+
+  describe("POST /api/datasources/:datasourceId/schema", () => {
+    let tableName: string
+
+    beforeEach(async () => {
+      tableName = uniqueTableName()
+    })
+
+    afterEach(async () => {
+      await rawQuery(rawDatasource, `DROP TABLE IF EXISTS \`${tableName}\``)
+    })
+
+    it("recognises enum columns as options", async () => {
+      const enumColumnName = "status"
+
+      const createTableQuery = `
+        CREATE TABLE \`${tableName}\` (
+          \`order_id\` INT AUTO_INCREMENT PRIMARY KEY,
+          \`customer_name\` VARCHAR(100) NOT NULL,
+          \`${enumColumnName}\` ENUM('pending', 'processing', 'shipped', 'delivered', 'cancelled')
+        );
+      `
+
+      await rawQuery(rawDatasource, createTableQuery)
+
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${datasource._id}/schema`
+      )
+
+      const table = response.body.datasource.entities[tableName]
+
+      expect(table).toBeDefined()
+      expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS)
+    })
+  })
 })

packages/server/src/integration-test/postgres.spec.ts

@@ -1122,6 +1122,37 @@ describe("postgres integrations", () => {
         [tableName]: "Table contains invalid columns.",
       })
     })
+
+    it("recognises enum columns as options", async () => {
+      const tableName = `orders_${generator
+        .guid()
+        .replaceAll("-", "")
+        .substring(0, 6)}`
+      const enumColumnName = "status"
+
+      await rawQuery(
+        rawDatasource,
+        `
+        CREATE TYPE order_status AS ENUM ('pending', 'processing', 'shipped', 'delivered', 'cancelled');
+        
+        CREATE TABLE ${tableName} (
+          order_id SERIAL PRIMARY KEY,
+          customer_name VARCHAR(100) NOT NULL,
+          ${enumColumnName} order_status
+        );
+      `
+      )
+
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${datasource._id}/schema`
+      )
+
+      const table = response.body.datasource.entities[tableName]
+
+      expect(table).toBeDefined()
+      expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS)
+    })
   })
 
   describe("Integration compatibility with postgres search_path", () => {

packages/server/src/integrations/base/sql.ts

@@ -231,8 +231,7 @@ class InternalBuilder {
     }
 
     const contains = (mode: object, any: boolean = false) => {
-      const fnc = allOr ? "orWhere" : "where"
-      const rawFnc = `${fnc}Raw`
+      const rawFnc = allOr ? "orWhereRaw" : "whereRaw"
       const not = mode === filters?.notContains ? "NOT " : ""
       function stringifyArray(value: Array<any>, quoteStyle = '"'): string {
         for (let i in value) {
@@ -245,24 +244,24 @@ class InternalBuilder {
       if (this.client === SqlClient.POSTGRES) {
         iterate(mode, (key: string, value: Array<any>) => {
           const wrap = any ? "" : "'"
-          const containsOp = any ? "\\?| array" : "@>"
+          const op = any ? "\\?| array" : "@>"
           const fieldNames = key.split(/\./g)
-          const tableName = fieldNames[0]
-          const columnName = fieldNames[1]
-          // @ts-ignore
+          const table = fieldNames[0]
+          const col = fieldNames[1]
           query = query[rawFnc](
-            `${not}"${tableName}"."${columnName}"::jsonb ${containsOp} ${wrap}${stringifyArray(
+            `${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray(
               value,
               any ? "'" : '"'
-            )}${wrap}`
+            )}${wrap}, FALSE)`
           )
         })
       } else if (this.client === SqlClient.MY_SQL) {
         const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
         iterate(mode, (key: string, value: Array<any>) => {
-          // @ts-ignore
           query = query[rawFnc](
-            `${not}${jsonFnc}(${key}, '${stringifyArray(value)}')`
+            `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray(
+              value
+            )}'), FALSE)`
           )
         })
       } else {
@@ -277,7 +276,7 @@ class InternalBuilder {
             }
             statement +=
               (statement ? andOr : "") +
-              `LOWER(${likeKey(this.client, key)}) LIKE ?`
+              `COALESCE(LOWER(${likeKey(this.client, key)}), '') LIKE ?`
           }
 
           if (statement === "") {
@@ -342,14 +341,34 @@ class InternalBuilder {
     }
     if (filters.equal) {
       iterate(filters.equal, (key, value) => {
-        const fnc = allOr ? "orWhere" : "where"
-        query = query[fnc]({ [key]: value })
+        const fnc = allOr ? "orWhereRaw" : "whereRaw"
+        if (this.client === SqlClient.MS_SQL) {
+          query = query[fnc](
+            `CASE WHEN ${likeKey(this.client, key)} = ? THEN 1 ELSE 0 END = 1`,
+            [value]
+          )
+        } else {
+          query = query[fnc](
+            `COALESCE(${likeKey(this.client, key)} = ?, FALSE)`,
+            [value]
+          )
+        }
       })
     }
     if (filters.notEqual) {
       iterate(filters.notEqual, (key, value) => {
-        const fnc = allOr ? "orWhereNot" : "whereNot"
-        query = query[fnc]({ [key]: value })
+        const fnc = allOr ? "orWhereRaw" : "whereRaw"
+        if (this.client === SqlClient.MS_SQL) {
+          query = query[fnc](
+            `CASE WHEN ${likeKey(this.client, key)} = ? THEN 1 ELSE 0 END = 0`,
+            [value]
+          )
+        } else {
+          query = query[fnc](
+            `COALESCE(${likeKey(this.client, key)} != ?, TRUE)`,
+            [value]
+          )
+        }
       })
     }
     if (filters.empty) {

packages/server/src/integrations/postgres.ts

@@ -329,14 +329,12 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
 
       // Fetch enum values
       const enumsResponse = await this.client.query(this.ENUM_VALUES())
+      // output array, allows for more than 1 single-select to be used at a time
       const enumValues = enumsResponse.rows?.reduce((acc, row) => {
-        if (!acc[row.typname]) {
-          return {
-            [row.typname]: [row.enumlabel],
-          }
+        return {
+          ...acc,
+          [row.typname]: [...(acc[row.typname] || []), row.enumlabel],
         }
-        acc[row.typname].push(row.enumlabel)
-        return acc
       }, {})
 
       for (let column of columnsResponse.rows) {

packages/server/src/integrations/tests/sql.spec.ts

@@ -189,7 +189,7 @@ describe("SQL query builder", () => {
     )
     expect(query).toEqual({
       bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit],
-      sql: `select * from (select * from (select * from "test" where (LOWER("test"."age") LIKE :1 AND LOWER("test"."age") LIKE :2) and (LOWER("test"."name") LIKE :3 AND LOWER("test"."name") LIKE :4)) where rownum <= :5) "test"`,
+      sql: `select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4)) where rownum <= :5) "test"`,
     })
 
     query = new Sql(SqlClient.ORACLE, limit)._query(

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -77,7 +77,7 @@ describe("Captures of real examples", () => {
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
               from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" 
               left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
-              left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 
+              left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where COALESCE("b"."taskname" = $2, FALSE)
               order by "a"."productname" asc nulls first limit $3`),
       })
     })
@@ -137,12 +137,12 @@ describe("Captures of real examples", () => {
              "c"."city" as "c.city", "c"."lastname" as "c.lastname", "c"."year" as "c.year", "c"."firstname" as "c.firstname", 
              "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", 
              "c"."city" as "c.city", "c"."lastname" as "c.lastname" 
-             from (select * from "tasks" as "a" where not "a"."completed" = $1 
+             from (select * from "tasks" as "a" where COALESCE("a"."completed" != $1, TRUE)
              order by "a"."taskname" asc nulls first limit $2) as "a" 
              left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" 
              left join "products" as "b" on "b"."productid" = "d"."productid" 
              left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" 
-             where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc nulls first limit $6`),
+             where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE) order by "a"."taskname" asc nulls first limit $6`),
       })
     })
   })
@@ -154,7 +154,7 @@ describe("Captures of real examples", () => {
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
         sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
-             "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`),
+             "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`),
       })
     })
 
@@ -164,7 +164,7 @@ describe("Captures of real examples", () => {
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
         sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
-             "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`),
+             "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`),
       })
     })
   })
@@ -175,8 +175,9 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: ["ddd", ""],
-        sql: multiline(`delete from "compositetable" as "a" where "a"."keypartone" = $1 and "a"."keyparttwo" = $2 
-             returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`),
+        sql: multiline(`delete from "compositetable" as "a" 
+          where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
+          returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`),
       })
     })
   })
@@ -197,7 +198,7 @@ describe("Captures of real examples", () => {
         returningQuery = input
       }, queryJson)
       expect(returningQuery).toEqual({
-        sql: "select * from (select top (@p0) * from [people] where [people].[name] = @p1 and [people].[age] = @p2 order by [people].[name] asc) as [people]",
+        sql: "select * from (select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]",
         bindings: [1, "Test", 22],
       })
     })

packages/server/src/integrations/utils/utils.ts

@@ -102,6 +102,7 @@ const SQL_OPTIONS_TYPE_MAP: Record<string, PrimitiveTypes> = {
 const SQL_MISC_TYPE_MAP: Record<string, PrimitiveTypes> = {
   json: FieldType.JSON,
   bigint: FieldType.BIGINT,
+  enum: FieldType.OPTIONS,
 }
 
 const SQL_TYPE_MAP: Record<string, PrimitiveTypes> = {

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -171,7 +171,8 @@ export async function search(
       sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
 
       const db = context.getAppDB()
-      return await db.sql<Row>(sql, bindings)
+      const rows = await db.sql<Row>(sql, bindings)
+      return rows
     })
 
     // process from the format of tableId.column to expected format

packages/server/src/sdk/app/rows/tests/utils.spec.ts

@@ -0,0 +1,335 @@
+import dayjs from "dayjs"
+import {
+  FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
+  Table,
+  TableSourceType,
+} from "@budibase/types"
+import { generateTableID } from "../../../../db/utils"
+import { validate } from "../utils"
+import { generator } from "@budibase/backend-core/tests"
+
+describe("validate", () => {
+  const hour = () => generator.hour().toString().padStart(2, "0")
+  const minute = () => generator.minute().toString().padStart(2, "0")
+  const second = minute
+
+  describe("time only", () => {
+    const getTable = (): Table => ({
+      type: "table",
+      _id: generateTableID(),
+      name: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+      schema: {
+        time: {
+          name: "time",
+          type: FieldType.DATETIME,
+          timeOnly: true,
+        },
+      },
+    })
+
+    it("should accept empty values", async () => {
+      const row = {}
+      const table = getTable()
+      const output = await validate({ table, tableId: table._id!, row })
+      expect(output.valid).toBe(true)
+      expect(output.errors).toEqual({})
+    })
+
+    it("should accept valid times with HH:mm format", async () => {
+      const row = {
+        time: `${hour()}:${minute()}`,
+      }
+      const table = getTable()
+      const output = await validate({ table, tableId: table._id!, row })
+      expect(output.valid).toBe(true)
+    })
+
+    it("should accept valid times with HH:mm:ss format", async () => {
+      const row = {
+        time: `${hour()}:${minute()}:${second()}`,
+      }
+      const table = getTable()
+      const output = await validate({ table, tableId: table._id!, row })
+      expect(output.valid).toBe(true)
+    })
+
+    it.each([
+      ["ISO datetimes", generator.date().toISOString()],
+      ["random values", generator.word()],
+    ])("should reject %s", async (_, time) => {
+      const row = {
+        time,
+      }
+      const table = getTable()
+      table.schema.time.constraints = {
+        presence: true,
+      }
+      const output = await validate({ table, tableId: table._id!, row })
+      expect(output.valid).toBe(false)
+      expect(output.errors).toEqual({ time: ['"time" is not a valid time'] })
+    })
+
+    describe("time constraints", () => {
+      describe("earliest only", () => {
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+          datetime: {
+            earliest: "10:00",
+            latest: "",
+          },
+        }
+
+        it.each([
+          "10:00",
+          "15:00",
+          `10:${minute()}`,
+          "12:34",
+          `${generator.integer({ min: 11, max: 23 })}:${minute()}`,
+        ])("should accept values after config value (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(true)
+        })
+
+        it.each([
+          "09:59:59",
+          `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
+        ])("should reject values before config value (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no earlier than 10:00"],
+          })
+        })
+      })
+
+      describe("latest only", () => {
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+          datetime: {
+            earliest: "",
+            latest: "15:16:17",
+          },
+        }
+
+        it.each([
+          "15:16:17",
+          "15:16",
+          "15:00",
+          `${generator.integer({ min: 0, max: 12 })}:${minute()}`,
+        ])("should accept values before config value (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(true)
+        })
+
+        it.each([
+          "15:16:18",
+          `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
+        ])("should reject values after config value (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no later than 15:16:17"],
+          })
+        })
+      })
+
+      describe("range", () => {
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+          datetime: {
+            earliest: "10:00",
+            latest: "15:00",
+          },
+        }
+
+        it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])(
+          "should accept values in range (%s)",
+          async time => {
+            const row = { time }
+            const output = await validate({ table, tableId: table._id!, row })
+            expect(output.valid).toBe(true)
+          }
+        )
+
+        it.each([
+          "9:59:50",
+          `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
+        ])("should reject values before range (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no earlier than 10:00"],
+          })
+        })
+
+        it.each([
+          "15:00:01",
+          `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
+        ])("should reject values after range (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no later than 15:00"],
+          })
+        })
+
+        describe("range crossing midnight", () => {
+          const table = getTable()
+          table.schema.time.constraints = {
+            presence: true,
+            datetime: {
+              earliest: "15:00",
+              latest: "10:00",
+            },
+          }
+
+          it.each(["10:00", "15:00", `9:${minute()}`, "16:34", "00:00"])(
+            "should accept values in range (%s)",
+            async time => {
+              const row = { time }
+              const output = await validate({ table, tableId: table._id!, row })
+              expect(output.valid).toBe(true)
+            }
+          )
+
+          it.each(["10:01", "14:59:59", `12:${minute()}`])(
+            "should reject values out range (%s)",
+            async time => {
+              const row = { time }
+              const output = await validate({ table, tableId: table._id!, row })
+              expect(output.valid).toBe(false)
+              expect(output.errors).toEqual({
+                time: ["must be no later than 10:00"],
+              })
+            }
+          )
+        })
+      })
+    })
+
+    describe("required", () => {
+      it("should reject empty values", async () => {
+        const row = {}
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+        }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({ time: ["can't be blank"] })
+      })
+
+      it.each([undefined, null])("should reject %s values", async time => {
+        const row = { time }
+        const table = getTable()
+        table.schema.time.constraints = {
+          presence: true,
+        }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({ time: ["can't be blank"] })
+      })
+    })
+
+    describe("range", () => {
+      const table = getTable()
+      table.schema.time.constraints = {
+        presence: true,
+        datetime: {
+          earliest: "10:00",
+          latest: "15:00",
+        },
+      }
+
+      it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])(
+        "should accept values in range (%s)",
+        async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(true)
+        }
+      )
+
+      it.each([
+        "9:59:50",
+        `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
+      ])("should reject values before range (%s)", async time => {
+        const row = { time }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({
+          time: ["must be no earlier than 10:00"],
+        })
+      })
+
+      it.each([
+        "15:00:01",
+        `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
+      ])("should reject values after range (%s)", async time => {
+        const row = { time }
+        const output = await validate({ table, tableId: table._id!, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toEqual({
+          time: ["must be no later than 15:00"],
+        })
+      })
+
+      describe("datetime ISO configs", () => {
+        const table = getTable()
+
+        table.schema.time.constraints = {
+          presence: true,
+          datetime: {
+            earliest: dayjs().hour(10).minute(0).second(0).toISOString(),
+            latest: dayjs().hour(15).minute(0).second(0).toISOString(),
+          },
+        }
+
+        it.each(["10:00", "15:00", `12:${minute()}`])(
+          "should accept values in range (%s)",
+          async time => {
+            const row = { time }
+            const output = await validate({ table, tableId: table._id!, row })
+            expect(output.valid).toBe(true)
+          }
+        )
+
+        it.each([
+          "09:59:50",
+          `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
+        ])("should reject values before range (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no earlier than 10:00"],
+          })
+        })
+
+        it.each([
+          "15:00:01",
+          `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
+        ])("should reject values after range (%s)", async time => {
+          const row = { time }
+          const output = await validate({ table, tableId: table._id!, row })
+          expect(output.valid).toBe(false)
+          expect(output.errors).toEqual({
+            time: ["must be no later than 15:00"],
+          })
+        })
+      })
+    })
+  })
+})

packages/server/src/sdk/app/rows/utils.ts

@@ -1,8 +1,10 @@
-import cloneDeep from "lodash/cloneDeep"
 import validateJs from "validate.js"
+import dayjs from "dayjs"
+import cloneDeep from "lodash/fp/cloneDeep"
 import {
   Datasource,
   DatasourcePlusQueryResponse,
+  FieldConstraints,
   FieldType,
   QueryJson,
   Row,
@@ -206,9 +208,7 @@ export async function validate({
         errors[fieldName] = [`Contains invalid JSON`]
       }
     } else if (type === FieldType.DATETIME && column.timeOnly) {
-      if (row[fieldName] && !row[fieldName].match(/^(\d+)(:[0-5]\d){1,2}$/)) {
-        errors[fieldName] = [`${fieldName} is not a valid time`]
-      }
+      res = validateTimeOnlyField(fieldName, row[fieldName], constraints)
     } else {
       res = validateJs.single(row[fieldName], constraints)
     }
@@ -216,3 +216,86 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
+
+function validateTimeOnlyField(
+  fieldName: string,
+  value: any,
+  constraints: FieldConstraints | undefined
+) {
+  let res
+  if (value && !value.match(/^(\d+)(:[0-5]\d){1,2}$/)) {
+    res = [`"${fieldName}" is not a valid time`]
+  } else if (constraints) {
+    let castedValue = value
+    const stringTimeToDate = (value: string) => {
+      const [hour, minute, second] = value.split(":").map((x: string) => +x)
+      let date = dayjs("2000-01-01T00:00:00.000Z").hour(hour).minute(minute)
+      if (!isNaN(second)) {
+        date = date.second(second)
+      }
+      return date
+    }
+
+    if (castedValue) {
+      castedValue = stringTimeToDate(castedValue)
+    }
+    let castedConstraints = cloneDeep(constraints)
+
+    let earliest, latest
+    let easliestTimeString: string, latestTimeString: string
+    if (castedConstraints.datetime?.earliest) {
+      easliestTimeString = castedConstraints.datetime.earliest
+      if (dayjs(castedConstraints.datetime.earliest).isValid()) {
+        easliestTimeString = dayjs(castedConstraints.datetime.earliest).format(
+          "HH:mm"
+        )
+      }
+      earliest = stringTimeToDate(easliestTimeString)
+    }
+    if (castedConstraints.datetime?.latest) {
+      latestTimeString = castedConstraints.datetime.latest
+      if (dayjs(castedConstraints.datetime.latest).isValid()) {
+        latestTimeString = dayjs(castedConstraints.datetime.latest).format(
+          "HH:mm"
+        )
+      }
+      latest = stringTimeToDate(latestTimeString)
+    }
+
+    if (earliest && latest && earliest.isAfter(latest)) {
+      latest = latest.add(1, "day")
+      if (earliest.isAfter(castedValue)) {
+        castedValue = castedValue.add(1, "day")
+      }
+    }
+
+    if (earliest || latest) {
+      castedConstraints.datetime = {
+        earliest: earliest?.toISOString() || "",
+        latest: latest?.toISOString() || "",
+      }
+    }
+
+    let jsValidation = validateJs.single(
+      castedValue?.toISOString(),
+      castedConstraints
+    )
+    jsValidation = jsValidation?.map((m: string) =>
+      m
+        ?.replace(
+          castedConstraints.datetime?.earliest || "",
+          easliestTimeString || ""
+        )
+        .replace(
+          castedConstraints.datetime?.latest || "",
+          latestTimeString || ""
+        )
+    )
+    if (jsValidation) {
+      res ??= []
+      res.push(...jsValidation)
+    }
+  }
+
+  return res
+}

packages/server/src/sdk/app/tables/external/index.ts

@@ -73,6 +73,16 @@ function validate(table: Table, oldTable?: Table) {
         `Column "${key}" has subtype "${column.subtype}" - this is not supported.`
       )
     }
+
+    if (column.type === FieldType.DATETIME) {
+      const oldColumn = oldTable?.schema[key] as typeof column
+
+      if (oldColumn && column.timeOnly !== oldColumn.timeOnly) {
+        throw new Error(
+          `Column "${key}" can not change from time to datetime or viceversa.`
+        )
+      }
+    }
   }
 }