From 253fa0def8ce95e11cb38edb613c5270e09d7704 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 16 Apr 2024 16:20:44 +0100
Subject: [PATCH 01/22] In progress: bigint tests.

---
 .../src/api/routes/tests/search.spec.ts       | 97 ++++++++++++++++++-
 .../src/sdk/app/tables/external/index.ts      |  6 ++
 .../server/src/sdk/app/tables/internal/sqs.ts |  2 +-
 3 files changed, 99 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 5b71ec9044..39ba0b589d 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -18,14 +18,15 @@ import _ from "lodash"
 jest.unmock("mssql")
 
 describe.each([
-  ["internal", undefined],
+  // ["internal", undefined],
   ["internal-sqs", undefined],
-  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
 ])("/api/:sourceId/search (%s)", (name, dsProvider) => {
   const isSqs = name === "internal-sqs"
+  const isInternal = name === "internal"
   const config = setup.getConfig()
 
   let envCleanup: (() => void) | undefined
@@ -550,4 +551,90 @@ describe.each([
         ]))
     })
   })
+
+  describe("bigints", () => {
+    const SMALL = "1"
+    const MEDIUM = "10000000"
+
+    // Our bigints are int64s in most datasources.
+    const BIG = "9223372036854775807"
+
+    beforeAll(async () => {
+      await createTable({
+        num: { name: "num", type: FieldType.BIGINT },
+      })
+      await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
+    })
+
+    describe("equal", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ equal: { num: SMALL } }).toContainExactly([
+          { num: SMALL },
+        ]))
+
+      it("successfully finds a big value", () =>
+        expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ equal: { num: "2" } }).toFindNothing())
+    })
+
+    describe("notEqual", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ notEqual: { num: SMALL } }).toContainExactly([
+          { num: MEDIUM },
+          { num: BIG },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ notEqual: { num: 10 } }).toContainExactly([
+          { num: SMALL },
+          { num: MEDIUM },
+          { num: BIG },
+        ]))
+    })
+
+    describe("oneOf", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([
+          { num: SMALL },
+        ]))
+
+      it("successfully finds all rows", () =>
+        expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([
+          { num: SMALL },
+          { num: MEDIUM },
+          { num: BIG },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ oneOf: { num: [2] } }).toFindNothing())
+    })
+
+    // Range searches against bigints don't seem to work at all in Lucene, and I
+    // couldn't figure out why. Given that we're replacing Lucene with SQS,
+    // we've decided not to spend time on it.
+    !isInternal &&
+      describe("range", () => {
+        it.only("successfully finds a row", () =>
+          expectQuery({
+            range: { num: { low: SMALL, high: "5" } },
+          }).toContainExactly([{ num: SMALL }]))
+
+        it("successfully finds multiple rows", () =>
+          expectQuery({
+            range: { num: { low: SMALL, high: MEDIUM } },
+          }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]))
+
+        it("successfully finds a row with a high bound", () =>
+          expectQuery({
+            range: { num: { low: MEDIUM, high: BIG } },
+          }).toContainExactly([{ num: MEDIUM }, { num: BIG }]))
+
+        it("successfully finds no rows", () =>
+          expectQuery({
+            range: { num: { low: "5", high: "5" } },
+          }).toFindNothing())
+      })
+  })
 })
diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts
index 65cd4a07c1..bc8430c72c 100644
--- a/packages/server/src/sdk/app/tables/external/index.ts
+++ b/packages/server/src/sdk/app/tables/external/index.ts
@@ -52,6 +52,12 @@ export async function save(
     !oldTable &&
     (tableToSave.primary == null || tableToSave.primary.length === 0)
   ) {
+    if (tableToSave.schema.id) {
+      throw new Error(
+        "External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column."
+      )
+    }
+
     tableToSave.primary = ["id"]
     tableToSave.schema.id = {
       type: FieldType.NUMBER,
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 79d9be2348..14313973c1 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -31,7 +31,7 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.ATTACHMENT_SINGLE]: SQLiteType.BLOB,
   [FieldType.ARRAY]: SQLiteType.BLOB,
   [FieldType.LINK]: SQLiteType.BLOB,
-  [FieldType.BIGINT]: SQLiteType.REAL,
+  [FieldType.BIGINT]: SQLiteType.TEXT,
   // TODO: consider the difference between multi-user and single user types (subtyping)
   [FieldType.BB_REFERENCE]: SQLiteType.TEXT,
 }

From 03b18234635445da730dc3f01bfbf7bb95c3de3b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 16 Apr 2024 17:28:21 +0100
Subject: [PATCH 02/22] Range tests passing.

---
 .../src/api/routes/tests/search.spec.ts       | 50 ++++++++++++++--
 packages/server/src/integrations/base/sql.ts  | 57 +++++++++++++++----
 .../server/src/sdk/app/rows/search/sqs.ts     |  2 +-
 packages/types/src/sdk/search.ts              | 11 ++--
 4 files changed, 97 insertions(+), 23 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 39ba0b589d..698ea0c10b 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -18,12 +18,12 @@ import _ from "lodash"
 jest.unmock("mssql")
 
 describe.each([
-  // ["internal", undefined],
+  ["internal", undefined],
   ["internal-sqs", undefined],
-  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
 ])("/api/:sourceId/search (%s)", (name, dsProvider) => {
   const isSqs = name === "internal-sqs"
   const isInternal = name === "internal"
@@ -337,6 +337,20 @@ describe.each([
         expectQuery({
           range: { age: { low: 5, high: 9 } },
         }).toFindNothing())
+
+      // We never implemented half-open ranges in Lucene.
+      !isInternal &&
+        it("can search using just a low value", () =>
+          expectQuery({
+            range: { age: { low: 5 } },
+          }).toContainExactly([{ age: 10 }]))
+
+      // We never implemented half-open ranges in Lucene.
+      !isInternal &&
+        it("can search using just a high value", () =>
+          expectQuery({
+            range: { age: { high: 5 } },
+          }).toContainExactly([{ age: 1 }]))
     })
 
     describe("sort", () => {
@@ -441,6 +455,20 @@ describe.each([
         expectQuery({
           range: { dob: { low: JAN_5TH, high: JAN_9TH } },
         }).toFindNothing())
+
+      // We never implemented half-open ranges in Lucene.
+      !isInternal &&
+        it("can search using just a low value", () =>
+          expectQuery({
+            range: { dob: { low: JAN_5TH } },
+          }).toContainExactly([{ dob: JAN_10TH }]))
+
+      // We never implemented half-open ranges in Lucene.
+      !isInternal &&
+        it("can search using just a high value", () =>
+          expectQuery({
+            range: { dob: { high: JAN_5TH } },
+          }).toContainExactly([{ dob: JAN_1ST }]))
     })
 
     describe("sort", () => {
@@ -616,7 +644,7 @@ describe.each([
     // we've decided not to spend time on it.
     !isInternal &&
       describe("range", () => {
-        it.only("successfully finds a row", () =>
+        it("successfully finds a row", () =>
           expectQuery({
             range: { num: { low: SMALL, high: "5" } },
           }).toContainExactly([{ num: SMALL }]))
@@ -635,6 +663,16 @@ describe.each([
           expectQuery({
             range: { num: { low: "5", high: "5" } },
           }).toFindNothing())
+
+        it("can search using just a low value", () =>
+          expectQuery({
+            range: { num: { low: MEDIUM } },
+          }).toContainExactly([{ num: MEDIUM }, { num: BIG }]))
+
+        it("can search using just a high value", () =>
+          expectQuery({
+            range: { num: { high: MEDIUM } },
+          }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]))
       })
   })
 })
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 259abec106..a3454d7a56 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -146,7 +146,7 @@ class InternalBuilder {
   addFilters(
     query: Knex.QueryBuilder,
     filters: SearchFilters | undefined,
-    tableName: string,
+    table: Table,
     opts: { aliases?: Record<string, string>; relationship?: boolean }
   ): Knex.QueryBuilder {
     function getTableName(name: string) {
@@ -161,7 +161,7 @@ class InternalBuilder {
         const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${getTableName(tableName)}.${updatedKey}`, value)
+          fn(`${getTableName(table.name)}.${updatedKey}`, value)
         }
         if (opts.relationship && isRelationshipField) {
           const [filterTableName, property] = updatedKey.split(".")
@@ -276,6 +276,9 @@ class InternalBuilder {
     }
     if (filters.range) {
       iterate(filters.range, (key, value) => {
+        const fieldName = key.split(".")[1]
+        const field = table.schema[fieldName]
+
         const isEmptyObject = (val: any) => {
           return (
             val &&
@@ -293,16 +296,46 @@ class InternalBuilder {
           highValid = isValidFilter(value.high)
         if (lowValid && highValid) {
           // Use a between operator if we have 2 valid range values
-          const fnc = allOr ? "orWhereBetween" : "whereBetween"
-          query = query[fnc](key, [value.low, value.high])
+          if (
+            field.type === FieldType.BIGINT &&
+            this.client === SqlClient.SQL_LITE
+          ) {
+            query = query.whereRaw(
+              `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
+              [value.low, value.high]
+            )
+          } else {
+            const fnc = allOr ? "orWhereBetween" : "whereBetween"
+            query = query[fnc](key, [value.low, value.high])
+          }
         } else if (lowValid) {
           // Use just a single greater than operator if we only have a low
-          const fnc = allOr ? "orWhere" : "where"
-          query = query[fnc](key, ">", value.low)
+          if (
+            field.type === FieldType.BIGINT &&
+            this.client === SqlClient.SQL_LITE
+          ) {
+            query = query.whereRaw(
+              `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`,
+              [value.low]
+            )
+          } else {
+            const fnc = allOr ? "orWhere" : "where"
+            query = query[fnc](key, ">=", value.low)
+          }
         } else if (highValid) {
           // Use just a single less than operator if we only have a high
-          const fnc = allOr ? "orWhere" : "where"
-          query = query[fnc](key, "<", value.high)
+          if (
+            field.type === FieldType.BIGINT &&
+            this.client === SqlClient.SQL_LITE
+          ) {
+            query = query.whereRaw(
+              `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`,
+              [value.high]
+            )
+          } else {
+            const fnc = allOr ? "orWhere" : "where"
+            query = query[fnc](key, "<=", value.high)
+          }
         }
       })
     }
@@ -532,7 +565,7 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(query, filters, tableName, {
+    query = this.addFilters(query, filters, json.meta?.table!, {
       aliases: tableAliases,
     })
     // add sorting to pre-query
@@ -553,7 +586,7 @@ class InternalBuilder {
       endpoint.schema,
       tableAliases
     )
-    return this.addFilters(query, filters, tableName, {
+    return this.addFilters(query, filters, json.meta?.table!, {
       relationship: true,
       aliases: tableAliases,
     })
@@ -563,7 +596,7 @@ class InternalBuilder {
     const { endpoint, body, filters, tableAliases } = json
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
     const parsedBody = parseBody(body)
-    query = this.addFilters(query, filters, endpoint.entityId, {
+    query = this.addFilters(query, filters, json.meta?.table!, {
       aliases: tableAliases,
     })
     // mysql can't use returning
@@ -577,7 +610,7 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, filters, tableAliases } = json
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
-    query = this.addFilters(query, filters, endpoint.entityId, {
+    query = this.addFilters(query, filters, json.meta?.table!, {
       aliases: tableAliases,
     })
     // mysql can't use returning
diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 7abd7d9e72..4517739d26 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -185,6 +185,6 @@ export async function search(
     }
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message
-    throw new Error(`Unable to search by SQL - ${msg}`)
+    throw new Error(`Unable to search by SQL - ${msg}`, { cause: err })
   }
 }
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index 9325f09eed..288618647f 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -13,10 +13,13 @@ export interface SearchFilters {
     [key: string]: string
   }
   range?: {
-    [key: string]: {
-      high: number | string
-      low: number | string
-    }
+    [key: string]:
+      | {
+          high: number | string
+          low: number | string
+        }
+      | { high: number | string }
+      | { low: number | string }
   }
   equal?: {
     [key: string]: any

From b01b260e396c5597c4cbe201bd3d92ec71124558 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 16 Apr 2024 17:36:51 +0100
Subject: [PATCH 03/22] Fix some of the broken tests.

---
 packages/server/src/integrations/base/sql.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 65709d8a82..e157590c5e 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -175,7 +175,7 @@ class InternalBuilder {
         const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${getTableName(table)}.${updatedKey}`, value)
+          fn(`${getTableAlias(table.name)}.${updatedKey}`, value)
         }
         if (opts.relationship && isRelationshipField) {
           const [filterTableName, property] = updatedKey.split(".")

From 38dc7ae39142e718aebdb767c1120ee8f49abc8e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 16 Apr 2024 17:39:05 +0100
Subject: [PATCH 04/22] Remove extraneous ? and ! operators.

---
 packages/server/src/integrations/base/sql.ts | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index e157590c5e..4b85f1cea2 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -392,7 +392,7 @@ class InternalBuilder {
 
   addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
     let { sort, paginate } = json
-    const table = json.meta?.table
+    const table = json.meta.table
     const tableName = getTableName(table)
     const aliases = json.tableAliases
     const aliased =
@@ -580,7 +580,7 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(query, filters, json.meta?.table!, {
+    query = this.addFilters(query, filters, json.meta.table, {
       aliases: tableAliases,
     })
     // add sorting to pre-query
@@ -601,7 +601,7 @@ class InternalBuilder {
       endpoint.schema,
       tableAliases
     )
-    return this.addFilters(query, filters, json.meta?.table!, {
+    return this.addFilters(query, filters, json.meta.table, {
       relationship: true,
       aliases: tableAliases,
     })
@@ -611,7 +611,7 @@ class InternalBuilder {
     const { endpoint, body, filters, tableAliases } = json
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
     const parsedBody = parseBody(body)
-    query = this.addFilters(query, filters, json.meta?.table!, {
+    query = this.addFilters(query, filters, json.meta.table, {
       aliases: tableAliases,
     })
     // mysql can't use returning
@@ -625,7 +625,7 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, filters, tableAliases } = json
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
-    query = this.addFilters(query, filters, json.meta?.table!, {
+    query = this.addFilters(query, filters, json.meta.table, {
       aliases: tableAliases,
     })
     // mysql can't use returning
@@ -717,7 +717,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
   // when creating if an ID has been inserted need to make sure
   // the id filter is enriched with it before trying to retrieve the row
   checkLookupKeys(id: any, json: QueryJson) {
-    if (!id || !json.meta?.table || !json.meta.table.primary) {
+    if (!id || !json.meta.table || !json.meta.table.primary) {
       return json
     }
     const primaryKey = json.meta.table.primary?.[0]

From affa546159c48ddf3e7f7ff309fff5aadef9813d Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 16 Apr 2024 17:45:06 +0100
Subject: [PATCH 05/22] Fix tests.

---
 packages/server/src/integrations/base/sql.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 4b85f1cea2..7ee22b5933 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -163,6 +163,9 @@ class InternalBuilder {
     table: Table,
     opts: { aliases?: Record<string, string>; relationship?: boolean }
   ): Knex.QueryBuilder {
+    const tableName =
+      this.client === SqlClient.SQL_LITE ? table._id! : table.name
+
     function getTableAlias(name: string) {
       const alias = opts.aliases?.[name]
       return alias || name
@@ -175,7 +178,7 @@ class InternalBuilder {
         const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${getTableAlias(table.name)}.${updatedKey}`, value)
+          fn(`${getTableAlias(tableName)}.${updatedKey}`, value)
         }
         if (opts.relationship && isRelationshipField) {
           const [filterTableName, property] = updatedKey.split(".")

From 828d78f2afa63fc1a5932c8ce733bd9d835bad76 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 17 Apr 2024 14:47:56 +0100
Subject: [PATCH 06/22] Fixing build after recent type updates.

---
 .../src/integrations/tests/sqlAlias.spec.ts    |  3 ++-
 packages/shared-core/src/filters.ts            | 18 ++++++++++--------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index f4edab8dad..fda2a091fa 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -117,7 +117,8 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       const filters = queryJson.filters
       const notEqualsValue = Object.values(filters?.notEqual!)[0]
-      const rangeValue = Object.values(filters?.range!)[0]
+      const rangeValue: { high?: string | number; low?: string | number } =
+        Object.values(filters?.range!)[0]
       const equalValue = Object.values(filters?.equal!)[0]
 
       expect(query).toEqual({
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 6010f064bf..0554e0c1e4 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -218,14 +218,16 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => {
           high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
         }
       }
-      if ((operator as any) === "rangeLow" && value != null && value !== "") {
-        query.range[field].low = value
-      } else if (
-        (operator as any) === "rangeHigh" &&
-        value != null &&
-        value !== ""
-      ) {
-        query.range[field].high = value
+      if (operator === "rangeLow" && value != null && value !== "") {
+        query.range[field] = {
+          ...query.range[field],
+          low: value,
+        }
+      } else if (operator === "rangeHigh" && value != null && value !== "") {
+        query.range[field] = {
+          ...query.range[field],
+          high: value,
+        }
       }
     } else if (query[queryOperator] && operator !== "onEmptyFilter") {
       if (type === "boolean") {

From a33c2599b51b793d13ba25a614a94d18b5d07898 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 17 Apr 2024 16:25:27 +0100
Subject: [PATCH 07/22] Adding error to catch scenario that caused tests to
 fail - fixing cases of table metadata not aligning with entityId

---
 .../api/controllers/row/ExternalRequest.ts    | 28 +++++++++++++------
 .../server/src/integrations/base/query.ts     | 12 ++++++++
 packages/types/src/sdk/datasources.ts         |  8 ++++++
 3 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 3dd3f9b8e7..132028b86f 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -374,38 +374,44 @@ export class ExternalRequest<T extends Operation> {
       ) {
         continue
       }
-      let tableId: string | undefined,
+      let relatedTableId: string | undefined,
         lookupField: string | undefined,
         fieldName: string | undefined
       if (isManyToMany(field)) {
-        tableId = field.through
+        relatedTableId = field.through
         lookupField = primaryKey
         fieldName = field.throughTo || primaryKey
       } else if (isManyToOne(field)) {
-        tableId = field.tableId
+        relatedTableId = field.tableId
         lookupField = field.foreignKey
         fieldName = field.fieldName
       }
-      if (!tableId || !lookupField || !fieldName) {
+      if (!relatedTableId || !lookupField || !fieldName) {
         throw new Error(
           "Unable to lookup relationships - undefined column properties."
         )
       }
-      const { tableName: relatedTableName } = breakExternalTableId(tableId)
+      const { tableName: relatedTableName } =
+        breakExternalTableId(relatedTableId)
       // @ts-ignore
       const linkPrimaryKey = this.tables[relatedTableName].primary[0]
       if (!lookupField || !row[lookupField]) {
         continue
       }
+      const endpoint = getEndpoint(relatedTableId, Operation.READ)
+      const relatedTable = this.tables[endpoint.entityId]
+      if (!relatedTable) {
+        throw new Error("unable to find related table")
+      }
       const response = await getDatasourceAndQuery({
-        endpoint: getEndpoint(tableId, Operation.READ),
+        endpoint: endpoint,
         filters: {
           equal: {
             [fieldName]: row[lookupField],
           },
         },
         meta: {
-          table,
+          table: relatedTable,
         },
       })
       // this is the response from knex if no rows found
@@ -414,7 +420,11 @@ export class ExternalRequest<T extends Operation> {
       const storeTo = isManyToMany(field)
         ? field.throughFrom || linkPrimaryKey
         : fieldName
-      related[storeTo] = { rows, isMany: isManyToMany(field), tableId }
+      related[storeTo] = {
+        rows,
+        isMany: isManyToMany(field),
+        tableId: relatedTableId,
+      }
     }
     return related
   }
@@ -484,7 +494,7 @@ export class ExternalRequest<T extends Operation> {
             body,
             filters: buildFilters(id, {}, linkTable),
             meta: {
-              table,
+              table: linkTable,
             },
           })
         )
diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts
index 03e6028e32..371592bece 100644
--- a/packages/server/src/integrations/base/query.ts
+++ b/packages/server/src/integrations/base/query.ts
@@ -2,6 +2,7 @@ import {
   QueryJson,
   Datasource,
   DatasourcePlusQueryResponse,
+  RowOperations,
 } from "@budibase/types"
 import { getIntegration } from "../index"
 import sdk from "../../sdk"
@@ -10,6 +11,17 @@ export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
 ): Promise<DatasourcePlusQueryResponse> {
+  const entityId = json.endpoint.entityId,
+    tableName = json.meta.table.name,
+    tableId = json.meta.table._id
+  // case found during testing - make sure this doesn't happen again
+  if (
+    RowOperations.includes(json.endpoint.operation) &&
+    entityId !== tableId &&
+    entityId !== tableName
+  ) {
+    throw new Error("Entity ID and table metadata do not align")
+  }
   datasource = await sdk.datasources.enrich(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function
diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts
index e1a012d81e..77e4877dfa 100644
--- a/packages/types/src/sdk/datasources.ts
+++ b/packages/types/src/sdk/datasources.ts
@@ -14,6 +14,14 @@ export enum Operation {
   DELETE_TABLE = "DELETE_TABLE",
 }
 
+export const RowOperations = [
+  Operation.CREATE,
+  Operation.READ,
+  Operation.UPDATE,
+  Operation.DELETE,
+  Operation.BULK_CREATE,
+]
+
 export enum SortDirection {
   ASCENDING = "ASCENDING",
   DESCENDING = "DESCENDING",

From 30077418eb267a66a2ff5d8c7594dec88ff73219 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 17 Apr 2024 16:31:45 +0100
Subject: [PATCH 08/22] Fixing linting issue.

---
 packages/server/src/api/controllers/row/ExternalRequest.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 132028b86f..be6ac885df 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -447,7 +447,6 @@ export class ExternalRequest<T extends Operation> {
     // if we're creating (in a through table) need to wipe the existing ones first
     const promises = []
     const related = await this.lookupRelations(mainTableId, row)
-    const table = this.getTable(mainTableId)!
     for (let relationship of relationships) {
       const { key, tableId, isUpdate, id, ...rest } = relationship
       const body: { [key: string]: any } = processObjectSync(rest, row, {})

From 60ed4d8443938ee6a72c8205413105db0ea74b14 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 17 Apr 2024 17:12:26 +0100
Subject: [PATCH 09/22] Updating how between/less than/greater than are handled
 for sqlite.

---
 packages/server/src/integrations/base/sql.ts  | 59 +++-----------
 .../server/src/integrations/tests/sql.spec.ts | 24 ++++--
 .../server/src/integrations/utils/index.ts    |  2 +
 .../src/integrations/utils/sqlStatements.ts   | 80 +++++++++++++++++++
 .../src/integrations/{ => utils}/utils.ts     |  6 +-
 5 files changed, 112 insertions(+), 59 deletions(-)
 create mode 100644 packages/server/src/integrations/utils/index.ts
 create mode 100644 packages/server/src/integrations/utils/sqlStatements.ts
 rename packages/server/src/integrations/{ => utils}/utils.ts (98%)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 7ee22b5933..4c975cbef7 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -6,6 +6,7 @@ import {
   SqlClient,
   isValidFilter,
   getNativeSql,
+  SqlStatements,
 } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
@@ -163,6 +164,13 @@ class InternalBuilder {
     table: Table,
     opts: { aliases?: Record<string, string>; relationship?: boolean }
   ): Knex.QueryBuilder {
+    if (!filters) {
+      return query
+    }
+    filters = parseFilters(filters)
+    // if all or specified in filters, then everything is an or
+    const allOr = filters.allOr
+    const sqlStatements = new SqlStatements(this.client, table, { allOr })
     const tableName =
       this.client === SqlClient.SQL_LITE ? table._id! : table.name
 
@@ -261,12 +269,6 @@ class InternalBuilder {
       }
     }
 
-    if (!filters) {
-      return query
-    }
-    filters = parseFilters(filters)
-    // if all or specified in filters, then everything is an or
-    const allOr = filters.allOr
     if (filters.oneOf) {
       iterate(filters.oneOf, (key, array) => {
         const fnc = allOr ? "orWhereIn" : "whereIn"
@@ -293,9 +295,6 @@ class InternalBuilder {
     }
     if (filters.range) {
       iterate(filters.range, (key, value) => {
-        const fieldName = key.split(".")[1]
-        const field = table.schema[fieldName]
-
         const isEmptyObject = (val: any) => {
           return (
             val &&
@@ -312,47 +311,11 @@ class InternalBuilder {
         const lowValid = isValidFilter(value.low),
           highValid = isValidFilter(value.high)
         if (lowValid && highValid) {
-          // Use a between operator if we have 2 valid range values
-          if (
-            field.type === FieldType.BIGINT &&
-            this.client === SqlClient.SQL_LITE
-          ) {
-            query = query.whereRaw(
-              `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
-              [value.low, value.high]
-            )
-          } else {
-            const fnc = allOr ? "orWhereBetween" : "whereBetween"
-            query = query[fnc](key, [value.low, value.high])
-          }
+          query = sqlStatements.between(query, key, value.low, value.high)
         } else if (lowValid) {
-          // Use just a single greater than operator if we only have a low
-          if (
-            field.type === FieldType.BIGINT &&
-            this.client === SqlClient.SQL_LITE
-          ) {
-            query = query.whereRaw(
-              `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`,
-              [value.low]
-            )
-          } else {
-            const fnc = allOr ? "orWhere" : "where"
-            query = query[fnc](key, ">=", value.low)
-          }
+          query = sqlStatements.lower(query, key, value.low)
         } else if (highValid) {
-          // Use just a single less than operator if we only have a high
-          if (
-            field.type === FieldType.BIGINT &&
-            this.client === SqlClient.SQL_LITE
-          ) {
-            query = query.whereRaw(
-              `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`,
-              [value.high]
-            )
-          } else {
-            const fnc = allOr ? "orWhere" : "where"
-            query = query[fnc](key, "<=", value.high)
-          }
+          query = sqlStatements.higher(query, key, value.high)
         }
       })
     }
diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index 4ee544cc5e..5de9cc4fbc 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -1,11 +1,11 @@
 import { SqlClient } from "../utils"
 import Sql from "../base/sql"
 import {
+  FieldType,
   Operation,
   QueryJson,
-  TableSourceType,
   Table,
-  FieldType,
+  TableSourceType,
 } from "@budibase/types"
 
 const TABLE_NAME = "test"
@@ -13,7 +13,12 @@ const TABLE: Table = {
   type: "table",
   sourceType: TableSourceType.EXTERNAL,
   sourceId: "SOURCE_ID",
-  schema: {},
+  schema: {
+    id: {
+      name: "id",
+      type: FieldType.NUMBER,
+    },
+  },
   name: TABLE_NAME,
   primary: ["id"],
 }
@@ -73,7 +78,7 @@ function generateUpdateJson({
   meta?: any
 }): QueryJson {
   if (!meta.table) {
-    meta.table = table
+    meta.table = TABLE
   }
   return {
     endpoint: endpoint(table, "UPDATE"),
@@ -158,6 +163,9 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) {
       },
     ],
     extra: { idFilter: {} },
+    meta: {
+      table: TABLE,
+    },
   }
 }
 
@@ -341,7 +349,7 @@ describe("SQL query builder", () => {
     )
     expect(query).toEqual({
       bindings: [date, limit],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" > $1 limit $2) as "${TABLE_NAME}"`,
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" >= $1 limit $2) as "${TABLE_NAME}"`,
     })
   })
 
@@ -360,7 +368,7 @@ describe("SQL query builder", () => {
     )
     expect(query).toEqual({
       bindings: [date, limit],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" < $1 limit $2) as "${TABLE_NAME}"`,
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" <= $1 limit $2) as "${TABLE_NAME}"`,
     })
   })
 
@@ -594,7 +602,7 @@ describe("SQL query builder", () => {
     )
     expect(query).toEqual({
       bindings: ["2000-01-01 00:00:00", 500],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" > $1 limit $2) as "${TABLE_NAME}"`,
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" >= $1 limit $2) as "${TABLE_NAME}"`,
     })
   })
 
@@ -613,7 +621,7 @@ describe("SQL query builder", () => {
     )
     expect(query).toEqual({
       bindings: ["2010-01-01 00:00:00", 500],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" < $1 limit $2) as "${TABLE_NAME}"`,
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" <= $1 limit $2) as "${TABLE_NAME}"`,
     })
   })
 
diff --git a/packages/server/src/integrations/utils/index.ts b/packages/server/src/integrations/utils/index.ts
new file mode 100644
index 0000000000..a9c2019ba2
--- /dev/null
+++ b/packages/server/src/integrations/utils/index.ts
@@ -0,0 +1,2 @@
+export * from "./utils"
+export { SqlStatements } from "./sqlStatements"
diff --git a/packages/server/src/integrations/utils/sqlStatements.ts b/packages/server/src/integrations/utils/sqlStatements.ts
new file mode 100644
index 0000000000..b05196f56f
--- /dev/null
+++ b/packages/server/src/integrations/utils/sqlStatements.ts
@@ -0,0 +1,80 @@
+import { FieldType, Table, FieldSchema } from "@budibase/types"
+import { SqlClient } from "./utils"
+import { Knex } from "knex"
+
+export class SqlStatements {
+  client: string
+  table: Table
+  allOr: boolean | undefined
+  constructor(
+    client: string,
+    table: Table,
+    { allOr }: { allOr?: boolean } = {}
+  ) {
+    this.client = client
+    this.table = table
+    this.allOr = allOr
+  }
+
+  getField(key: string): FieldSchema | undefined {
+    const fieldName = key.split(".")[1]
+    return this.table.schema[fieldName]
+  }
+
+  between(
+    query: Knex.QueryBuilder,
+    key: string,
+    low: number | string,
+    high: number | string
+  ) {
+    // Use a between operator if we have 2 valid range values
+    const field = this.getField(key)
+    if (
+      field?.type === FieldType.BIGINT &&
+      this.client === SqlClient.SQL_LITE
+    ) {
+      query = query.whereRaw(
+        `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
+        [low, high]
+      )
+    } else {
+      const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
+      query = query[fnc](key, [low, high])
+    }
+    return query
+  }
+
+  lower(query: Knex.QueryBuilder, key: string, low: number | string) {
+    // Use just a single greater than operator if we only have a low
+    const field = this.getField(key)
+    if (
+      field?.type === FieldType.BIGINT &&
+      this.client === SqlClient.SQL_LITE
+    ) {
+      query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
+        low,
+      ])
+    } else {
+      const fnc = this.allOr ? "orWhere" : "where"
+      query = query[fnc](key, ">=", low)
+    }
+    return query
+  }
+
+  higher(query: Knex.QueryBuilder, key: string, high: number | string) {
+    const field = this.getField(key)
+    // Use just a single less than operator if we only have a high
+    if (
+      field?.type === FieldType.BIGINT &&
+      this.client === SqlClient.SQL_LITE
+    ) {
+      query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
+        high,
+      ])
+    } else {
+      const fnc = this.allOr ? "orWhere" : "where"
+      query = query[fnc](key, "<=", high)
+    }
+    return query
+  }
+}
diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils/utils.ts
similarity index 98%
rename from packages/server/src/integrations/utils.ts
rename to packages/server/src/integrations/utils/utils.ts
index d5f6d191e1..8f2bed04a2 100644
--- a/packages/server/src/integrations/utils.ts
+++ b/packages/server/src/integrations/utils/utils.ts
@@ -5,10 +5,10 @@ import {
   FieldType,
   TableSourceType,
 } from "@budibase/types"
-import { DocumentType, SEPARATOR } from "../db/utils"
-import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants"
+import { DocumentType, SEPARATOR } from "../../db/utils"
+import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
 import { helpers } from "@budibase/shared-core"
-import env from "../environment"
+import env from "../../environment"
 import { Knex } from "knex"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`

From 58d4b2b56e79d2f09b6e96ce4efd6f3cf53d481d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 17 Apr 2024 17:13:08 +0100
Subject: [PATCH 10/22] renaming some sqlstatement generation to lte/gte.

---
 packages/server/src/integrations/base/sql.ts            | 4 ++--
 packages/server/src/integrations/utils/sqlStatements.ts | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 4c975cbef7..a8d746add9 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -313,9 +313,9 @@ class InternalBuilder {
         if (lowValid && highValid) {
           query = sqlStatements.between(query, key, value.low, value.high)
         } else if (lowValid) {
-          query = sqlStatements.lower(query, key, value.low)
+          query = sqlStatements.lte(query, key, value.low)
         } else if (highValid) {
-          query = sqlStatements.higher(query, key, value.high)
+          query = sqlStatements.gte(query, key, value.high)
         }
       })
     }
diff --git a/packages/server/src/integrations/utils/sqlStatements.ts b/packages/server/src/integrations/utils/sqlStatements.ts
index b05196f56f..7a5482830b 100644
--- a/packages/server/src/integrations/utils/sqlStatements.ts
+++ b/packages/server/src/integrations/utils/sqlStatements.ts
@@ -44,7 +44,7 @@ export class SqlStatements {
     return query
   }
 
-  lower(query: Knex.QueryBuilder, key: string, low: number | string) {
+  lte(query: Knex.QueryBuilder, key: string, low: number | string) {
     // Use just a single greater than operator if we only have a low
     const field = this.getField(key)
     if (
@@ -61,7 +61,7 @@ export class SqlStatements {
     return query
   }
 
-  higher(query: Knex.QueryBuilder, key: string, high: number | string) {
+  gte(query: Knex.QueryBuilder, key: string, high: number | string) {
     const field = this.getField(key)
     // Use just a single less than operator if we only have a high
     if (

From e90e2b214e413a832902cfc55b45e5f269f66c9c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 17 Apr 2024 17:36:19 +0100
Subject: [PATCH 11/22] Making sure meta.table is always available.

---
 .../routes/tests/queries/generic-sql.spec.ts   | 18 ++++++++++++++++--
 packages/server/src/sdk/app/rows/utils.ts      |  6 ++++++
 .../src/tests/utilities/api/datasource.ts      |  2 +-
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
index 7790f909e7..e8a38dcfaa 100644
--- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
+++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
@@ -4,6 +4,7 @@ import {
   Query,
   QueryPreview,
   SourceName,
+  TableSourceType,
 } from "@budibase/types"
 import * as setup from "../utilities"
 import {
@@ -740,12 +741,25 @@ describe.each(
   })
 
   describe("query through datasource", () => {
-    it("should be able to query a pg datasource", async () => {
+    it("should be able to query the datasource", async () => {
+      const entityId = "test_table"
+      await config.api.datasource.update({
+        ...datasource,
+        entities: {
+          [entityId]: {
+            name: entityId,
+            schema: {},
+            type: "table",
+            sourceId: datasource._id!,
+            sourceType: TableSourceType.EXTERNAL,
+          },
+        },
+      })
       const res = await config.api.datasource.query({
         endpoint: {
           datasourceId: datasource._id!,
           operation: Operation.READ,
-          entityId: "test_table",
+          entityId,
         },
         resource: {
           fields: ["id", "name"],
diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index d307b17947..a9df4f89cd 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -52,6 +52,12 @@ export async function getDatasourceAndQuery(
 ): Promise<DatasourcePlusQueryResponse> {
   const datasourceId = json.endpoint.datasourceId
   const datasource = await sdk.datasources.get(datasourceId)
+  const table = datasource.entities?.[json.endpoint.entityId]
+  if (!json.meta && table) {
+    json.meta = {
+      table,
+    }
+  }
   return makeExternalQuery(datasource, json)
 }
 
diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
index 0362a25940..6ac624f0db 100644
--- a/packages/server/src/tests/utilities/api/datasource.ts
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -61,7 +61,7 @@ export class DatasourceAPI extends TestAPI {
   }
 
   query = async (
-    query: Omit<QueryJson, "meta">,
+    query: Omit<QueryJson, "meta"> & Partial<Pick<QueryJson, "meta">>,
     expectations?: Expectations
   ) => {
     return await this._post<any>(`/api/datasources/query`, {

From c36ca27292375c5a85f991aeae3306224d802216 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 12 Apr 2024 13:48:50 +0200
Subject: [PATCH 12/22] Use field types

---
 .../DataTable/modals/CreateEditColumn.svelte   | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index ff9830c92a..e8e4180c3d 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -34,15 +34,15 @@
   import { RowUtils } from "@budibase/frontend-core"
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
 
-  const AUTO_TYPE = FIELDS.AUTO.type
-  const FORMULA_TYPE = FIELDS.FORMULA.type
-  const LINK_TYPE = FIELDS.LINK.type
-  const STRING_TYPE = FIELDS.STRING.type
-  const NUMBER_TYPE = FIELDS.NUMBER.type
-  const JSON_TYPE = FIELDS.JSON.type
-  const DATE_TYPE = FIELDS.DATETIME.type
-  const USER_TYPE = FIELDS.USER.subtype
-  const USERS_TYPE = FIELDS.USERS.subtype
+  const AUTO_TYPE = FieldType.AUTO
+  const FORMULA_TYPE = FieldType.FORMULA
+  const LINK_TYPE = FieldType.LINK
+  const STRING_TYPE = FieldType.STRING
+  const NUMBER_TYPE = FieldType.NUMBER
+  const JSON_TYPE = FieldType.JSON
+  const DATE_TYPE = FieldType.DATETIME
+  const USER_TYPE = FieldSubtype.USER
+  const USERS_TYPE = FieldSubtype.USERS
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]

From e327ebd613ce25736c375ed60b18eccc33a2e521 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 18 Apr 2024 10:06:34 +0200
Subject: [PATCH 13/22] Do not display relationship selector if autocolumn
 links

---
 .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index e8e4180c3d..d271462f3e 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -626,7 +626,7 @@
         />
       </div>
     </div>
-  {:else if editableColumn.type === FieldType.LINK}
+  {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn}
     <RelationshipSelector
       bind:relationshipPart1
       bind:relationshipPart2

From b5c7dc4e70bfce51f2f1daf3fc3bf478b7fcd126 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 18 Apr 2024 08:26:29 +0000
Subject: [PATCH 14/22] Bump version to 2.23.7

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index a2be7be7b4..19f6486f22 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.23.6",
+  "version": "2.23.7",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From e90d40c0be5aa16dd893b9528e6bedb7ecec6cc9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 18 Apr 2024 10:49:22 +0200
Subject: [PATCH 15/22] Add all scripts as nx cache inputs

---
 nx.json | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/nx.json b/nx.json
index 618395ec90..8ba8798946 100644
--- a/nx.json
+++ b/nx.json
@@ -9,10 +9,7 @@
   },
   "targetDefaults": {
     "build": {
-      "inputs": [
-        "{workspaceRoot}/scripts/build.js",
-        "{workspaceRoot}/lerna.json"
-      ]
+      "inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"]
     }
   }
 }

From 8e2fb778233a9b8a3d7373655f80db2cd0968c81 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 18 Apr 2024 10:53:23 +0200
Subject: [PATCH 16/22] Prevent dev:docker running for oss

---
 package.json         |  2 +-
 scripts/devDocker.sh | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100755 scripts/devDocker.sh

diff --git a/package.json b/package.json
index e520b7c2cf..e60a086e17 100644
--- a/package.json
+++ b/package.json
@@ -59,7 +59,7 @@
     "dev:camunda": "./scripts/deploy-camunda.sh",
     "dev:all": "yarn run kill-all && lerna run --stream dev",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
-    "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
+    "dev:docker": "./scripts/devDocker.sh",
     "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream",
     "lint:eslint": "eslint packages --max-warnings=0",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh
new file mode 100755
index 0000000000..3d454bafb5
--- /dev/null
+++ b/scripts/devDocker.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Check if the pro submodule is loaded
+if [ ! -d "./packages/pro/src" ]; then
+  echo "Submodule is not loaded. This is only allowed with loaded submodules."
+  exit 1
+fi
+
+yarn build --scope @budibase/server --scope @budibase/worker
+docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0
+
+

From ccbbb2e1ee278a556543c32a4812c02dce06a5f4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 18 Apr 2024 10:53:58 +0200
Subject: [PATCH 17/22] Add message

---
 scripts/devDocker.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh
index 3d454bafb5..5e01e5813a 100755
--- a/scripts/devDocker.sh
+++ b/scripts/devDocker.sh
@@ -2,7 +2,7 @@
 
 # Check if the pro submodule is loaded
 if [ ! -d "./packages/pro/src" ]; then
-  echo "Submodule is not loaded. This is only allowed with loaded submodules."
+  echo "[ERROR] Submodule is not loaded. This is only allowed with loaded submodules."
   exit 1
 fi
 

From 6bdc726d55d9a4def42c67f00ab0e2f37be8e410 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Thu, 18 Apr 2024 13:52:00 +0100
Subject: [PATCH 18/22] Allow a displayName to be passed when creating admin
 user (#13516)

* Allow a displayName to be passed when creating admin user

* Set the first and last names

* Don't format handlebars files on save

* Use familyName and givenName
---
 .vscode/settings.json                               | 3 +++
 packages/backend-core/src/users/db.ts               | 4 ++++
 packages/types/src/api/web/user.ts                  | 2 ++
 packages/types/src/documents/global/user.ts         | 7 +++++++
 packages/worker/src/api/controllers/global/users.ts | 5 ++++-
 packages/worker/src/api/routes/global/users.ts      | 2 ++
 6 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/.vscode/settings.json b/.vscode/settings.json
index e22d5a8866..0723219a8b 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -24,5 +24,8 @@
   },
   "[svelte]": {
     "editor.defaultFormatter": "svelte.svelte-vscode"
+  },
+  "[handlebars]": {
+    "editor.formatOnSave": false
   }
 }
diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index 6165a68e57..f77c6385ba 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -50,6 +50,8 @@ type CreateAdminUserOpts = {
   hashPassword?: boolean
   requirePassword?: boolean
   skipPasswordValidation?: boolean
+  firstName?: string
+  lastName?: string
 }
 type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn }
 
@@ -517,6 +519,8 @@ export class UserDB {
         global: true,
       },
       tenantId,
+      firstName: opts?.firstName,
+      lastName: opts?.lastName,
     }
     if (opts?.ssoId) {
       user.ssoId = opts.ssoId
diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts
index 10630c272c..75f00760bf 100644
--- a/packages/types/src/api/web/user.ts
+++ b/packages/types/src/api/web/user.ts
@@ -66,6 +66,8 @@ export interface CreateAdminUserRequest {
   password?: string
   tenantId: string
   ssoId?: string
+  familyName?: string
+  givenName?: string
 }
 
 export interface AddSSoUserRequest {
diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts
index a4e6b613c6..9c7dc80e49 100644
--- a/packages/types/src/documents/global/user.ts
+++ b/packages/types/src/documents/global/user.ts
@@ -22,6 +22,13 @@ export interface UserSSO {
   providerType: SSOProviderType
   oauth2?: OAuth2
   thirdPartyProfile?: SSOProfileJson
+  profile?: {
+    displayName?: string
+    name?: {
+      givenName?: string
+      familyName?: string
+    }
+  }
 }
 
 export type SSOUser = User & UserSSO
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index 4c1af90d38..b610ecce1a 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -116,7 +116,8 @@ const parseBooleanParam = (param: any) => {
 export const adminUser = async (
   ctx: Ctx<CreateAdminUserRequest, CreateAdminUserResponse>
 ) => {
-  const { email, password, tenantId, ssoId } = ctx.request.body
+  const { email, password, tenantId, ssoId, givenName, familyName } =
+    ctx.request.body
 
   if (await platform.tenants.exists(tenantId)) {
     ctx.throw(403, "Organisation already exists.")
@@ -151,6 +152,8 @@ export const adminUser = async (
         ssoId,
         hashPassword,
         requirePassword,
+        firstName: givenName,
+        lastName: familyName,
       })
 
       // events
diff --git a/packages/worker/src/api/routes/global/users.ts b/packages/worker/src/api/routes/global/users.ts
index e7c77678fc..b40c491830 100644
--- a/packages/worker/src/api/routes/global/users.ts
+++ b/packages/worker/src/api/routes/global/users.ts
@@ -16,6 +16,8 @@ function buildAdminInitValidation() {
       password: OPTIONAL_STRING,
       tenantId: Joi.string().required(),
       ssoId: Joi.string(),
+      familyName: OPTIONAL_STRING,
+      givenName: OPTIONAL_STRING,
     })
       .required()
       .unknown(false)

From 21f554a6b89049b402cf05e3cf12de847dde2065 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 18 Apr 2024 12:53:32 +0000
Subject: [PATCH 19/22] Bump version to 2.23.8

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 19f6486f22..99a3e46ab2 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.23.7",
+  "version": "2.23.8",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From dafa20ce47f6619985f95c75bd1fce2e53bc770a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 18 Apr 2024 11:20:11 +0200
Subject: [PATCH 20/22] Add platform docs to mssql docker-compose

---
 packages/server/scripts/integrations/mssql/docker-compose.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/server/scripts/integrations/mssql/docker-compose.yaml b/packages/server/scripts/integrations/mssql/docker-compose.yaml
index 89222eddaa..0f10eee456 100644
--- a/packages/server/scripts/integrations/mssql/docker-compose.yaml
+++ b/packages/server/scripts/integrations/mssql/docker-compose.yaml
@@ -4,6 +4,7 @@ services:
   # user: sa
   # database: master
   mssql:
+    # platform: linux/amd64
     image: bb/mssql
     build:
       context: .

From 642f30416a19eae3c36882b469459720a749ac7b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 18 Apr 2024 16:40:43 +0200
Subject: [PATCH 21/22] Fix schema and aliasing usage

---
 packages/server/src/integrations/base/sql.ts | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index a8d746add9..28b7eb910b 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -472,14 +472,13 @@ class InternalBuilder {
   ): Knex.QueryBuilder {
     const tableName = endpoint.entityId
     const tableAlias = aliases?.[tableName]
-    let table: string | Record<string, string> = tableName
-    if (tableAlias) {
-      table = { [tableAlias]: tableName }
-    }
-    let query = knex(table)
-    if (endpoint.schema) {
-      query = query.withSchema(endpoint.schema)
-    }
+
+    const query = knex(
+      this.tableNameWithSchema(tableName, {
+        alias: tableAlias,
+        schema: endpoint.schema,
+      })
+    )
     return query
   }
 

From 008c8e8c9d01d9a73649aeb2c528018e0a5d73b6 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 18 Apr 2024 16:11:25 +0000
Subject: [PATCH 22/22] Bump version to 2.23.9

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 99a3e46ab2..2ce2b69508 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.23.8",
+  "version": "2.23.9",
   "npmClient": "yarn",
   "packages": [
     "packages/*",