From c7c2cb48e8180a5e9cdd678e504de1cc2456616f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 23 Jul 2024 14:41:34 +0100
Subject: [PATCH 01/34] wip

---
 .../src/api/routes/tests/search.spec.ts       | 17 ++---
 .../src/integrations/tests/utils/index.ts     |  6 ++
 .../src/integrations/tests/utils/oracle.ts    | 62 +++++++++++++++++++
 3 files changed, 77 insertions(+), 8 deletions(-)
 create mode 100644 packages/server/src/integrations/tests/utils/oracle.ts

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index e774158c23..6e685b13ab 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -40,13 +40,14 @@ import { structures } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 
 describe.each([
-  ["in-memory", undefined],
-  ["lucene", undefined],
-  ["sqs", undefined],
-  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  //["in-memory", undefined],
+  //["lucene", undefined],
+  //["sqs", undefined],
+  //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
   const isLucene = name === "lucene"
@@ -291,7 +292,7 @@ describe.each([
     })
 
     describe("equal", () => {
-      it("successfully finds true row", async () => {
+      it.only("successfully finds true row", async () => {
         await expectQuery({ equal: { isTrue: true } }).toMatchExactly([
           { isTrue: true },
         ])
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index b888f1adc1..da7d0d8666 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -4,6 +4,7 @@ import * as mongodb from "./mongodb"
 import * as mysql from "./mysql"
 import * as mssql from "./mssql"
 import * as mariadb from "./mariadb"
+import * as oracle from "./oracle"
 import { GenericContainer, StartedTestContainer } from "testcontainers"
 import { testContainerUtils } from "@budibase/backend-core/tests"
 import cloneDeep from "lodash/cloneDeep"
@@ -16,6 +17,7 @@ export enum DatabaseName {
   MYSQL = "mysql",
   SQL_SERVER = "mssql",
   MARIADB = "mariadb",
+  ORACLE = "oracle",
 }
 
 const providers: Record<DatabaseName, DatasourceProvider> = {
@@ -24,6 +26,7 @@ const providers: Record<DatabaseName, DatasourceProvider> = {
   [DatabaseName.MYSQL]: mysql.getDatasource,
   [DatabaseName.SQL_SERVER]: mssql.getDatasource,
   [DatabaseName.MARIADB]: mariadb.getDatasource,
+  [DatabaseName.ORACLE]: oracle.getDatasource,
 }
 
 export function getDatasourceProviders(
@@ -59,6 +62,9 @@ export async function knexClient(ds: Datasource) {
     case SourceName.SQL_SERVER: {
       return mssql.knexClient(ds)
     }
+    case SourceName.ORACLE: {
+      return oracle.knexClient(ds)
+    }
     default: {
       throw new Error(`Unsupported source: ${ds.source}`)
     }
diff --git a/packages/server/src/integrations/tests/utils/oracle.ts b/packages/server/src/integrations/tests/utils/oracle.ts
new file mode 100644
index 0000000000..a4f294d7ba
--- /dev/null
+++ b/packages/server/src/integrations/tests/utils/oracle.ts
@@ -0,0 +1,62 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait } from "testcontainers"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+import knex from "knex"
+
+let ports: Promise<testContainerUtils.Port[]>
+
+export async function getDatasource(): Promise<Datasource> {
+  if (!ports) {
+    let image = "oracle/database:19.3.0.0-ee"
+    if (process.arch.startsWith("arm")) {
+      image = "samhuang78/oracle-database:19.3.0-ee-slim-faststart"
+    }
+
+    ports = startContainer(
+      new GenericContainer(image)
+        .withExposedPorts(1521)
+        .withEnvironment({ ORACLE_PASSWORD: "password" })
+        .withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(10000))
+    )
+  }
+
+  const port = (await ports).find(x => x.container === 1521)?.host
+  if (!port) {
+    throw new Error("Oracle port not found")
+  }
+
+  const datasource: Datasource = {
+    type: "datasource_plus",
+    source: SourceName.ORACLE,
+    plus: true,
+    config: {
+      host: "127.0.0.1",
+      port,
+      database: "postgres",
+      user: "SYS",
+      password: "password",
+    },
+  }
+
+  const database = generator.guid().replaceAll("-", "")
+  const client = await knexClient(datasource)
+  await client.raw(`CREATE DATABASE "${database}"`)
+  datasource.config!.database = database
+
+  return datasource
+}
+
+export async function knexClient(ds: Datasource) {
+  if (!ds.config) {
+    throw new Error("Datasource config is missing")
+  }
+  if (ds.source !== SourceName.ORACLE) {
+    throw new Error("Datasource source is not Oracle")
+  }
+
+  return knex({
+    client: "oracledb",
+    connection: ds.config,
+  })
+}

From 50d1972127facb73c0c26c2e5b36b07b2d7bdb86 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 29 Jul 2024 09:57:24 +0100
Subject: [PATCH 02/34] wip

---
 packages/backend-core/src/sql/sql.ts          | 45 ++++++++++++--
 packages/server/__mocks__/oracledb.ts         | 21 -------
 .../src/api/routes/tests/search.spec.ts       | 61 ++++++++++---------
 packages/server/src/integrations/oracle.ts    | 13 +++-
 .../src/integrations/tests/utils/oracle.ts    | 39 +++++++-----
 .../src/utilities/rowProcessor/index.ts       |  7 +++
 6 files changed, 114 insertions(+), 72 deletions(-)
 delete mode 100644 packages/server/__mocks__/oracledb.ts

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index a67da7bc10..621afe7f3e 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -109,6 +109,26 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters {
   return filters
 }
 
+// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
+// so when we use them we need to wrap them in to_char(). This function
+// converts a field name to the appropriate identifier.
+function convertClobs(client: SqlClient, table: Table, field: string): string {
+  const parts = field.split(".")
+  const col = parts.pop()!
+  const schema = table.schema[col]
+  let identifier = quotedIdentifier(client, field)
+  if (
+    schema.type === FieldType.STRING ||
+    schema.type === FieldType.LONGFORM ||
+    schema.type === FieldType.BB_REFERENCE_SINGLE ||
+    schema.type === FieldType.OPTIONS ||
+    schema.type === FieldType.BARCODEQR
+  ) {
+    identifier = `to_char(${identifier})`
+  }
+  return identifier
+}
+
 function generateSelectStatement(
   json: QueryJson,
   knex: Knex
@@ -372,7 +392,15 @@ class InternalBuilder {
       iterate(
         filters.oneOf,
         (key: string, array) => {
-          query = query[fnc](key, Array.isArray(array) ? array : [array])
+          if (this.client === SqlClient.ORACLE) {
+            key = convertClobs(this.client, table, key)
+            query = query.whereRaw(
+              `${key} IN (?)`,
+              Array.isArray(array) ? array : [array]
+            )
+          } else {
+            query = query[fnc](key, Array.isArray(array) ? array : [array])
+          }
         },
         (key: string[], array) => {
           query = query[fnc](key, Array.isArray(array) ? array : [array])
@@ -436,8 +464,9 @@ class InternalBuilder {
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
+          const identifier = convertClobs(this.client, table, key)
           query = query[fnc](
-            `COALESCE(${quotedIdentifier(this.client, key)}, -1) = ?`,
+            `(${identifier} IS NOT NULL AND ${identifier} = ?)`,
             [value]
           )
         } else {
@@ -460,8 +489,9 @@ class InternalBuilder {
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
+          const identifier = convertClobs(this.client, table, key)
           query = query[fnc](
-            `COALESCE(${quotedIdentifier(this.client, key)}, -1) != ?`,
+            `(${identifier} IS NOT NULL AND ${identifier} != ?)`,
             [value]
           )
         } else {
@@ -707,8 +737,11 @@ class InternalBuilder {
       }
       const ret = query.insert(parsedBody).onConflict(primary).merge()
       return ret
-    } else if (this.client === SqlClient.MS_SQL) {
-      // No upsert or onConflict support in MSSQL yet, see:
+    } else if (
+      this.client === SqlClient.MS_SQL ||
+      this.client === SqlClient.ORACLE
+    ) {
+      // No upsert or onConflict support in MSSQL/Oracle yet, see:
       //   https://github.com/knex/knex/pull/6050
       return query.insert(parsedBody)
     }
@@ -867,7 +900,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     const config: Knex.Config = {
       client: sqlClient,
     }
-    if (sqlClient === SqlClient.SQL_LITE) {
+    if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) {
       config.useNullAsDefault = true
     }
 
diff --git a/packages/server/__mocks__/oracledb.ts b/packages/server/__mocks__/oracledb.ts
deleted file mode 100644
index 0172ace0e6..0000000000
--- a/packages/server/__mocks__/oracledb.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-const executeMock = jest.fn(() => ({
-  rows: [
-    {
-      a: "string",
-      b: 1,
-    },
-  ],
-}))
-
-const closeMock = jest.fn()
-
-class Connection {
-  execute = executeMock
-  close = closeMock
-}
-
-module.exports = {
-  getConnection: jest.fn(() => new Connection()),
-  executeMock,
-  closeMock,
-}
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 6e685b13ab..57afdb4853 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -40,13 +40,13 @@ import { structures } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 
 describe.each([
-  //["in-memory", undefined],
-  //["lucene", undefined],
-  //["sqs", undefined],
-  //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  // ["in-memory", undefined],
+  // ["lucene", undefined],
+  // ["sqs", undefined],
+  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
   [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
@@ -292,7 +292,7 @@ describe.each([
     })
 
     describe("equal", () => {
-      it.only("successfully finds true row", async () => {
+      it("successfully finds true row", async () => {
         await expectQuery({ equal: { isTrue: true } }).toMatchExactly([
           { isTrue: true },
         ])
@@ -1577,12 +1577,15 @@ describe.each([
     })
   })
 
-  describe("bigints", () => {
+  describe.only("bigints", () => {
     const SMALL = "1"
     const MEDIUM = "10000000"
 
     // Our bigints are int64s in most datasources.
-    const BIG = "9223372036854775807"
+    let BIG = "9223372036854775807"
+    if (name === DatabaseName.ORACLE) {
+      // BIG = "9223372036854775808"
+    }
 
     beforeAll(async () => {
       table = await createTable({
@@ -2415,25 +2418,25 @@ describe.each([
 
   describe.each([
     "名前", // Japanese for "name"
-    "Benutzer-ID", // German for "user ID", includes a hyphen
-    "numéro", // French for "number", includes an accent
-    "år", // Swedish for "year", includes a ring above
-    "naïve", // English word borrowed from French, includes an umlaut
-    "الاسم", // Arabic for "name"
-    "оплата", // Russian for "payment"
-    "पता", // Hindi for "address"
-    "用戶名", // Chinese for "username"
-    "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
-    "preço", // Portuguese for "price", includes a cedilla
-    "사용자명", // Korean for "username"
-    "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
-    "файл", // Bulgarian for "file"
-    "δεδομένα", // Greek for "data"
-    "geändert_am", // German for "modified on", includes an umlaut
-    "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
-    "São_Paulo", // Portuguese, includes an underscore and a tilde
-    "età", // Italian for "age", includes an accent
-    "ชื่อผู้ใช้", // Thai for "username"
+    // "Benutzer-ID", // German for "user ID", includes a hyphen
+    // "numéro", // French for "number", includes an accent
+    // "år", // Swedish for "year", includes a ring above
+    // "naïve", // English word borrowed from French, includes an umlaut
+    // "الاسم", // Arabic for "name"
+    // "оплата", // Russian for "payment"
+    // "पता", // Hindi for "address"
+    // "用戶名", // Chinese for "username"
+    // "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
+    // "preço", // Portuguese for "price", includes a cedilla
+    // "사용자명", // Korean for "username"
+    // "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
+    // "файл", // Bulgarian for "file"
+    // "δεδομένα", // Greek for "data"
+    // "geändert_am", // German for "modified on", includes an umlaut
+    // "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
+    // "São_Paulo", // Portuguese, includes an underscore and a tilde
+    // "età", // Italian for "age", includes an accent
+    // "ชื่อผู้ใช้", // Thai for "username"
   ])("non-ascii column name: %s", name => {
     beforeAll(async () => {
       table = await createTable({
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index 9f40372546..b36501525b 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -360,11 +360,20 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       this.index = 1
       connection = await this.getConnection()
 
-      const options: ExecuteOptions = { autoCommit: true }
+      const options: ExecuteOptions = {
+        autoCommit: true,
+        fetchTypeHandler: function (metaData) {
+          if (metaData.dbType === oracledb.CLOB) {
+            return { type: oracledb.STRING }
+          }
+          return undefined
+        },
+      }
       const bindings: BindParameters = query.bindings || []
 
       this.log(query.sql, bindings)
-      return await connection.execute<T>(query.sql, bindings, options)
+      const result = await connection.execute(query.sql, bindings, options)
+      return result as Result<T>
     } finally {
       if (connection) {
         try {
diff --git a/packages/server/src/integrations/tests/utils/oracle.ts b/packages/server/src/integrations/tests/utils/oracle.ts
index a4f294d7ba..9b75a52da7 100644
--- a/packages/server/src/integrations/tests/utils/oracle.ts
+++ b/packages/server/src/integrations/tests/utils/oracle.ts
@@ -8,7 +8,7 @@ let ports: Promise<testContainerUtils.Port[]>
 
 export async function getDatasource(): Promise<Datasource> {
   if (!ports) {
-    let image = "oracle/database:19.3.0.0-ee"
+    let image = "oracle/database:19.3.0.0-ee-slim-faststart"
     if (process.arch.startsWith("arm")) {
       image = "samhuang78/oracle-database:19.3.0-ee-slim-faststart"
     }
@@ -17,7 +17,7 @@ export async function getDatasource(): Promise<Datasource> {
       new GenericContainer(image)
         .withExposedPorts(1521)
         .withEnvironment({ ORACLE_PASSWORD: "password" })
-        .withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(10000))
+        .withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(60000))
     )
   }
 
@@ -26,23 +26,25 @@ export async function getDatasource(): Promise<Datasource> {
     throw new Error("Oracle port not found")
   }
 
+  const host = "127.0.0.1"
+  const user = "SYSTEM"
+  const password = "password"
+
   const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.ORACLE,
     plus: true,
-    config: {
-      host: "127.0.0.1",
-      port,
-      database: "postgres",
-      user: "SYS",
-      password: "password",
-    },
+    config: { host, port, user, password, database: "FREEPDB1" },
   }
 
-  const database = generator.guid().replaceAll("-", "")
+  const newUser = "a" + generator.guid().replaceAll("-", "")
   const client = await knexClient(datasource)
-  await client.raw(`CREATE DATABASE "${database}"`)
-  datasource.config!.database = database
+  await client.raw(`CREATE USER ${newUser} IDENTIFIED BY password`)
+  await client.raw(
+    `GRANT CONNECT, RESOURCE, CREATE VIEW, CREATE SESSION TO ${newUser}`
+  )
+  await client.raw(`GRANT UNLIMITED TABLESPACE TO ${newUser}`)
+  datasource.config!.user = newUser
 
   return datasource
 }
@@ -55,8 +57,17 @@ export async function knexClient(ds: Datasource) {
     throw new Error("Datasource source is not Oracle")
   }
 
-  return knex({
+  const db = ds.config.database || "FREEPDB1"
+  const connectString = `${ds.config.host}:${ds.config.port}/${db}`
+
+  const c = knex({
     client: "oracledb",
-    connection: ds.config,
+    connection: {
+      connectString,
+      user: ds.config.user,
+      password: ds.config.password,
+    },
   })
+
+  return c
 }
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 71de056814..a367c6da1e 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -315,6 +315,13 @@ export async function outputProcessing<T extends Row[] | Row>(
           column.subtype
         )
       }
+    } else if (column.type === FieldType.BIGINT) {
+      for (const row of enriched) {
+        if (row[property] == null) {
+          continue
+        }
+        row[property] = row[property].toString()
+      }
     }
   }
 

From a4b66e00e46c97c46614961746bb047a6608bf16 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 29 Jul 2024 14:32:28 +0100
Subject: [PATCH 03/34] Fix bigints.

---
 packages/server/src/api/routes/tests/search.spec.ts |  2 +-
 packages/server/src/integrations/oracle.ts          | 12 ++++++++++++
 packages/server/src/utilities/rowProcessor/index.ts |  7 -------
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 57afdb4853..4c7410eb76 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1577,7 +1577,7 @@ describe.each([
     })
   })
 
-  describe.only("bigints", () => {
+  describe("bigints", () => {
     const SMALL = "1"
     const MEDIUM = "10000000"
 
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index b36501525b..6b86fba00d 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -365,6 +365,18 @@ class OracleIntegration extends Sql implements DatasourcePlus {
         fetchTypeHandler: function (metaData) {
           if (metaData.dbType === oracledb.CLOB) {
             return { type: oracledb.STRING }
+          } else if (
+            // When we create a new table in OracleDB from Budibase, bigints get
+            // created as NUMBER(20,0). Budibase expects bigints to be returned
+            // as strings, which is what we're doing here. However, this is
+            // likely to be brittle if we connect to externally created
+            // databases that have used different precisions and scales.
+            // We shold find a way to do better.
+            metaData.dbType === oracledb.NUMBER &&
+            metaData.precision === 20 &&
+            metaData.scale === 0
+          ) {
+            return { type: oracledb.STRING }
           }
           return undefined
         },
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index a367c6da1e..71de056814 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -315,13 +315,6 @@ export async function outputProcessing<T extends Row[] | Row>(
           column.subtype
         )
       }
-    } else if (column.type === FieldType.BIGINT) {
-      for (const row of enriched) {
-        if (row[property] == null) {
-          continue
-        }
-        row[property] = row[property].toString()
-      }
     }
   }
 

From 5cb294f33e5b1b0fd137ce8ae513271deb28b3a6 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 29 Jul 2024 14:54:58 +0100
Subject: [PATCH 04/34] Fix binding mismatch problem in oneOf queries.

---
 packages/backend-core/src/sql/sql.ts   | 7 +++----
 packages/backend-core/src/sql/utils.ts | 1 +
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 621afe7f3e..797e4b646a 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -394,10 +394,9 @@ class InternalBuilder {
         (key: string, array) => {
           if (this.client === SqlClient.ORACLE) {
             key = convertClobs(this.client, table, key)
-            query = query.whereRaw(
-              `${key} IN (?)`,
-              Array.isArray(array) ? array : [array]
-            )
+            array = Array.isArray(array) ? array : [array]
+            const binding = new Array(array.length).fill("?").join(",")
+            query = query.whereRaw(`${key} IN (${binding})`, array)
           } else {
             query = query[fnc](key, Array.isArray(array) ? array : [array])
           }
diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts
index 67b5d2081b..1b32cc6da7 100644
--- a/packages/backend-core/src/sql/utils.ts
+++ b/packages/backend-core/src/sql/utils.ts
@@ -22,6 +22,7 @@ export function getNativeSql(
   query: Knex.SchemaBuilder | Knex.QueryBuilder
 ): SqlQuery | SqlQuery[] {
   let sql = query.toSQL()
+
   if (Array.isArray(sql)) {
     return sql as SqlQuery[]
   }

From ad414b982e1cf58075ccce98932c495329523cc6 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 29 Jul 2024 16:54:59 +0100
Subject: [PATCH 05/34] Gone some way toward getting time-only fields to work.
 Still test failures though.

---
 packages/backend-core/src/sql/sql.ts          | 398 +++++++++---------
 .../src/api/routes/tests/search.spec.ts       |   2 +-
 packages/server/src/integrations/oracle.ts    |   1 -
 .../src/utilities/rowProcessor/index.ts       |   9 +
 4 files changed, 206 insertions(+), 204 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 797e4b646a..5d0a251900 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -42,176 +42,6 @@ const envLimit = environment.SQL_MAX_ROWS
   : null
 const BASE_LIMIT = envLimit || 5000
 
-// Takes a string like foo and returns a quoted string like [foo] for SQL Server
-// and "foo" for Postgres.
-function quote(client: SqlClient, str: string): string {
-  switch (client) {
-    case SqlClient.SQL_LITE:
-    case SqlClient.ORACLE:
-    case SqlClient.POSTGRES:
-      return `"${str}"`
-    case SqlClient.MS_SQL:
-      return `[${str}]`
-    case SqlClient.MY_SQL:
-      return `\`${str}\``
-  }
-}
-
-// Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c]
-// for SQL Server and `a`.`b`.`c` for MySQL.
-function quotedIdentifier(client: SqlClient, key: string): string {
-  return key
-    .split(".")
-    .map(part => quote(client, part))
-    .join(".")
-}
-
-function parse(input: any) {
-  if (Array.isArray(input)) {
-    return JSON.stringify(input)
-  }
-  if (input == undefined) {
-    return null
-  }
-  if (typeof input !== "string") {
-    return input
-  }
-  if (isInvalidISODateString(input)) {
-    return null
-  }
-  if (isValidISODateString(input)) {
-    return new Date(input.trim())
-  }
-  return input
-}
-
-function parseBody(body: any) {
-  for (let [key, value] of Object.entries(body)) {
-    body[key] = parse(value)
-  }
-  return body
-}
-
-function parseFilters(filters: SearchFilters | undefined): SearchFilters {
-  if (!filters) {
-    return {}
-  }
-  for (let [key, value] of Object.entries(filters)) {
-    let parsed
-    if (typeof value === "object") {
-      parsed = parseFilters(value)
-    } else {
-      parsed = parse(value)
-    }
-    // @ts-ignore
-    filters[key] = parsed
-  }
-  return filters
-}
-
-// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
-// so when we use them we need to wrap them in to_char(). This function
-// converts a field name to the appropriate identifier.
-function convertClobs(client: SqlClient, table: Table, field: string): string {
-  const parts = field.split(".")
-  const col = parts.pop()!
-  const schema = table.schema[col]
-  let identifier = quotedIdentifier(client, field)
-  if (
-    schema.type === FieldType.STRING ||
-    schema.type === FieldType.LONGFORM ||
-    schema.type === FieldType.BB_REFERENCE_SINGLE ||
-    schema.type === FieldType.OPTIONS ||
-    schema.type === FieldType.BARCODEQR
-  ) {
-    identifier = `to_char(${identifier})`
-  }
-  return identifier
-}
-
-function generateSelectStatement(
-  json: QueryJson,
-  knex: Knex
-): (string | Knex.Raw)[] | "*" {
-  const { resource, meta } = json
-  const client = knex.client.config.client as SqlClient
-
-  if (!resource || !resource.fields || resource.fields.length === 0) {
-    return "*"
-  }
-
-  const schema = meta.table.schema
-  return resource.fields.map(field => {
-    const parts = field.split(/\./g)
-    let table: string | undefined = undefined
-    let column: string | undefined = undefined
-
-    // Just a column name, e.g.: "column"
-    if (parts.length === 1) {
-      column = parts[0]
-    }
-
-    // A table name and a column name, e.g.: "table.column"
-    if (parts.length === 2) {
-      table = parts[0]
-      column = parts[1]
-    }
-
-    // A link doc, e.g.: "table.doc1.fieldName"
-    if (parts.length > 2) {
-      table = parts[0]
-      column = parts.slice(1).join(".")
-    }
-
-    if (!column) {
-      throw new Error(`Invalid field name: ${field}`)
-    }
-
-    const columnSchema = schema[column]
-
-    if (
-      client === SqlClient.POSTGRES &&
-      columnSchema?.externalType?.includes("money")
-    ) {
-      return knex.raw(
-        `${quotedIdentifier(
-          client,
-          [table, column].join(".")
-        )}::money::numeric as ${quote(client, field)}`
-      )
-    }
-
-    if (
-      client === SqlClient.MS_SQL &&
-      columnSchema?.type === FieldType.DATETIME &&
-      columnSchema.timeOnly
-    ) {
-      // Time gets returned as timestamp from mssql, not matching the expected
-      // HH:mm format
-      return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
-    }
-
-    // There's at least two edge cases being handled in the expression below.
-    //  1. The column name could start/end with a space, and in that case we
-    //     want to preseve that space.
-    //  2. Almost all column names are specified in the form table.column, except
-    //     in the case of relationships, where it's table.doc1.column. In that
-    //     case, we want to split it into `table`.`doc1.column` for reasons that
-    //     aren't actually clear to me, but `table`.`doc1` breaks things with the
-    //     sample data tests.
-    if (table) {
-      return knex.raw(
-        `${quote(client, table)}.${quote(client, column)} as ${quote(
-          client,
-          field
-        )}`
-      )
-    } else {
-      return knex.raw(`${quote(client, field)} as ${quote(client, field)}`)
-    }
-  })
-}
-
 function getTableName(table?: Table): string | undefined {
   // SQS uses the table ID rather than the table name
   if (
@@ -247,6 +77,181 @@ class InternalBuilder {
     this.client = client
   }
 
+  // Takes a string like foo and returns a quoted string like [foo] for SQL Server
+  // and "foo" for Postgres.
+  private quote(str: string): string {
+    switch (this.client) {
+      case SqlClient.SQL_LITE:
+      case SqlClient.ORACLE:
+      case SqlClient.POSTGRES:
+        return `"${str}"`
+      case SqlClient.MS_SQL:
+        return `[${str}]`
+      case SqlClient.MY_SQL:
+        return `\`${str}\``
+    }
+  }
+
+  // Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c]
+  // for SQL Server and `a`.`b`.`c` for MySQL.
+  private quotedIdentifier(key: string): string {
+    return key
+      .split(".")
+      .map(part => this.quote(part))
+      .join(".")
+  }
+
+  private generateSelectStatement(
+    json: QueryJson,
+    knex: Knex
+  ): (string | Knex.Raw)[] | "*" {
+    const { resource, meta } = json
+    const client = knex.client.config.client as SqlClient
+
+    if (!resource || !resource.fields || resource.fields.length === 0) {
+      return "*"
+    }
+
+    const schema = meta.table.schema
+    return resource.fields.map(field => {
+      const parts = field.split(/\./g)
+      let table: string | undefined = undefined
+      let column: string | undefined = undefined
+
+      // Just a column name, e.g.: "column"
+      if (parts.length === 1) {
+        column = parts[0]
+      }
+
+      // A table name and a column name, e.g.: "table.column"
+      if (parts.length === 2) {
+        table = parts[0]
+        column = parts[1]
+      }
+
+      // A link doc, e.g.: "table.doc1.fieldName"
+      if (parts.length > 2) {
+        table = parts[0]
+        column = parts.slice(1).join(".")
+      }
+
+      if (!column) {
+        throw new Error(`Invalid field name: ${field}`)
+      }
+
+      const columnSchema = schema[column]
+
+      if (
+        client === SqlClient.POSTGRES &&
+        columnSchema?.externalType?.includes("money")
+      ) {
+        return knex.raw(
+          `${this.quotedIdentifier(
+            [table, column].join(".")
+          )}::money::numeric as ${this.quote(field)}`
+        )
+      }
+
+      if (
+        client === SqlClient.MS_SQL &&
+        columnSchema?.type === FieldType.DATETIME &&
+        columnSchema.timeOnly
+      ) {
+        // Time gets returned as timestamp from mssql, not matching the expected
+        // HH:mm format
+        return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
+      }
+
+      // There's at least two edge cases being handled in the expression below.
+      //  1. The column name could start/end with a space, and in that case we
+      //     want to preseve that space.
+      //  2. Almost all column names are specified in the form table.column, except
+      //     in the case of relationships, where it's table.doc1.column. In that
+      //     case, we want to split it into `table`.`doc1.column` for reasons that
+      //     aren't actually clear to me, but `table`.`doc1` breaks things with the
+      //     sample data tests.
+      if (table) {
+        return knex.raw(
+          `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}`
+        )
+      } else {
+        return knex.raw(`${this.quote(field)} as ${this.quote(field)}`)
+      }
+    })
+  }
+
+  // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
+  // so when we use them we need to wrap them in to_char(). This function
+  // converts a field name to the appropriate identifier.
+  private convertClobs(table: Table, field: string): string {
+    const parts = field.split(".")
+    const col = parts.pop()!
+    const schema = table.schema[col]
+    let identifier = this.quotedIdentifier(field)
+    if (
+      schema.type === FieldType.STRING ||
+      schema.type === FieldType.LONGFORM ||
+      schema.type === FieldType.BB_REFERENCE_SINGLE ||
+      schema.type === FieldType.OPTIONS ||
+      schema.type === FieldType.BARCODEQR
+    ) {
+      identifier = `to_char(${identifier})`
+    }
+    return identifier
+  }
+
+  private parse(input: any, schema: FieldSchema) {
+    if (schema.type === FieldType.DATETIME && schema.timeOnly) {
+      if (this.client === SqlClient.ORACLE) {
+        return new Date(`1970-01-01 ${input}`)
+      }
+    }
+
+    if (Array.isArray(input)) {
+      return JSON.stringify(input)
+    }
+    if (input == undefined) {
+      return null
+    }
+    if (typeof input !== "string") {
+      return input
+    }
+    if (isInvalidISODateString(input)) {
+      return null
+    }
+    if (isValidISODateString(input)) {
+      return new Date(input.trim())
+    }
+    return input
+  }
+
+  private parseBody(body: any, table: Table) {
+    for (let [key, value] of Object.entries(body)) {
+      body[key] = this.parse(value, table.schema[key])
+    }
+    return body
+  }
+
+  private parseFilters(
+    filters: SearchFilters | undefined,
+    table: Table
+  ): SearchFilters {
+    if (!filters) {
+      return {}
+    }
+    for (let [key, value] of Object.entries(filters)) {
+      let parsed
+      if (typeof value === "object") {
+        parsed = this.parseFilters(value, table)
+      } else {
+        parsed = this.parse(value, table.schema[key])
+      }
+      // @ts-ignore
+      filters[key] = parsed
+    }
+    return filters
+  }
+
   // right now we only do filters on the specific table being queried
   addFilters(
     query: Knex.QueryBuilder,
@@ -261,7 +266,7 @@ class InternalBuilder {
     if (!filters) {
       return query
     }
-    filters = parseFilters(filters)
+    filters = this.parseFilters(filters, table)
     // if all or specified in filters, then everything is an or
     const allOr = filters.allOr
     const sqlStatements = new SqlStatements(this.client, table, {
@@ -318,10 +323,9 @@ class InternalBuilder {
       } else {
         const rawFnc = `${fnc}Raw`
         // @ts-ignore
-        query = query[rawFnc](
-          `LOWER(${quotedIdentifier(this.client, key)}) LIKE ?`,
-          [`%${value.toLowerCase()}%`]
-        )
+        query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
+          `%${value.toLowerCase()}%`,
+        ])
       }
     }
 
@@ -371,10 +375,7 @@ class InternalBuilder {
             }
             statement +=
               (statement ? andOr : "") +
-              `COALESCE(LOWER(${quotedIdentifier(
-                this.client,
-                key
-              )}), '') LIKE ?`
+              `COALESCE(LOWER(${this.quotedIdentifier(key)}), '') LIKE ?`
           }
 
           if (statement === "") {
@@ -393,7 +394,7 @@ class InternalBuilder {
         filters.oneOf,
         (key: string, array) => {
           if (this.client === SqlClient.ORACLE) {
-            key = convertClobs(this.client, table, key)
+            key = this.convertClobs(table, key)
             array = Array.isArray(array) ? array : [array]
             const binding = new Array(array.length).fill("?").join(",")
             query = query.whereRaw(`${key} IN (${binding})`, array)
@@ -415,10 +416,9 @@ class InternalBuilder {
         } else {
           const rawFnc = `${fnc}Raw`
           // @ts-ignore
-          query = query[rawFnc](
-            `LOWER(${quotedIdentifier(this.client, key)}) LIKE ?`,
-            [`${value.toLowerCase()}%`]
-          )
+          query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
+            `${value.toLowerCase()}%`,
+          ])
         }
       })
     }
@@ -456,21 +456,18 @@ class InternalBuilder {
         const fnc = allOr ? "orWhereRaw" : "whereRaw"
         if (this.client === SqlClient.MS_SQL) {
           query = query[fnc](
-            `CASE WHEN ${quotedIdentifier(
-              this.client,
-              key
-            )} = ? THEN 1 ELSE 0 END = 1`,
+            `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`,
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
-          const identifier = convertClobs(this.client, table, key)
+          const identifier = this.convertClobs(table, key)
           query = query[fnc](
             `(${identifier} IS NOT NULL AND ${identifier} = ?)`,
             [value]
           )
         } else {
           query = query[fnc](
-            `COALESCE(${quotedIdentifier(this.client, key)} = ?, FALSE)`,
+            `COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`,
             [value]
           )
         }
@@ -481,21 +478,18 @@ class InternalBuilder {
         const fnc = allOr ? "orWhereRaw" : "whereRaw"
         if (this.client === SqlClient.MS_SQL) {
           query = query[fnc](
-            `CASE WHEN ${quotedIdentifier(
-              this.client,
-              key
-            )} = ? THEN 1 ELSE 0 END = 0`,
+            `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`,
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
-          const identifier = convertClobs(this.client, table, key)
+          const identifier = this.convertClobs(table, key)
           query = query[fnc](
             `(${identifier} IS NOT NULL AND ${identifier} != ?)`,
             [value]
           )
         } else {
           query = query[fnc](
-            `COALESCE(${quotedIdentifier(this.client, key)} != ?, TRUE)`,
+            `COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`,
             [value]
           )
         }
@@ -692,7 +686,7 @@ class InternalBuilder {
   create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, body } = json
     let query = this.knexWithAlias(knex, endpoint)
-    const parsedBody = parseBody(body)
+    const parsedBody = this.parseBody(body, json.meta.table)
     // make sure no null values in body for creation
     for (let [key, value] of Object.entries(parsedBody)) {
       if (value == null) {
@@ -714,7 +708,7 @@ class InternalBuilder {
     if (!Array.isArray(body)) {
       return query
     }
-    const parsedBody = body.map(row => parseBody(row))
+    const parsedBody = body.map(row => this.parseBody(row, json.meta.table))
     return query.insert(parsedBody)
   }
 
@@ -724,7 +718,7 @@ class InternalBuilder {
     if (!Array.isArray(body)) {
       return query
     }
-    const parsedBody = body.map(row => parseBody(row))
+    const parsedBody = body.map(row => this.parseBody(row, json.meta.table))
     if (
       this.client === SqlClient.POSTGRES ||
       this.client === SqlClient.SQL_LITE ||
@@ -806,7 +800,7 @@ class InternalBuilder {
     })
     // if counting, use distinct count, else select
     preQuery = !counting
-      ? preQuery.select(generateSelectStatement(json, knex))
+      ? preQuery.select(this.generateSelectStatement(json, knex))
       : this.addDistinctCount(preQuery, json)
     // have to add after as well (this breaks MS-SQL)
     if (this.client !== SqlClient.MS_SQL && !counting) {
@@ -837,7 +831,7 @@ class InternalBuilder {
   update(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, body, filters, tableAliases } = json
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
-    const parsedBody = parseBody(body)
+    const parsedBody = this.parseBody(body, json.meta.table)
     query = this.addFilters(query, filters, json.meta.table, {
       columnPrefix: json.meta.columnPrefix,
       aliases: tableAliases,
@@ -861,7 +855,7 @@ class InternalBuilder {
     if (opts.disableReturning) {
       return query.delete()
     } else {
-      return query.delete().returning(generateSelectStatement(json, knex))
+      return query.delete().returning(this.generateSelectStatement(json, knex))
     }
   }
 }
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 4c7410eb76..c6c5786e53 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1318,7 +1318,7 @@ describe.each([
   })
 
   !isInternal &&
-    describe("datetime - time only", () => {
+    describe.only("datetime - time only", () => {
       const T_1000 = "10:00:00"
       const T_1045 = "10:45:00"
       const T_1200 = "12:00:00"
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index 6b86fba00d..b8fcd63e7f 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -398,7 +398,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
   }
 
   private getConnection = async (): Promise<Connection> => {
-    //connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))"
     const connectString = `${this.config.host}:${this.config.port || 1521}/${
       this.config.database
     }`
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 71de056814..139f3a5b8d 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -315,6 +315,15 @@ export async function outputProcessing<T extends Row[] | Row>(
           column.subtype
         )
       }
+    } else if (column.type === FieldType.DATETIME && column.timeOnly) {
+      for (let row of enriched) {
+        if (row[property] instanceof Date) {
+          const hours = row[property].getHours().toString().padStart(2, "0")
+          const minutes = row[property].getMinutes().toString().padStart(2, "0")
+          const seconds = row[property].getSeconds().toString().padStart(2, "0")
+          row[property] = `${hours}:${minutes}:${seconds}`
+        }
+      }
     }
   }
 

From e1ef66bf56df1fc8bd95832743a62dbc8329035c Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 29 Jul 2024 18:11:05 +0100
Subject: [PATCH 06/34] Refactor InternalBuilder to give me more access to
 query state.

---
 packages/backend-core/src/sql/sql.ts          | 253 +++++++++---------
 .../src/api/routes/tests/search.spec.ts       |  64 ++---
 .../src/sdk/app/rows/search/external.ts       |   3 +-
 3 files changed, 158 insertions(+), 162 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 5d0a251900..fd24c8e12c 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -34,6 +34,8 @@ import {
 } from "@budibase/types"
 import environment from "../environment"
 import { helpers } from "@budibase/shared-core"
+import { isPlainObject } from "lodash"
+import { ColumnSplitter } from "@budibase/shared-core/src/filters"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
 
@@ -72,9 +74,15 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
 
 class InternalBuilder {
   private readonly client: SqlClient
+  private readonly query: QueryJson
 
-  constructor(client: SqlClient) {
+  constructor(client: SqlClient, query: QueryJson) {
     this.client = client
+    this.query = query
+  }
+
+  get table(): Table {
+    return this.query.meta.table
   }
 
   // Takes a string like foo and returns a quoted string like [foo] for SQL Server
@@ -101,11 +109,8 @@ class InternalBuilder {
       .join(".")
   }
 
-  private generateSelectStatement(
-    json: QueryJson,
-    knex: Knex
-  ): (string | Knex.Raw)[] | "*" {
-    const { resource, meta } = json
+  private generateSelectStatement(knex: Knex): (string | Knex.Raw)[] | "*" {
+    const { resource, meta } = this.query
     const client = knex.client.config.client as SqlClient
 
     if (!resource || !resource.fields || resource.fields.length === 0) {
@@ -183,10 +188,10 @@ class InternalBuilder {
   // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
   // so when we use them we need to wrap them in to_char(). This function
   // converts a field name to the appropriate identifier.
-  private convertClobs(table: Table, field: string): string {
+  private convertClobs(field: string): string {
     const parts = field.split(".")
     const col = parts.pop()!
-    const schema = table.schema[col]
+    const schema = this.table.schema[col]
     let identifier = this.quotedIdentifier(field)
     if (
       schema.type === FieldType.STRING ||
@@ -201,54 +206,60 @@ class InternalBuilder {
   }
 
   private parse(input: any, schema: FieldSchema) {
+    if (input == undefined) {
+      return null
+    }
+
+    if (isPlainObject(input)) {
+      for (const [key, value] of Object.entries(input)) {
+        input[key] = this.parse(value, schema)
+      }
+      return input
+    }
+
     if (schema.type === FieldType.DATETIME && schema.timeOnly) {
       if (this.client === SqlClient.ORACLE) {
         return new Date(`1970-01-01 ${input}`)
       }
     }
 
-    if (Array.isArray(input)) {
-      return JSON.stringify(input)
-    }
-    if (input == undefined) {
-      return null
-    }
-    if (typeof input !== "string") {
-      return input
-    }
-    if (isInvalidISODateString(input)) {
-      return null
-    }
-    if (isValidISODateString(input)) {
-      return new Date(input.trim())
+    if (typeof input === "string") {
+      if (isInvalidISODateString(input)) {
+        return null
+      }
+      if (isValidISODateString(input)) {
+        return new Date(input.trim())
+      }
     }
+
     return input
   }
 
-  private parseBody(body: any, table: Table) {
+  private parseBody(body: any) {
     for (let [key, value] of Object.entries(body)) {
-      body[key] = this.parse(value, table.schema[key])
+      body[key] = this.parse(value, this.table.schema[key])
     }
     return body
   }
 
-  private parseFilters(
-    filters: SearchFilters | undefined,
-    table: Table
-  ): SearchFilters {
+  private parseFilters(filters: SearchFilters | undefined): SearchFilters {
     if (!filters) {
       return {}
     }
-    for (let [key, value] of Object.entries(filters)) {
-      let parsed
-      if (typeof value === "object") {
-        parsed = this.parseFilters(value, table)
-      } else {
-        parsed = this.parse(value, table.schema[key])
+
+    for (const [_, filter] of Object.entries(filters)) {
+      for (const [key, value] of Object.entries(filter)) {
+        const { column } = new ColumnSplitter([this.table]).run(key)
+        const schema = this.table.schema[column]
+        if (!schema) {
+          throw new Error(
+            `Column ${key} does not exist in table ${this.table._id}`
+          )
+        }
+        filter[key] = this.parse(value, schema)
       }
-      // @ts-ignore
-      filters[key] = parsed
     }
+
     return filters
   }
 
@@ -256,28 +267,26 @@ class InternalBuilder {
   addFilters(
     query: Knex.QueryBuilder,
     filters: SearchFilters | undefined,
-    table: Table,
-    opts: {
-      aliases?: Record<string, string>
+    opts?: {
       relationship?: boolean
-      columnPrefix?: string
     }
   ): Knex.QueryBuilder {
     if (!filters) {
       return query
     }
-    filters = this.parseFilters(filters, table)
+    filters = this.parseFilters(filters)
+    const aliases = this.query.tableAliases
     // if all or specified in filters, then everything is an or
     const allOr = filters.allOr
-    const sqlStatements = new SqlStatements(this.client, table, {
+    const sqlStatements = new SqlStatements(this.client, this.table, {
       allOr,
-      columnPrefix: opts.columnPrefix,
+      columnPrefix: this.query.meta.columnPrefix,
     })
     const tableName =
-      this.client === SqlClient.SQL_LITE ? table._id! : table.name
+      this.client === SqlClient.SQL_LITE ? this.table._id! : this.table.name
 
     function getTableAlias(name: string) {
-      const alias = opts.aliases?.[name]
+      const alias = aliases?.[name]
       return alias || name
     }
     function iterate(
@@ -303,10 +312,10 @@ class InternalBuilder {
             ),
             castedTypeValue.values
           )
-        } else if (!opts.relationship && !isRelationshipField) {
+        } else if (!opts?.relationship && !isRelationshipField) {
           const alias = getTableAlias(tableName)
           fn(alias ? `${alias}.${updatedKey}` : updatedKey, value)
-        } else if (opts.relationship && isRelationshipField) {
+        } else if (opts?.relationship && isRelationshipField) {
           const [filterTableName, property] = updatedKey.split(".")
           const alias = getTableAlias(filterTableName)
           fn(alias ? `${alias}.${property}` : property, value)
@@ -394,7 +403,7 @@ class InternalBuilder {
         filters.oneOf,
         (key: string, array) => {
           if (this.client === SqlClient.ORACLE) {
-            key = this.convertClobs(table, key)
+            key = this.convertClobs(key)
             array = Array.isArray(array) ? array : [array]
             const binding = new Array(array.length).fill("?").join(",")
             query = query.whereRaw(`${key} IN (${binding})`, array)
@@ -460,7 +469,7 @@ class InternalBuilder {
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
-          const identifier = this.convertClobs(table, key)
+          const identifier = this.convertClobs(key)
           query = query[fnc](
             `(${identifier} IS NOT NULL AND ${identifier} = ?)`,
             [value]
@@ -482,7 +491,7 @@ class InternalBuilder {
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
-          const identifier = this.convertClobs(table, key)
+          const identifier = this.convertClobs(key)
           query = query[fnc](
             `(${identifier} IS NOT NULL AND ${identifier} != ?)`,
             [value]
@@ -517,9 +526,9 @@ class InternalBuilder {
       contains(filters.containsAny, true)
     }
 
-    const tableRef = opts?.aliases?.[table._id!] || table._id
+    const tableRef = aliases?.[this.table._id!] || this.table._id
     // when searching internal tables make sure long looking for rows
-    if (filters.documentType && !isExternalTable(table) && tableRef) {
+    if (filters.documentType && !isExternalTable(this.table) && tableRef) {
       // has to be its own option, must always be AND onto the search
       query.andWhereLike(
         `${tableRef}._id`,
@@ -530,29 +539,26 @@ class InternalBuilder {
     return query
   }
 
-  addDistinctCount(
-    query: Knex.QueryBuilder,
-    json: QueryJson
-  ): Knex.QueryBuilder {
-    const table = json.meta.table
-    const primary = table.primary
-    const aliases = json.tableAliases
+  addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder {
+    const primary = this.table.primary
+    const aliases = this.query.tableAliases
     const aliased =
-      table.name && aliases?.[table.name] ? aliases[table.name] : table.name
+      this.table.name && aliases?.[this.table.name]
+        ? aliases[this.table.name]
+        : this.table.name
     if (!primary) {
       throw new Error("SQL counting requires primary key to be supplied")
     }
     return query.countDistinct(`${aliased}.${primary[0]} as total`)
   }
 
-  addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
-    let { sort } = json
-    const table = json.meta.table
-    const primaryKey = table.primary
-    const tableName = getTableName(table)
-    const aliases = json.tableAliases
+  addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
+    let { sort } = this.query
+    const primaryKey = this.table.primary
+    const tableName = getTableName(this.table)
+    const aliases = this.query.tableAliases
     const aliased =
-      tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
+      tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name
     if (!Array.isArray(primaryKey)) {
       throw new Error("Sorting requires primary key to be specified for table")
     }
@@ -667,26 +673,28 @@ class InternalBuilder {
     return query
   }
 
-  knexWithAlias(
+  qualifiedKnex(
     knex: Knex,
-    endpoint: QueryJson["endpoint"],
-    aliases?: QueryJson["tableAliases"]
+    opts?: { alias?: string | boolean }
   ): Knex.QueryBuilder {
-    const tableName = endpoint.entityId
-    const tableAlias = aliases?.[tableName]
-
+    let alias = this.query.tableAliases?.[this.query.endpoint.entityId]
+    if (opts?.alias === false) {
+      alias = undefined
+    } else if (typeof opts?.alias === "string") {
+      alias = opts.alias
+    }
     return knex(
-      this.tableNameWithSchema(tableName, {
-        alias: tableAlias,
-        schema: endpoint.schema,
+      this.tableNameWithSchema(this.query.endpoint.entityId, {
+        alias,
+        schema: this.query.endpoint.schema,
       })
     )
   }
 
-  create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
-    const { endpoint, body } = json
-    let query = this.knexWithAlias(knex, endpoint)
-    const parsedBody = this.parseBody(body, json.meta.table)
+  create(knex: Knex, opts: QueryOptions): Knex.QueryBuilder {
+    const { body } = this.query
+    let query = this.qualifiedKnex(knex, { alias: false })
+    const parsedBody = this.parseBody(body)
     // make sure no null values in body for creation
     for (let [key, value] of Object.entries(parsedBody)) {
       if (value == null) {
@@ -702,29 +710,29 @@ class InternalBuilder {
     }
   }
 
-  bulkCreate(knex: Knex, json: QueryJson): Knex.QueryBuilder {
-    const { endpoint, body } = json
-    let query = this.knexWithAlias(knex, endpoint)
+  bulkCreate(knex: Knex): Knex.QueryBuilder {
+    const { body } = this.query
+    let query = this.qualifiedKnex(knex, { alias: false })
     if (!Array.isArray(body)) {
       return query
     }
-    const parsedBody = body.map(row => this.parseBody(row, json.meta.table))
+    const parsedBody = body.map(row => this.parseBody(row))
     return query.insert(parsedBody)
   }
 
-  bulkUpsert(knex: Knex, json: QueryJson): Knex.QueryBuilder {
-    const { endpoint, body } = json
-    let query = this.knexWithAlias(knex, endpoint)
+  bulkUpsert(knex: Knex): Knex.QueryBuilder {
+    const { body } = this.query
+    let query = this.qualifiedKnex(knex, { alias: false })
     if (!Array.isArray(body)) {
       return query
     }
-    const parsedBody = body.map(row => this.parseBody(row, json.meta.table))
+    const parsedBody = body.map(row => this.parseBody(row))
     if (
       this.client === SqlClient.POSTGRES ||
       this.client === SqlClient.SQL_LITE ||
       this.client === SqlClient.MY_SQL
     ) {
-      const primary = json.meta.table.primary
+      const primary = this.table.primary
       if (!primary) {
         throw new Error("Primary key is required for upsert")
       }
@@ -743,18 +751,18 @@ class InternalBuilder {
 
   read(
     knex: Knex,
-    json: QueryJson,
     opts: {
       limits?: { base: number; query: number }
     } = {}
   ): Knex.QueryBuilder {
-    let { endpoint, filters, paginate, relationships, tableAliases } = json
+    let { endpoint, filters, paginate, relationships, tableAliases } =
+      this.query
     const { limits } = opts
     const counting = endpoint.operation === Operation.COUNT
 
     const tableName = endpoint.entityId
     // start building the query
-    let query = this.knexWithAlias(knex, endpoint, tableAliases)
+    let query = this.qualifiedKnex(knex)
     // handle pagination
     let foundOffset: number | null = null
     let foundLimit = limits?.query || limits?.base
@@ -782,13 +790,10 @@ class InternalBuilder {
       }
       // add sorting to pre-query
       // no point in sorting when counting
-      query = this.addSorting(query, json)
+      query = this.addSorting(query)
     }
     // add filters to the query (where)
-    query = this.addFilters(query, filters, json.meta.table, {
-      columnPrefix: json.meta.columnPrefix,
-      aliases: tableAliases,
-    })
+    query = this.addFilters(query, filters)
 
     const alias = tableAliases?.[tableName] || tableName
     let preQuery: Knex.QueryBuilder = knex({
@@ -800,11 +805,11 @@ class InternalBuilder {
     })
     // if counting, use distinct count, else select
     preQuery = !counting
-      ? preQuery.select(this.generateSelectStatement(json, knex))
-      : this.addDistinctCount(preQuery, json)
+      ? preQuery.select(this.generateSelectStatement(knex))
+      : this.addDistinctCount(preQuery)
     // have to add after as well (this breaks MS-SQL)
     if (this.client !== SqlClient.MS_SQL && !counting) {
-      preQuery = this.addSorting(preQuery, json)
+      preQuery = this.addSorting(preQuery)
     }
     // handle joins
     query = this.addRelationships(
@@ -821,21 +826,14 @@ class InternalBuilder {
       query = query.limit(limits.base)
     }
 
-    return this.addFilters(query, filters, json.meta.table, {
-      columnPrefix: json.meta.columnPrefix,
-      relationship: true,
-      aliases: tableAliases,
-    })
+    return this.addFilters(query, filters, { relationship: true })
   }
 
-  update(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
-    const { endpoint, body, filters, tableAliases } = json
-    let query = this.knexWithAlias(knex, endpoint, tableAliases)
-    const parsedBody = this.parseBody(body, json.meta.table)
-    query = this.addFilters(query, filters, json.meta.table, {
-      columnPrefix: json.meta.columnPrefix,
-      aliases: tableAliases,
-    })
+  update(knex: Knex, opts: QueryOptions): Knex.QueryBuilder {
+    const { body, filters } = this.query
+    let query = this.qualifiedKnex(knex)
+    const parsedBody = this.parseBody(body)
+    query = this.addFilters(query, filters)
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.update(parsedBody)
@@ -844,18 +842,15 @@ class InternalBuilder {
     }
   }
 
-  delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
-    const { endpoint, filters, tableAliases } = json
-    let query = this.knexWithAlias(knex, endpoint, tableAliases)
-    query = this.addFilters(query, filters, json.meta.table, {
-      columnPrefix: json.meta.columnPrefix,
-      aliases: tableAliases,
-    })
+  delete(knex: Knex, opts: QueryOptions): Knex.QueryBuilder {
+    const { filters } = this.query
+    let query = this.qualifiedKnex(knex)
+    query = this.addFilters(query, filters)
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()
     } else {
-      return query.delete().returning(this.generateSelectStatement(json, knex))
+      return query.delete().returning(this.generateSelectStatement(knex))
     }
   }
 }
@@ -899,13 +894,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
 
     const client = knex(config)
     let query: Knex.QueryBuilder
-    const builder = new InternalBuilder(sqlClient)
+    const builder = new InternalBuilder(sqlClient, json)
     switch (this._operation(json)) {
       case Operation.CREATE:
-        query = builder.create(client, json, opts)
+        query = builder.create(client, opts)
         break
       case Operation.READ:
-        query = builder.read(client, json, {
+        query = builder.read(client, {
           limits: {
             query: this.limit,
             base: BASE_LIMIT,
@@ -914,19 +909,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         break
       case Operation.COUNT:
         // read without any limits to count
-        query = builder.read(client, json)
+        query = builder.read(client)
         break
       case Operation.UPDATE:
-        query = builder.update(client, json, opts)
+        query = builder.update(client, opts)
         break
       case Operation.DELETE:
-        query = builder.delete(client, json, opts)
+        query = builder.delete(client, opts)
         break
       case Operation.BULK_CREATE:
-        query = builder.bulkCreate(client, json)
+        query = builder.bulkCreate(client)
         break
       case Operation.BULK_UPSERT:
-        query = builder.bulkUpsert(client, json)
+        query = builder.bulkUpsert(client)
         break
       case Operation.CREATE_TABLE:
       case Operation.UPDATE_TABLE:
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index c6c5786e53..a08fac7396 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -40,14 +40,14 @@ import { structures } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 
 describe.each([
-  // ["in-memory", undefined],
-  // ["lucene", undefined],
-  // ["sqs", undefined],
-  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+  ["in-memory", undefined],
+  ["lucene", undefined],
+  ["sqs", undefined],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
   const isLucene = name === "lucene"
@@ -1318,7 +1318,7 @@ describe.each([
   })
 
   !isInternal &&
-    describe.only("datetime - time only", () => {
+    describe("datetime - time only", () => {
       const T_1000 = "10:00:00"
       const T_1045 = "10:45:00"
       const T_1200 = "12:00:00"
@@ -2389,9 +2389,9 @@ describe.each([
 
   describe.each([
     { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" },
-    { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" },
-    { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
-    { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
+    // { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" },
+    // { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
+    // { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
   ])("date special cases", ({ low, high }) => {
     const earlyDate = "2024-07-03T10:00:00.000Z",
       laterDate = "2024-07-03T11:00:00.000Z"
@@ -2405,7 +2405,7 @@ describe.each([
       await createRows([{ date: earlyDate }, { date: laterDate }])
     })
 
-    it("should be able to handle a date search", async () => {
+    it.only("should be able to handle a date search", async () => {
       await expectSearch({
         query: {
           range: {
@@ -2418,25 +2418,25 @@ describe.each([
 
   describe.each([
     "名前", // Japanese for "name"
-    // "Benutzer-ID", // German for "user ID", includes a hyphen
-    // "numéro", // French for "number", includes an accent
-    // "år", // Swedish for "year", includes a ring above
-    // "naïve", // English word borrowed from French, includes an umlaut
-    // "الاسم", // Arabic for "name"
-    // "оплата", // Russian for "payment"
-    // "पता", // Hindi for "address"
-    // "用戶名", // Chinese for "username"
-    // "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
-    // "preço", // Portuguese for "price", includes a cedilla
-    // "사용자명", // Korean for "username"
-    // "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
-    // "файл", // Bulgarian for "file"
-    // "δεδομένα", // Greek for "data"
-    // "geändert_am", // German for "modified on", includes an umlaut
-    // "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
-    // "São_Paulo", // Portuguese, includes an underscore and a tilde
-    // "età", // Italian for "age", includes an accent
-    // "ชื่อผู้ใช้", // Thai for "username"
+    "Benutzer-ID", // German for "user ID", includes a hyphen
+    "numéro", // French for "number", includes an accent
+    "år", // Swedish for "year", includes a ring above
+    "naïve", // English word borrowed from French, includes an umlaut
+    "الاسم", // Arabic for "name"
+    "оплата", // Russian for "payment"
+    "पता", // Hindi for "address"
+    "用戶名", // Chinese for "username"
+    "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
+    "preço", // Portuguese for "price", includes a cedilla
+    "사용자명", // Korean for "username"
+    "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
+    "файл", // Bulgarian for "file"
+    "δεδομένα", // Greek for "data"
+    "geändert_am", // German for "modified on", includes an umlaut
+    "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
+    "São_Paulo", // Portuguese, includes an underscore and a tilde
+    "età", // Italian for "age", includes an accent
+    "ชื่อผู้ใช้", // Thai for "username"
   ])("non-ascii column name: %s", name => {
     beforeAll(async () => {
       table = await createTable({
diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts
index cd0650e4c4..815094eeeb 100644
--- a/packages/server/src/sdk/app/rows/search/external.ts
+++ b/packages/server/src/sdk/app/rows/search/external.ts
@@ -123,7 +123,8 @@ export async function search(
   } catch (err: any) {
     if (err.message && err.message.includes("does not exist")) {
       throw new Error(
-        `Table updated externally, please re-fetch - ${err.message}`
+        `Table updated externally, please re-fetch - ${err.message}`,
+        { cause: err }
       )
     } else {
       throw err

From 25ab2e26894bbecfbd4ff9aa60c1b1ec2db31768 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 29 Jul 2024 18:20:49 +0100
Subject: [PATCH 07/34] Revert schema plumbing, need to revisit.

---
 packages/backend-core/src/sql/sql.ts          | 109 ++++++++++++------
 .../src/api/routes/tests/search.spec.ts       |   2 +-
 2 files changed, 77 insertions(+), 34 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index fd24c8e12c..917182f7d2 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -205,39 +205,28 @@ class InternalBuilder {
     return identifier
   }
 
-  private parse(input: any, schema: FieldSchema) {
+  private parse(input: any) {
+    if (Array.isArray(input)) {
+      return JSON.stringify(input)
+    }
     if (input == undefined) {
       return null
     }
-
-    if (isPlainObject(input)) {
-      for (const [key, value] of Object.entries(input)) {
-        input[key] = this.parse(value, schema)
-      }
+    if (typeof input !== "string") {
       return input
     }
-
-    if (schema.type === FieldType.DATETIME && schema.timeOnly) {
-      if (this.client === SqlClient.ORACLE) {
-        return new Date(`1970-01-01 ${input}`)
-      }
+    if (isInvalidISODateString(input)) {
+      return null
     }
-
-    if (typeof input === "string") {
-      if (isInvalidISODateString(input)) {
-        return null
-      }
-      if (isValidISODateString(input)) {
-        return new Date(input.trim())
-      }
+    if (isValidISODateString(input)) {
+      return new Date(input.trim())
     }
-
     return input
   }
 
   private parseBody(body: any) {
     for (let [key, value] of Object.entries(body)) {
-      body[key] = this.parse(value, this.table.schema[key])
+      body[key] = this.parse(value)
     }
     return body
   }
@@ -246,23 +235,77 @@ class InternalBuilder {
     if (!filters) {
       return {}
     }
-
-    for (const [_, filter] of Object.entries(filters)) {
-      for (const [key, value] of Object.entries(filter)) {
-        const { column } = new ColumnSplitter([this.table]).run(key)
-        const schema = this.table.schema[column]
-        if (!schema) {
-          throw new Error(
-            `Column ${key} does not exist in table ${this.table._id}`
-          )
-        }
-        filter[key] = this.parse(value, schema)
+    for (let [key, value] of Object.entries(filters)) {
+      let parsed
+      if (typeof value === "object") {
+        parsed = this.parseFilters(value)
+      } else {
+        parsed = this.parse(value)
       }
+      // @ts-ignore
+      filters[key] = parsed
     }
-
     return filters
   }
 
+  // private parse(input: any, schema: FieldSchema) {
+  //   if (input == undefined) {
+  //     return null
+  //   }
+
+  //   if (isPlainObject(input)) {
+  //     for (const [key, value] of Object.entries(input)) {
+  //       input[key] = this.parse(value, schema)
+  //     }
+  //     return input
+  //   }
+
+  //   if (schema.type === FieldType.DATETIME && schema.timeOnly) {
+  //     if (this.client === SqlClient.ORACLE) {
+  //       return new Date(`1970-01-01 ${input}`)
+  //     }
+  //   }
+
+  //   if (typeof input === "string") {
+  //     if (isInvalidISODateString(input)) {
+  //       return null
+  //     }
+  //     if (isValidISODateString(input)) {
+  //       return new Date(input.trim())
+  //     }
+  //   }
+
+  //   return input
+  // }
+
+  // private parseBody(body: any) {
+  //   for (let [key, value] of Object.entries(body)) {
+  //     body[key] = this.parse(value, this.table.schema[key])
+  //   }
+  //   return body
+  // }
+
+  // private parseFilters(filters: SearchFilters | undefined): SearchFilters {
+  //   if (!filters) {
+  //     return {}
+  //   }
+
+  //   for (const [_, filter] of Object.entries(filters)) {
+  //     for (const [key, value] of Object.entries(filter)) {
+  //       const { column } = new ColumnSplitter([this.table]).run(key)
+  //       const schema = this.table.schema[column]
+  //       if (!schema) {
+  //         throw new Error(
+  //           `Column ${key} does not exist in table ${this.table._id}`
+  //         )
+  //       }
+  //       filter[key] = this.parse(value, schema)
+  //     }
+  //   }
+
+  //   return filters
+  // }
+
   // right now we only do filters on the specific table being queried
   addFilters(
     query: Knex.QueryBuilder,
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index a08fac7396..110a9ae699 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2405,7 +2405,7 @@ describe.each([
       await createRows([{ date: earlyDate }, { date: laterDate }])
     })
 
-    it.only("should be able to handle a date search", async () => {
+    it("should be able to handle a date search", async () => {
       await expectSearch({
         query: {
           range: {

From fe36b76fe9a16f610a7763c0d7a34d01a59bb971 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 29 Jul 2024 18:56:15 +0100
Subject: [PATCH 08/34] Getting something working - using a different version
 and a pre-built image.

---
 .../scripts/integrations/oracle/docker-compose.yml    |  6 +++---
 .../server/src/integrations/tests/utils/oracle.ts     | 11 +++++++----
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/packages/server/scripts/integrations/oracle/docker-compose.yml b/packages/server/scripts/integrations/oracle/docker-compose.yml
index 586f0b683d..07992b6544 100644
--- a/packages/server/scripts/integrations/oracle/docker-compose.yml
+++ b/packages/server/scripts/integrations/oracle/docker-compose.yml
@@ -6,9 +6,9 @@ services:
   db:
     restart: unless-stopped
     platform: linux/x86_64
-    image: container-registry.oracle.com/database/express:18.4.0-xe
+    image: gvenzl/oracle-free:23.2-slim-faststart
     environment:
-      ORACLE_PWD: oracle
+      ORACLE_PWD: Password1
     ports:
       - 1521:1521
       - 5500:5500
@@ -16,4 +16,4 @@ services:
       - oracle_data:/opt/oracle/oradata
 
 volumes:
- oracle_data:
\ No newline at end of file
+ oracle_data:
diff --git a/packages/server/src/integrations/tests/utils/oracle.ts b/packages/server/src/integrations/tests/utils/oracle.ts
index 9b75a52da7..c904c094e1 100644
--- a/packages/server/src/integrations/tests/utils/oracle.ts
+++ b/packages/server/src/integrations/tests/utils/oracle.ts
@@ -7,8 +7,10 @@ import knex from "knex"
 let ports: Promise<testContainerUtils.Port[]>
 
 export async function getDatasource(): Promise<Datasource> {
+  // password needs to conform to Oracle standards
+  const password = "password"
   if (!ports) {
-    let image = "oracle/database:19.3.0.0-ee-slim-faststart"
+    let image = "gvenzl/oracle-free:23.2-slim-faststart"
     if (process.arch.startsWith("arm")) {
       image = "samhuang78/oracle-database:19.3.0-ee-slim-faststart"
     }
@@ -16,8 +18,10 @@ export async function getDatasource(): Promise<Datasource> {
     ports = startContainer(
       new GenericContainer(image)
         .withExposedPorts(1521)
-        .withEnvironment({ ORACLE_PASSWORD: "password" })
-        .withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(60000))
+        .withEnvironment({
+          ORACLE_PASSWORD: password,
+        })
+        .withWaitStrategy(Wait.forLogMessage("DATABASE IS READY TO USE!"))
     )
   }
 
@@ -28,7 +32,6 @@ export async function getDatasource(): Promise<Datasource> {
 
   const host = "127.0.0.1"
   const user = "SYSTEM"
-  const password = "password"
 
   const datasource: Datasource = {
     type: "datasource_plus",

From d448f469f14ae1ed528f019d5bbf7b85214d442c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 29 Jul 2024 19:00:25 +0100
Subject: [PATCH 09/34] Updating image names.

---
 packages/server/src/integrations/tests/utils/oracle.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/integrations/tests/utils/oracle.ts b/packages/server/src/integrations/tests/utils/oracle.ts
index c904c094e1..5c788fd130 100644
--- a/packages/server/src/integrations/tests/utils/oracle.ts
+++ b/packages/server/src/integrations/tests/utils/oracle.ts
@@ -10,9 +10,11 @@ export async function getDatasource(): Promise<Datasource> {
   // password needs to conform to Oracle standards
   const password = "password"
   if (!ports) {
-    let image = "gvenzl/oracle-free:23.2-slim-faststart"
+    // couldn't build 19.3.0 for X64
+    let image = "budibase/oracle-database:23.2-slim-faststart"
     if (process.arch.startsWith("arm")) {
-      image = "samhuang78/oracle-database:19.3.0-ee-slim-faststart"
+      // there isn't an ARM compatible 23.2 build
+      image = "budibase/oracle-database:19.3.0-ee-slim-faststart"
     }
 
     ports = startContainer(

From 5bce8e595d109678090a7fdaa08baf7b71e91611 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 30 Jul 2024 11:03:54 +0100
Subject: [PATCH 10/34] Plumb FieldSchema into parse.

---
 packages/backend-core/src/sql/sql.ts          | 161 +++++++++---------
 .../server/src/sdk/app/rows/search/sqs.ts     |  10 +-
 .../server/src/sdk/app/tables/internal/sqs.ts |  22 +--
 packages/shared-core/src/filters.ts           |  43 ++++-
 packages/shared-core/src/helpers/schema.ts    |  19 +++
 5 files changed, 143 insertions(+), 112 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 917182f7d2..26d6545868 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -12,6 +12,8 @@ import { SqlStatements } from "./sqlStatements"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
   AnySearchFilter,
+  ArrayOperator,
+  BasicOperator,
   BBReferenceFieldMetadata,
   FieldSchema,
   FieldType,
@@ -23,6 +25,7 @@ import {
   prefixed,
   QueryJson,
   QueryOptions,
+  RangeOperator,
   RelationshipsJson,
   SearchFilters,
   SortOrder,
@@ -33,9 +36,7 @@ import {
   TableSourceType,
 } from "@budibase/types"
 import environment from "../environment"
-import { helpers } from "@budibase/shared-core"
-import { isPlainObject } from "lodash"
-import { ColumnSplitter } from "@budibase/shared-core/src/filters"
+import { dataFilters, helpers } from "@budibase/shared-core"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
 
@@ -75,10 +76,16 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
 class InternalBuilder {
   private readonly client: SqlClient
   private readonly query: QueryJson
+  private readonly splitter: dataFilters.ColumnSplitter
 
   constructor(client: SqlClient, query: QueryJson) {
     this.client = client
     this.query = query
+
+    this.splitter = new dataFilters.ColumnSplitter([this.table], {
+      aliases: this.query.tableAliases,
+      columnPrefix: this.query.meta.columnPrefix,
+    })
   }
 
   get table(): Table {
@@ -205,107 +212,95 @@ class InternalBuilder {
     return identifier
   }
 
-  private parse(input: any) {
+  private parse(input: any, schema: FieldSchema) {
     if (Array.isArray(input)) {
       return JSON.stringify(input)
     }
     if (input == undefined) {
       return null
     }
-    if (typeof input !== "string") {
-      return input
-    }
-    if (isInvalidISODateString(input)) {
-      return null
-    }
-    if (isValidISODateString(input)) {
-      return new Date(input.trim())
+    if (typeof input === "string") {
+      if (isInvalidISODateString(input)) {
+        return null
+      }
+      if (isValidISODateString(input)) {
+        return new Date(input.trim())
+      }
     }
     return input
   }
 
   private parseBody(body: any) {
     for (let [key, value] of Object.entries(body)) {
-      body[key] = this.parse(value)
+      const { column } = this.splitter.run(key)
+      const schema = this.table.schema[column]
+      if (!schema) {
+        continue
+      }
+      body[key] = this.parse(value, schema)
     }
     return body
   }
 
-  private parseFilters(filters: SearchFilters | undefined): SearchFilters {
-    if (!filters) {
-      return {}
-    }
-    for (let [key, value] of Object.entries(filters)) {
-      let parsed
-      if (typeof value === "object") {
-        parsed = this.parseFilters(value)
-      } else {
-        parsed = this.parse(value)
+  private parseFilters(filters: SearchFilters): SearchFilters {
+    for (const op of Object.values(BasicOperator)) {
+      const filter = filters[op]
+      if (!filter) {
+        continue
+      }
+      for (const key of Object.keys(filter)) {
+        if (Array.isArray(filter[key])) {
+          filter[key] = JSON.stringify(filter[key])
+          continue
+        }
+        const { column } = this.splitter.run(key)
+        const schema = this.table.schema[column]
+        if (!schema) {
+          continue
+        }
+        filter[key] = this.parse(filter[key], schema)
       }
-      // @ts-ignore
-      filters[key] = parsed
     }
+
+    for (const op of Object.values(ArrayOperator)) {
+      const filter = filters[op]
+      if (!filter) {
+        continue
+      }
+      for (const key of Object.keys(filter)) {
+        const { column } = this.splitter.run(key)
+        const schema = this.table.schema[column]
+        if (!schema) {
+          continue
+        }
+        filter[key] = filter[key].map(v => this.parse(v, schema))
+      }
+    }
+
+    for (const op of Object.values(RangeOperator)) {
+      const filter = filters[op]
+      if (!filter) {
+        continue
+      }
+      for (const key of Object.keys(filter)) {
+        const { column } = this.splitter.run(key)
+        const schema = this.table.schema[column]
+        if (!schema) {
+          continue
+        }
+        const value = filter[key]
+        if ("low" in value) {
+          value.low = this.parse(value.low, schema)
+        }
+        if ("high" in value) {
+          value.high = this.parse(value.high, schema)
+        }
+      }
+    }
+
     return filters
   }
 
-  // private parse(input: any, schema: FieldSchema) {
-  //   if (input == undefined) {
-  //     return null
-  //   }
-
-  //   if (isPlainObject(input)) {
-  //     for (const [key, value] of Object.entries(input)) {
-  //       input[key] = this.parse(value, schema)
-  //     }
-  //     return input
-  //   }
-
-  //   if (schema.type === FieldType.DATETIME && schema.timeOnly) {
-  //     if (this.client === SqlClient.ORACLE) {
-  //       return new Date(`1970-01-01 ${input}`)
-  //     }
-  //   }
-
-  //   if (typeof input === "string") {
-  //     if (isInvalidISODateString(input)) {
-  //       return null
-  //     }
-  //     if (isValidISODateString(input)) {
-  //       return new Date(input.trim())
-  //     }
-  //   }
-
-  //   return input
-  // }
-
-  // private parseBody(body: any) {
-  //   for (let [key, value] of Object.entries(body)) {
-  //     body[key] = this.parse(value, this.table.schema[key])
-  //   }
-  //   return body
-  // }
-
-  // private parseFilters(filters: SearchFilters | undefined): SearchFilters {
-  //   if (!filters) {
-  //     return {}
-  //   }
-
-  //   for (const [_, filter] of Object.entries(filters)) {
-  //     for (const [key, value] of Object.entries(filter)) {
-  //       const { column } = new ColumnSplitter([this.table]).run(key)
-  //       const schema = this.table.schema[column]
-  //       if (!schema) {
-  //         throw new Error(
-  //           `Column ${key} does not exist in table ${this.table._id}`
-  //         )
-  //       }
-  //       filter[key] = this.parse(value, schema)
-  //     }
-  //   }
-
-  //   return filters
-  // }
-
   // right now we only do filters on the specific table being queried
   addFilters(
     query: Knex.QueryBuilder,
diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 44fd718871..650321a9a7 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -19,11 +19,7 @@ import {
   buildInternalRelationships,
   sqlOutputProcessing,
 } from "../../../../api/controllers/row/utils"
-import {
-  decodeNonAscii,
-  mapToUserColumn,
-  USER_COLUMN_PREFIX,
-} from "../../tables/internal/sqs"
+import { mapToUserColumn, USER_COLUMN_PREFIX } from "../../tables/internal/sqs"
 import sdk from "../../../index"
 import {
   context,
@@ -44,7 +40,7 @@ import {
   getRelationshipColumns,
   getTableIDList,
 } from "./filters"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, helpers } from "@budibase/shared-core"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
@@ -164,7 +160,7 @@ function reverseUserColumnMapping(rows: Row[]) {
       if (index !== -1) {
         // cut out the prefix
         const newKey = key.slice(0, index) + key.slice(index + prefixLength)
-        const decoded = decodeNonAscii(newKey)
+        const decoded = helpers.schema.decodeNonAscii(newKey)
         finalRow[decoded] = row[key]
       } else {
         finalRow[key] = row[key]
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 3c14e2fc67..6199adcfba 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -16,6 +16,7 @@ import {
 } from "../../../../db/utils"
 import { isEqual } from "lodash"
 import { DEFAULT_TABLES } from "../../../../db/defaultData/datasource_bb_default"
+import { helpers } from "@budibase/shared-core"
 
 const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
@@ -65,29 +66,10 @@ function buildRelationshipDefinitions(
 
 export const USER_COLUMN_PREFIX = "data_"
 
-// SQS does not support non-ASCII characters in column names, so we need to
-// replace them with unicode escape sequences.
-function encodeNonAscii(str: string): string {
-  return str
-    .split("")
-    .map(char => {
-      return char.charCodeAt(0) > 127
-        ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
-        : char
-    })
-    .join("")
-}
-
-export function decodeNonAscii(str: string): string {
-  return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
-    String.fromCharCode(parseInt(p1, 16))
-  )
-}
-
 // utility function to denote that columns in SQLite are mapped to avoid overlap issues
 // the overlaps can occur due to case insensitivity and some of the columns which Budibase requires
 export function mapToUserColumn(key: string) {
-  return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}`
+  return `${USER_COLUMN_PREFIX}${helpers.schema.encodeNonAscii(key)}`
 }
 
 // this can generate relationship tables as part of the mapping
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 6db89dd2f3..1c45cb4338 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -22,6 +22,7 @@ import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
 import { deepGet, schema } from "./helpers"
 import { isPlainObject, isEmpty } from "lodash"
+import { decodeNonAscii } from "./helpers/schema"
 
 const HBS_REGEX = /{{([^{].*?)}}/g
 
@@ -181,8 +182,16 @@ export class ColumnSplitter {
   tableIds: string[]
   relationshipColumnNames: string[]
   relationships: string[]
+  aliases?: Record<string, string>
+  columnPrefix?: string
 
-  constructor(tables: Table[]) {
+  constructor(
+    tables: Table[],
+    opts?: {
+      aliases?: Record<string, string>
+      columnPrefix?: string
+    }
+  ) {
     this.tableNames = tables.map(table => table.name)
     this.tableIds = tables.map(table => table._id!)
     this.relationshipColumnNames = tables.flatMap(table =>
@@ -195,16 +204,38 @@ export class ColumnSplitter {
       .concat(this.relationshipColumnNames)
       // sort by length - makes sure there's no mis-matches due to similarities (sub column names)
       .sort((a, b) => b.length - a.length)
+
+    if (opts?.aliases) {
+      this.aliases = {}
+      for (const [key, value] of Object.entries(opts.aliases)) {
+        this.aliases[value] = key
+      }
+    }
+
+    this.columnPrefix = opts?.columnPrefix
   }
 
   run(key: string): {
     numberPrefix?: string
     relationshipPrefix?: string
+    tableName?: string
     column: string
   } {
     let { prefix, key: splitKey } = getKeyNumbering(key)
+
+    let tableName: string | undefined = undefined
+    if (this.aliases) {
+      for (const possibleAlias of Object.keys(this.aliases || {})) {
+        const withDot = `${possibleAlias}.`
+        if (splitKey.startsWith(withDot)) {
+          tableName = this.aliases[possibleAlias]!
+          splitKey = splitKey.slice(withDot.length)
+        }
+      }
+    }
+
     let relationship: string | undefined
-    for (let possibleRelationship of this.relationships) {
+    for (const possibleRelationship of this.relationships) {
       const withDot = `${possibleRelationship}.`
       if (splitKey.startsWith(withDot)) {
         const finalKeyParts = splitKey.split(withDot)
@@ -214,7 +245,15 @@ export class ColumnSplitter {
         break
       }
     }
+
+    if (this.columnPrefix) {
+      if (splitKey.startsWith(this.columnPrefix)) {
+        splitKey = decodeNonAscii(splitKey.slice(this.columnPrefix.length))
+      }
+    }
+
     return {
+      tableName,
       numberPrefix: prefix,
       relationshipPrefix: relationship,
       column: splitKey,
diff --git a/packages/shared-core/src/helpers/schema.ts b/packages/shared-core/src/helpers/schema.ts
index caf562a8cb..d0035cc305 100644
--- a/packages/shared-core/src/helpers/schema.ts
+++ b/packages/shared-core/src/helpers/schema.ts
@@ -26,3 +26,22 @@ export function isRequired(constraints: FieldConstraints | undefined) {
       constraints.presence === true)
   return isRequired
 }
+
+// SQS does not support non-ASCII characters in column names, so we need to
+// replace them with unicode escape sequences.
+export function encodeNonAscii(str: string): string {
+  return str
+    .split("")
+    .map(char => {
+      return char.charCodeAt(0) > 127
+        ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
+        : char
+    })
+    .join("")
+}
+
+export function decodeNonAscii(str: string): string {
+  return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
+    String.fromCharCode(parseInt(p1, 16))
+  )
+}

From 05992579352fa25b50617593ea041681b49d272c Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 30 Jul 2024 11:26:16 +0100
Subject: [PATCH 11/34] Fix time-only columns.

---
 packages/backend-core/src/sql/sql.ts          | 27 ++++++++++++++++---
 .../src/api/routes/tests/search.spec.ts       | 18 ++++++-------
 2 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 26d6545868..7a76e09d3f 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -219,6 +219,23 @@ class InternalBuilder {
     if (input == undefined) {
       return null
     }
+
+    if (
+      this.client === SqlClient.ORACLE &&
+      schema.type === FieldType.DATETIME &&
+      schema.timeOnly
+    ) {
+      if (input instanceof Date) {
+        const hours = input.getHours().toString().padStart(2, "0")
+        const minutes = input.getMinutes().toString().padStart(2, "0")
+        const seconds = input.getSeconds().toString().padStart(2, "0")
+        return `${hours}:${minutes}:${seconds}`
+      }
+      if (typeof input === "string") {
+        return new Date(`1970-01-01 ${input}`)
+      }
+    }
+
     if (typeof input === "string") {
       if (isInvalidISODateString(input)) {
         return null
@@ -531,7 +548,7 @@ class InternalBuilder {
         } else if (this.client === SqlClient.ORACLE) {
           const identifier = this.convertClobs(key)
           query = query[fnc](
-            `(${identifier} IS NOT NULL AND ${identifier} != ?)`,
+            `(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`,
             [value]
           )
         } else {
@@ -605,8 +622,12 @@ class InternalBuilder {
         const direction =
           value.direction === SortOrder.ASCENDING ? "asc" : "desc"
         let nulls
-        if (this.client === SqlClient.POSTGRES) {
-          // All other clients already sort this as expected by default, and adding this to the rest of the clients is causing issues
+        if (
+          this.client === SqlClient.POSTGRES ||
+          this.client === SqlClient.ORACLE
+        ) {
+          // All other clients already sort this as expected by default, and
+          // adding this to the rest of the clients is causing issues
           nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
         }
 
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 110a9ae699..d1fc361993 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -40,14 +40,14 @@ import { structures } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 
 describe.each([
-  ["in-memory", undefined],
-  ["lucene", undefined],
-  ["sqs", undefined],
-  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+  // ["in-memory", undefined],
+  // ["lucene", undefined],
+  // ["sqs", undefined],
+  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
   const isLucene = name === "lucene"
@@ -958,7 +958,7 @@ describe.each([
           }).toMatchExactly([{ name: "bar" }, { name: "foo" }])
         })
 
-        it("sorts descending", async () => {
+        it.only("sorts descending", async () => {
           await expectSearch({
             query: {},
             sort: "name",

From bc7501f72bca2ed8bc3057df4209fc96d793758e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 30 Jul 2024 11:54:46 +0100
Subject: [PATCH 12/34] Fix range queries.

---
 packages/backend-core/src/sql/sql.ts          | 150 +++++++++++-------
 .../backend-core/src/sql/sqlStatements.ts     |  87 ----------
 .../src/api/routes/tests/search.spec.ts       |   2 +-
 3 files changed, 96 insertions(+), 143 deletions(-)
 delete mode 100644 packages/backend-core/src/sql/sqlStatements.ts

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 7a76e09d3f..9803f8588b 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -8,7 +8,6 @@ import {
   sqlLog,
   isInvalidISODateString,
 } from "./utils"
-import { SqlStatements } from "./sqlStatements"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
   AnySearchFilter,
@@ -77,10 +76,12 @@ class InternalBuilder {
   private readonly client: SqlClient
   private readonly query: QueryJson
   private readonly splitter: dataFilters.ColumnSplitter
+  private readonly knex: Knex
 
-  constructor(client: SqlClient, query: QueryJson) {
+  constructor(client: SqlClient, knex: Knex, query: QueryJson) {
     this.client = client
     this.query = query
+    this.knex = knex
 
     this.splitter = new dataFilters.ColumnSplitter([this.table], {
       aliases: this.query.tableAliases,
@@ -92,6 +93,11 @@ class InternalBuilder {
     return this.query.meta.table
   }
 
+  getFieldSchema(key: string): FieldSchema | undefined {
+    const { column } = this.splitter.run(key)
+    return this.table.schema[column]
+  }
+
   // Takes a string like foo and returns a quoted string like [foo] for SQL Server
   // and "foo" for Postgres.
   private quote(str: string): string {
@@ -116,9 +122,8 @@ class InternalBuilder {
       .join(".")
   }
 
-  private generateSelectStatement(knex: Knex): (string | Knex.Raw)[] | "*" {
+  private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
     const { resource, meta } = this.query
-    const client = knex.client.config.client as SqlClient
 
     if (!resource || !resource.fields || resource.fields.length === 0) {
       return "*"
@@ -154,10 +159,10 @@ class InternalBuilder {
       const columnSchema = schema[column]
 
       if (
-        client === SqlClient.POSTGRES &&
+        this.client === SqlClient.POSTGRES &&
         columnSchema?.externalType?.includes("money")
       ) {
-        return knex.raw(
+        return this.knex.raw(
           `${this.quotedIdentifier(
             [table, column].join(".")
           )}::money::numeric as ${this.quote(field)}`
@@ -165,13 +170,13 @@ class InternalBuilder {
       }
 
       if (
-        client === SqlClient.MS_SQL &&
+        this.client === SqlClient.MS_SQL &&
         columnSchema?.type === FieldType.DATETIME &&
         columnSchema.timeOnly
       ) {
         // Time gets returned as timestamp from mssql, not matching the expected
         // HH:mm format
-        return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
+        return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
       }
 
       // There's at least two edge cases being handled in the expression below.
@@ -183,11 +188,11 @@ class InternalBuilder {
       //     aren't actually clear to me, but `table`.`doc1` breaks things with the
       //     sample data tests.
       if (table) {
-        return knex.raw(
+        return this.knex.raw(
           `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}`
         )
       } else {
-        return knex.raw(`${this.quote(field)} as ${this.quote(field)}`)
+        return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`)
       }
     })
   }
@@ -333,10 +338,6 @@ class InternalBuilder {
     const aliases = this.query.tableAliases
     // if all or specified in filters, then everything is an or
     const allOr = filters.allOr
-    const sqlStatements = new SqlStatements(this.client, this.table, {
-      allOr,
-      columnPrefix: this.query.meta.columnPrefix,
-    })
     const tableName =
       this.client === SqlClient.SQL_LITE ? this.table._id! : this.table.name
 
@@ -506,12 +507,53 @@ class InternalBuilder {
         }
         const lowValid = isValidFilter(value.low),
           highValid = isValidFilter(value.high)
+
+        const schema = this.getFieldSchema(key)
+
+        if (this.client === SqlClient.ORACLE) {
+          // @ts-ignore
+          key = this.knex.raw(this.convertClobs(key))
+        }
+
         if (lowValid && highValid) {
-          query = sqlStatements.between(query, key, value.low, value.high)
+          if (
+            schema?.type === FieldType.BIGINT &&
+            this.client === SqlClient.SQL_LITE
+          ) {
+            query = query.whereRaw(
+              `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
+              [value.low, value.high]
+            )
+          } else {
+            const fnc = allOr ? "orWhereBetween" : "whereBetween"
+            query = query[fnc](key, [value.low, value.high])
+          }
         } else if (lowValid) {
-          query = sqlStatements.lte(query, key, value.low)
+          if (
+            schema?.type === FieldType.BIGINT &&
+            this.client === SqlClient.SQL_LITE
+          ) {
+            query = query.whereRaw(
+              `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`,
+              [value.low]
+            )
+          } else {
+            const fnc = allOr ? "orWhere" : "where"
+            query = query[fnc](key, ">=", value.low)
+          }
         } else if (highValid) {
-          query = sqlStatements.gte(query, key, value.high)
+          if (
+            schema?.type === FieldType.BIGINT &&
+            this.client === SqlClient.SQL_LITE
+          ) {
+            query = query.whereRaw(
+              `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`,
+              [value.high]
+            )
+          } else {
+            const fnc = allOr ? "orWhere" : "where"
+            query = query[fnc](key, "<=", value.high)
+          }
         }
       })
     }
@@ -621,17 +663,19 @@ class InternalBuilder {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
           value.direction === SortOrder.ASCENDING ? "asc" : "desc"
-        let nulls
-        if (
-          this.client === SqlClient.POSTGRES ||
-          this.client === SqlClient.ORACLE
-        ) {
-          // All other clients already sort this as expected by default, and
-          // adding this to the rest of the clients is causing issues
-          nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
-        }
+        const nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
 
-        query = query.orderBy(`${aliased}.${key}`, direction, nulls)
+        let composite = `${aliased}.${key}`
+        if (this.client === SqlClient.ORACLE) {
+          query = query.orderBy(
+            // @ts-ignore
+            this.knex.raw(this.convertClobs(composite)),
+            direction,
+            nulls
+          )
+        } else {
+          query = query.orderBy(composite, direction, nulls)
+        }
       }
     }
 
@@ -732,17 +776,14 @@ class InternalBuilder {
     return query
   }
 
-  qualifiedKnex(
-    knex: Knex,
-    opts?: { alias?: string | boolean }
-  ): Knex.QueryBuilder {
+  qualifiedKnex(opts?: { alias?: string | boolean }): Knex.QueryBuilder {
     let alias = this.query.tableAliases?.[this.query.endpoint.entityId]
     if (opts?.alias === false) {
       alias = undefined
     } else if (typeof opts?.alias === "string") {
       alias = opts.alias
     }
-    return knex(
+    return this.knex(
       this.tableNameWithSchema(this.query.endpoint.entityId, {
         alias,
         schema: this.query.endpoint.schema,
@@ -750,9 +791,9 @@ class InternalBuilder {
     )
   }
 
-  create(knex: Knex, opts: QueryOptions): Knex.QueryBuilder {
+  create(opts: QueryOptions): Knex.QueryBuilder {
     const { body } = this.query
-    let query = this.qualifiedKnex(knex, { alias: false })
+    let query = this.qualifiedKnex({ alias: false })
     const parsedBody = this.parseBody(body)
     // make sure no null values in body for creation
     for (let [key, value] of Object.entries(parsedBody)) {
@@ -769,9 +810,9 @@ class InternalBuilder {
     }
   }
 
-  bulkCreate(knex: Knex): Knex.QueryBuilder {
+  bulkCreate(): Knex.QueryBuilder {
     const { body } = this.query
-    let query = this.qualifiedKnex(knex, { alias: false })
+    let query = this.qualifiedKnex({ alias: false })
     if (!Array.isArray(body)) {
       return query
     }
@@ -779,9 +820,9 @@ class InternalBuilder {
     return query.insert(parsedBody)
   }
 
-  bulkUpsert(knex: Knex): Knex.QueryBuilder {
+  bulkUpsert(): Knex.QueryBuilder {
     const { body } = this.query
-    let query = this.qualifiedKnex(knex, { alias: false })
+    let query = this.qualifiedKnex({ alias: false })
     if (!Array.isArray(body)) {
       return query
     }
@@ -809,7 +850,6 @@ class InternalBuilder {
   }
 
   read(
-    knex: Knex,
     opts: {
       limits?: { base: number; query: number }
     } = {}
@@ -821,7 +861,7 @@ class InternalBuilder {
 
     const tableName = endpoint.entityId
     // start building the query
-    let query = this.qualifiedKnex(knex)
+    let query = this.qualifiedKnex()
     // handle pagination
     let foundOffset: number | null = null
     let foundLimit = limits?.query || limits?.base
@@ -855,7 +895,7 @@ class InternalBuilder {
     query = this.addFilters(query, filters)
 
     const alias = tableAliases?.[tableName] || tableName
-    let preQuery: Knex.QueryBuilder = knex({
+    let preQuery: Knex.QueryBuilder = this.knex({
       // the typescript definition for the knex constructor doesn't support this
       // syntax, but it is the only way to alias a pre-query result as part of
       // a query - there is an alias dictionary type, but it assumes it can only
@@ -864,7 +904,7 @@ class InternalBuilder {
     })
     // if counting, use distinct count, else select
     preQuery = !counting
-      ? preQuery.select(this.generateSelectStatement(knex))
+      ? preQuery.select(this.generateSelectStatement())
       : this.addDistinctCount(preQuery)
     // have to add after as well (this breaks MS-SQL)
     if (this.client !== SqlClient.MS_SQL && !counting) {
@@ -888,9 +928,9 @@ class InternalBuilder {
     return this.addFilters(query, filters, { relationship: true })
   }
 
-  update(knex: Knex, opts: QueryOptions): Knex.QueryBuilder {
+  update(opts: QueryOptions): Knex.QueryBuilder {
     const { body, filters } = this.query
-    let query = this.qualifiedKnex(knex)
+    let query = this.qualifiedKnex()
     const parsedBody = this.parseBody(body)
     query = this.addFilters(query, filters)
     // mysql can't use returning
@@ -901,15 +941,15 @@ class InternalBuilder {
     }
   }
 
-  delete(knex: Knex, opts: QueryOptions): Knex.QueryBuilder {
+  delete(opts: QueryOptions): Knex.QueryBuilder {
     const { filters } = this.query
-    let query = this.qualifiedKnex(knex)
+    let query = this.qualifiedKnex()
     query = this.addFilters(query, filters)
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()
     } else {
-      return query.delete().returning(this.generateSelectStatement(knex))
+      return query.delete().returning(this.generateSelectStatement())
     }
   }
 }
@@ -953,13 +993,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
 
     const client = knex(config)
     let query: Knex.QueryBuilder
-    const builder = new InternalBuilder(sqlClient, json)
+    const builder = new InternalBuilder(sqlClient, client, json)
     switch (this._operation(json)) {
       case Operation.CREATE:
-        query = builder.create(client, opts)
+        query = builder.create(opts)
         break
       case Operation.READ:
-        query = builder.read(client, {
+        query = builder.read({
           limits: {
             query: this.limit,
             base: BASE_LIMIT,
@@ -968,19 +1008,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         break
       case Operation.COUNT:
         // read without any limits to count
-        query = builder.read(client)
+        query = builder.read()
         break
       case Operation.UPDATE:
-        query = builder.update(client, opts)
+        query = builder.update(opts)
         break
       case Operation.DELETE:
-        query = builder.delete(client, opts)
+        query = builder.delete(opts)
         break
       case Operation.BULK_CREATE:
-        query = builder.bulkCreate(client)
+        query = builder.bulkCreate()
         break
       case Operation.BULK_UPSERT:
-        query = builder.bulkUpsert(client)
+        query = builder.bulkUpsert()
         break
       case Operation.CREATE_TABLE:
       case Operation.UPDATE_TABLE:
diff --git a/packages/backend-core/src/sql/sqlStatements.ts b/packages/backend-core/src/sql/sqlStatements.ts
deleted file mode 100644
index 311f7c7d49..0000000000
--- a/packages/backend-core/src/sql/sqlStatements.ts
+++ /dev/null
@@ -1,87 +0,0 @@
-import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types"
-import { Knex } from "knex"
-
-export class SqlStatements {
-  client: string
-  table: Table
-  allOr: boolean | undefined
-  columnPrefix: string | undefined
-
-  constructor(
-    client: string,
-    table: Table,
-    { allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {}
-  ) {
-    this.client = client
-    this.table = table
-    this.allOr = allOr
-    this.columnPrefix = columnPrefix
-  }
-
-  getField(key: string): FieldSchema | undefined {
-    const fieldName = key.split(".")[1]
-    let found = this.table.schema[fieldName]
-    if (!found && this.columnPrefix) {
-      const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "")
-      found = this.table.schema[prefixRemovedFieldName]
-    }
-    return found
-  }
-
-  between(
-    query: Knex.QueryBuilder,
-    key: string,
-    low: number | string,
-    high: number | string
-  ) {
-    // Use a between operator if we have 2 valid range values
-    const field = this.getField(key)
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(
-        `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
-        [low, high]
-      )
-    } else {
-      const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
-      query = query[fnc](key, [low, high])
-    }
-    return query
-  }
-
-  lte(query: Knex.QueryBuilder, key: string, low: number | string) {
-    // Use just a single greater than operator if we only have a low
-    const field = this.getField(key)
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
-        low,
-      ])
-    } else {
-      const fnc = this.allOr ? "orWhere" : "where"
-      query = query[fnc](key, ">=", low)
-    }
-    return query
-  }
-
-  gte(query: Knex.QueryBuilder, key: string, high: number | string) {
-    const field = this.getField(key)
-    // Use just a single less than operator if we only have a high
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
-        high,
-      ])
-    } else {
-      const fnc = this.allOr ? "orWhere" : "where"
-      query = query[fnc](key, "<=", high)
-    }
-    return query
-  }
-}
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index d1fc361993..00badcbad5 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -958,7 +958,7 @@ describe.each([
           }).toMatchExactly([{ name: "bar" }, { name: "foo" }])
         })
 
-        it.only("sorts descending", async () => {
+        it("sorts descending", async () => {
           await expectSearch({
             query: {},
             sort: "name",

From b6560d1d7b4f58f8b644db2b655cf385a1d3b5fb Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 30 Jul 2024 11:58:28 +0100
Subject: [PATCH 13/34] Fix sorting.

---
 packages/backend-core/src/sql/sql.ts             |  9 ++++++++-
 .../server/src/api/routes/tests/search.spec.ts   | 16 ++++++++--------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 9803f8588b..df70ea6fea 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -663,7 +663,14 @@ class InternalBuilder {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
           value.direction === SortOrder.ASCENDING ? "asc" : "desc"
-        const nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
+
+        let nulls: "first" | "last" | undefined = undefined
+        if (
+          this.client === SqlClient.ORACLE ||
+          this.client === SqlClient.POSTGRES
+        ) {
+          nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
+        }
 
         let composite = `${aliased}.${key}`
         if (this.client === SqlClient.ORACLE) {
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 00badcbad5..110a9ae699 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -40,14 +40,14 @@ import { structures } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 
 describe.each([
-  // ["in-memory", undefined],
-  // ["lucene", undefined],
-  // ["sqs", undefined],
-  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+  ["in-memory", undefined],
+  ["lucene", undefined],
+  ["sqs", undefined],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
   const isLucene = name === "lucene"

From ff2802873ec24cd13b055d92bbfd93dae8f16aae Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 30 Jul 2024 12:29:16 +0100
Subject: [PATCH 14/34] Fixing an issue with to_char testing in sql.spec.ts.

---
 .../server/src/integrations/tests/sql.spec.ts | 39 ++++++++++++++-----
 1 file changed, 30 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index fe7ab761ca..47fc4228e9 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -1,12 +1,16 @@
 import {
   FieldType,
   Operation,
+  PaginationJson,
   QueryJson,
+  SearchFilters,
+  SortJson,
+  SqlClient,
   Table,
   TableSourceType,
-  SqlClient,
 } from "@budibase/types"
 import { sql } from "@budibase/backend-core"
+import { merge } from "lodash"
 
 const Sql = sql.Sql
 
@@ -25,7 +29,16 @@ const TABLE: Table = {
   primary: ["id"],
 }
 
-function endpoint(table: any, operation: any) {
+const ORACLE_TABLE: Partial<Table> = {
+  schema: {
+    name: {
+      name: "name",
+      type: FieldType.STRING,
+    },
+  },
+}
+
+function endpoint(table: string, operation: Operation) {
   return {
     datasourceId: "Postgres",
     operation: operation,
@@ -39,19 +52,25 @@ function generateReadJson({
   filters,
   sort,
   paginate,
-}: any = {}): QueryJson {
-  const tableObj = { ...TABLE }
+}: {
+  table?: Partial<Table>
+  fields?: string[]
+  filters?: SearchFilters
+  sort?: SortJson
+  paginate?: PaginationJson
+} = {}): QueryJson {
+  let tableObj: Table = { ...TABLE }
   if (table) {
-    tableObj.name = table
+    tableObj = merge(TABLE, table)
   }
   return {
-    endpoint: endpoint(table || TABLE_NAME, "READ"),
+    endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ),
     resource: {
       fields: fields || [],
     },
     filters: filters || {},
     sort: sort || {},
-    paginate: paginate || {},
+    paginate: paginate || undefined,
     meta: {
       table: tableObj,
     },
@@ -212,6 +231,7 @@ describe("SQL query builder", () => {
   it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => {
     let query = new Sql(SqlClient.ORACLE, limit)._query(
       generateReadJson({
+        table: ORACLE_TABLE,
         filters: {
           equal: {
             name: "John",
@@ -222,13 +242,14 @@ describe("SQL query builder", () => {
 
     expect(query).toEqual({
       bindings: ["John", limit, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) = :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
     })
   })
 
   it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => {
     let query = new Sql(SqlClient.ORACLE, limit)._query(
       generateReadJson({
+        table: ORACLE_TABLE,
         filters: {
           notEqual: {
             name: "John",
@@ -239,7 +260,7 @@ describe("SQL query builder", () => {
 
     expect(query).toEqual({
       bindings: ["John", limit, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) != :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
     })
   })
 })

From f4afa3270ebb06b5baa1ea7d2b7f0ccaebf08117 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 30 Jul 2024 12:44:31 +0100
Subject: [PATCH 15/34] Removing invalid test.

---
 .../src/integrations/tests/oracle.spec.ts     | 100 ------------------
 1 file changed, 100 deletions(-)
 delete mode 100644 packages/server/src/integrations/tests/oracle.spec.ts

diff --git a/packages/server/src/integrations/tests/oracle.spec.ts b/packages/server/src/integrations/tests/oracle.spec.ts
deleted file mode 100644
index 7b620d68ad..0000000000
--- a/packages/server/src/integrations/tests/oracle.spec.ts
+++ /dev/null
@@ -1,100 +0,0 @@
-const oracledb = require("oracledb")
-
-import { default as OracleIntegration } from "../oracle"
-
-jest.mock("oracledb")
-
-class TestConfiguration {
-  integration: any
-
-  constructor(config: any = {}) {
-    this.integration = new OracleIntegration.integration(config)
-  }
-}
-
-const options = { autoCommit: true }
-
-describe("Oracle Integration", () => {
-  let config: any
-
-  beforeEach(() => {
-    jest.clearAllMocks()
-    config = new TestConfiguration()
-  })
-
-  it("calls the create method with the correct params", async () => {
-    const sql = "insert into users (name, age) values ('Joe', 123);"
-    await config.integration.create({
-      sql,
-    })
-    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
-    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-  })
-
-  it("calls the read method with the correct params", async () => {
-    const sql = "select * from users;"
-    await config.integration.read({
-      sql,
-    })
-    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
-    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-  })
-
-  it("calls the update method with the correct params", async () => {
-    const sql = "update table users set name = 'test';"
-    await config.integration.update({
-      sql,
-    })
-    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
-    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-  })
-
-  it("calls the delete method with the correct params", async () => {
-    const sql = "delete from users where name = 'todelete';"
-    await config.integration.delete({
-      sql,
-    })
-    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
-    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-  })
-
-  describe("no rows returned", () => {
-    beforeEach(() => {
-      oracledb.executeMock.mockImplementation(() => ({ rows: [] }))
-    })
-
-    it("returns the correct response when the create response has no rows", async () => {
-      const sql = "insert into users (name, age) values ('Joe', 123);"
-      const response = await config.integration.create({
-        sql,
-      })
-      expect(response).toEqual([{ created: true }])
-      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-      expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-    })
-
-    it("returns the correct response when the update response has no rows", async () => {
-      const sql = "update table users set name = 'test';"
-      const response = await config.integration.update({
-        sql,
-      })
-      expect(response).toEqual([{ updated: true }])
-      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-      expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-    })
-
-    it("returns the correct response when the delete response has no rows", async () => {
-      const sql = "delete from users where name = 'todelete';"
-      const response = await config.integration.delete({
-        sql,
-      })
-      expect(response).toEqual([{ deleted: true }])
-      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-      expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-    })
-  })
-})

From c6ec710abebbb71f6489e76ab48ba337cf348259 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 30 Jul 2024 16:03:03 +0100
Subject: [PATCH 16/34] Solve _some_ date problems.

---
 packages/backend-core/src/sql/sql.ts          |  2 +-
 .../src/api/routes/tests/search.spec.ts       | 22 +++++++++----------
 packages/server/src/integrations/oracle.ts    |  4 +++-
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index df70ea6fea..a06af6e318 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -237,7 +237,7 @@ class InternalBuilder {
         return `${hours}:${minutes}:${seconds}`
       }
       if (typeof input === "string") {
-        return new Date(`1970-01-01 ${input}`)
+        return new Date(`1970-01-01T${input}Z`)
       }
     }
 
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 32e4735f3a..e2df279603 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -40,14 +40,14 @@ import { structures } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 
 describe.each([
-  ["in-memory", undefined],
-  ["lucene", undefined],
-  ["sqs", undefined],
-  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+  // ["in-memory", undefined],
+  // ["lucene", undefined],
+  // ["sqs", undefined],
+  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
   const isLucene = name === "lucene"
@@ -2389,9 +2389,9 @@ describe.each([
 
   describe.each([
     { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" },
-    // { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" },
-    // { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
-    // { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
+    { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" },
+    { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
+    { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
   ])("date special cases", ({ low, high }) => {
     const earlyDate = "2024-07-03T10:00:00.000Z",
       laterDate = "2024-07-03T11:00:00.000Z"
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index b8fcd63e7f..a9ce05302c 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -406,7 +406,9 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       password: this.config.password,
       connectString,
     }
-    return oracledb.getConnection(attributes)
+    const connection = await oracledb.getConnection(attributes)
+    await connection.execute(`ALTER SESSION SET TIME_ZONE='UTC'`)
+    return connection
   }
 
   async create(query: SqlQuery | string): Promise<any[]> {

From aa7894604ff9111c012308ccf61c9ef5f49ee77b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 30 Jul 2024 16:56:59 +0100
Subject: [PATCH 17/34] Fix dates and times for good? maybe?

---
 packages/backend-core/src/sql/sql.ts                | 11 ++++-------
 packages/server/src/integrations/oracle.ts          |  4 +---
 packages/server/src/utilities/rowProcessor/index.ts | 12 +++++++++---
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index a06af6e318..e41d744812 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -666,19 +666,16 @@ class InternalBuilder {
 
         let nulls: "first" | "last" | undefined = undefined
         if (
-          this.client === SqlClient.ORACLE ||
-          this.client === SqlClient.POSTGRES
+          this.client === SqlClient.POSTGRES ||
+          this.client === SqlClient.ORACLE
         ) {
           nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
         }
 
         let composite = `${aliased}.${key}`
         if (this.client === SqlClient.ORACLE) {
-          query = query.orderBy(
-            // @ts-ignore
-            this.knex.raw(this.convertClobs(composite)),
-            direction,
-            nulls
+          query = query.orderByRaw(
+            `${this.convertClobs(composite)} ${direction} nulls ${nulls}`
           )
         } else {
           query = query.orderBy(composite, direction, nulls)
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index a9ce05302c..41762576dd 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -406,9 +406,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       password: this.config.password,
       connectString,
     }
-    const connection = await oracledb.getConnection(attributes)
-    await connection.execute(`ALTER SESSION SET TIME_ZONE='UTC'`)
-    return connection
+    return await oracledb.getConnection(attributes)
   }
 
   async create(query: SqlQuery | string): Promise<any[]> {
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 139f3a5b8d..62a3b2dd74 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -318,9 +318,15 @@ export async function outputProcessing<T extends Row[] | Row>(
     } else if (column.type === FieldType.DATETIME && column.timeOnly) {
       for (let row of enriched) {
         if (row[property] instanceof Date) {
-          const hours = row[property].getHours().toString().padStart(2, "0")
-          const minutes = row[property].getMinutes().toString().padStart(2, "0")
-          const seconds = row[property].getSeconds().toString().padStart(2, "0")
+          const hours = row[property].getUTCHours().toString().padStart(2, "0")
+          const minutes = row[property]
+            .getUTCMinutes()
+            .toString()
+            .padStart(2, "0")
+          const seconds = row[property]
+            .getUTCSeconds()
+            .toString()
+            .padStart(2, "0")
           row[property] = `${hours}:${minutes}:${seconds}`
         }
       }

From 383132d06c5301596f287f93d49a52b269651de8 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 30 Jul 2024 17:26:00 +0100
Subject: [PATCH 18/34] Actually fix time zone problems this time.

---
 packages/server/src/integrations/oracle.ts          | 5 ++++-
 packages/server/src/utilities/rowProcessor/index.ts | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index 41762576dd..7895692076 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -406,7 +406,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       password: this.config.password,
       connectString,
     }
-    return await oracledb.getConnection(attributes)
+    const tz = Intl.DateTimeFormat().resolvedOptions().timeZone
+    const connection = await oracledb.getConnection(attributes)
+    await connection.execute(`ALTER SESSION SET TIME_ZONE = '${tz}'`)
+    return connection
   }
 
   async create(query: SqlQuery | string): Promise<any[]> {
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 62a3b2dd74..82676442dc 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -28,6 +28,7 @@ import {
 import { isExternalTableID } from "../../integrations/utils"
 import { helpers } from "@budibase/shared-core"
 import { processString } from "@budibase/string-templates"
+import { DateTime } from "mssql"
 
 export * from "./utils"
 export * from "./attachments"

From d7199c9def6099571d0cc542aeab07047800320d Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 30 Jul 2024 17:41:39 +0100
Subject: [PATCH 19/34] Fix a few more clob problems.

---
 packages/backend-core/src/sql/sql.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index e41d744812..fc712cb3c3 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -209,6 +209,7 @@ class InternalBuilder {
       schema.type === FieldType.STRING ||
       schema.type === FieldType.LONGFORM ||
       schema.type === FieldType.BB_REFERENCE_SINGLE ||
+      schema.type === FieldType.BB_REFERENCE ||
       schema.type === FieldType.OPTIONS ||
       schema.type === FieldType.BARCODEQR
     ) {

From f0bdbd5b4dbbe459f9b44a61dfe408e5ab540104 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 30 Jul 2024 17:53:53 +0100
Subject: [PATCH 20/34] Fixing not contains with oracle.

---
 packages/backend-core/src/sql/sql.ts |  12 +-
 yarn.lock                            | 192 ++++++++++++++++++++++++---
 2 files changed, 182 insertions(+), 22 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index e41d744812..d499cf765d 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -438,9 +438,13 @@ class InternalBuilder {
             } else {
               value[i] = `%${value[i]}%`
             }
-            statement +=
-              (statement ? andOr : "") +
-              `COALESCE(LOWER(${this.quotedIdentifier(key)}), '') LIKE ?`
+            const identifier = this.quotedIdentifier(key)
+            statement += statement ? andOr : ""
+            if (not) {
+              statement += `(NOT COALESCE(LOWER(${identifier}), '') LIKE ? OR ${identifier} IS NULL)`
+            } else {
+              statement += `COALESCE(LOWER(${identifier}), '') LIKE ?`
+            }
           }
 
           if (statement === "") {
@@ -448,7 +452,7 @@ class InternalBuilder {
           }
 
           // @ts-ignore
-          query = query[rawFnc](`${not}(${statement})`, value)
+          query = query[rawFnc](statement, value)
         })
       }
     }
diff --git a/yarn.lock b/yarn.lock
index 2d69b37cc6..607db0b7bb 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6707,22 +6707,39 @@ acorn-import-assertions@^1.9.0:
   resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz#507276249d684797c84e0734ef84860334cfb1ac"
   integrity sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==
 
-acorn-jsx@^5.3.2:
+acorn-jsx-walk@2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/acorn-jsx-walk/-/acorn-jsx-walk-2.0.0.tgz#a5ed648264e68282d7c2aead80216bfdf232573a"
+  integrity sha512-uuo6iJj4D4ygkdzd6jPtcxs8vZgDX9YFIkqczGImoypX2fQ4dVImmu3UzA4ynixCIMTrEOWW+95M2HuBaCEOVA==
+
+acorn-jsx@5.3.2, acorn-jsx@^5.3.2:
   version "5.3.2"
   resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937"
   integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==
 
+acorn-loose@8.4.0:
+  version "8.4.0"
+  resolved "https://registry.yarnpkg.com/acorn-loose/-/acorn-loose-8.4.0.tgz#26d3e219756d1e180d006f5bcc8d261a28530f55"
+  integrity sha512-M0EUka6rb+QC4l9Z3T0nJEzNOO7JcoJlYMrBlyBCiFSXRyxjLKayd4TbQs2FDRWQU1h9FR7QVNHt+PEaoNL5rQ==
+  dependencies:
+    acorn "^8.11.0"
+
+acorn-walk@8.3.3, acorn-walk@^8.0.2, acorn-walk@^8.1.1, acorn-walk@^8.2.0, acorn-walk@^8.3.2:
+  version "8.3.3"
+  resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.3.3.tgz#9caeac29eefaa0c41e3d4c65137de4d6f34df43e"
+  integrity sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==
+  dependencies:
+    acorn "^8.11.0"
+
 acorn-walk@^7.1.1:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
   integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==
 
-acorn-walk@^8.0.2, acorn-walk@^8.1.1, acorn-walk@^8.2.0, acorn-walk@^8.3.2:
-  version "8.3.3"
-  resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.3.3.tgz#9caeac29eefaa0c41e3d4c65137de4d6f34df43e"
-  integrity sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==
-  dependencies:
-    acorn "^8.11.0"
+acorn@8.12.1:
+  version "8.12.1"
+  resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.1.tgz#71616bdccbe25e27a54439e0046e89ca76df2248"
+  integrity sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==
 
 acorn@^5.2.1, acorn@^5.7.3:
   version "5.7.4"
@@ -6791,6 +6808,16 @@ ajv-formats@^2.0.2:
   dependencies:
     ajv "^8.0.0"
 
+ajv@8.17.1:
+  version "8.17.1"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.17.1.tgz#37d9a5c776af6bc92d7f4f9510eba4c0a60d11a6"
+  integrity sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==
+  dependencies:
+    fast-deep-equal "^3.1.3"
+    fast-uri "^3.0.1"
+    json-schema-traverse "^1.0.0"
+    require-from-string "^2.0.2"
+
 ajv@^6.12.3, ajv@^6.12.4:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
@@ -8484,6 +8511,11 @@ combos@^0.2.0:
   resolved "https://registry.yarnpkg.com/combos/-/combos-0.2.0.tgz#dc31c5a899b42293d55fe19c064d3e6e207ba4f7"
   integrity sha512-Z6YfvgiTCERWJTj3wQiXamFhssdvz1n4ok447rS330lw3uL72WAx8IvrLU7xiE71uyb5WF8JEP+BWB5KhOoGeg==
 
+commander@12.1.0:
+  version "12.1.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-12.1.0.tgz#01423b36f501259fdaac4d0e4d60c96c991585d3"
+  integrity sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==
+
 commander@6.2.0:
   version "6.2.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.0.tgz#b990bfb8ac030aedc6d11bc04d1488ffef56db75"
@@ -9551,6 +9583,34 @@ depd@^1.1.0, depd@~1.1.2:
   resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9"
   integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==
 
+dependency-cruiser@^16.3.7:
+  version "16.3.10"
+  resolved "https://registry.yarnpkg.com/dependency-cruiser/-/dependency-cruiser-16.3.10.tgz#fe26a50d5e10a4496bc2b70d027fca6ded48814f"
+  integrity sha512-WkCnibHBfvaiaQ+S46LZ6h4AR6oj42Vsf5/0Vgtrwdwn7ZekMJdZ/ALoTwNp/RaGlKW+MbV/fhSZOvmhAWVWzQ==
+  dependencies:
+    acorn "8.12.1"
+    acorn-jsx "5.3.2"
+    acorn-jsx-walk "2.0.0"
+    acorn-loose "8.4.0"
+    acorn-walk "8.3.3"
+    ajv "8.17.1"
+    commander "12.1.0"
+    enhanced-resolve "5.17.1"
+    ignore "5.3.1"
+    interpret "^3.1.1"
+    is-installed-globally "1.0.0"
+    json5 "2.2.3"
+    memoize "10.0.0"
+    picocolors "1.0.1"
+    picomatch "4.0.2"
+    prompts "2.4.2"
+    rechoir "^0.8.0"
+    safe-regex "2.1.1"
+    semver "^7.6.3"
+    teamcity-service-messages "0.1.14"
+    tsconfig-paths-webpack-plugin "4.1.0"
+    watskeburt "4.1.0"
+
 dependency-tree@^9.0.0:
   version "9.0.0"
   resolved "https://registry.yarnpkg.com/dependency-tree/-/dependency-tree-9.0.0.tgz#9288dd6daf35f6510c1ea30d9894b75369aa50a2"
@@ -10221,6 +10281,14 @@ engine.io@~6.5.2:
     engine.io-parser "~5.2.1"
     ws "~8.17.1"
 
+enhanced-resolve@5.17.1, enhanced-resolve@^5.7.0:
+  version "5.17.1"
+  resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz#67bfbbcc2f81d511be77d686a90267ef7f898a15"
+  integrity sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==
+  dependencies:
+    graceful-fs "^4.2.4"
+    tapable "^2.2.0"
+
 enhanced-resolve@^5.8.3:
   version "5.14.1"
   resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.14.1.tgz#de684b6803724477a4af5d74ccae5de52c25f6b3"
@@ -11016,6 +11084,11 @@ fast-text-encoding@^1.0.0:
   resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz#0aa25f7f638222e3396d72bf936afcf1d42d6867"
   integrity sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==
 
+fast-uri@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.0.1.tgz#cddd2eecfc83a71c1be2cc2ef2061331be8a7134"
+  integrity sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==
+
 fast-url-parser@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/fast-url-parser/-/fast-url-parser-1.1.3.tgz#f4af3ea9f34d8a271cf58ad2b3759f431f0b318d"
@@ -11877,6 +11950,13 @@ global-agent@3.0.0:
     semver "^7.3.2"
     serialize-error "^7.0.1"
 
+global-directory@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/global-directory/-/global-directory-4.0.1.tgz#4d7ac7cfd2cb73f304c53b8810891748df5e361e"
+  integrity sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==
+  dependencies:
+    ini "4.1.1"
+
 global-dirs@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.1.tgz#0c488971f066baceda21447aecb1a8b911d22485"
@@ -12541,6 +12621,11 @@ ignore-walk@^6.0.0:
   dependencies:
     minimatch "^7.4.2"
 
+ignore@5.3.1:
+  version "5.3.1"
+  resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef"
+  integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==
+
 ignore@^5.0.4, ignore@^5.2.0, ignore@^5.2.4:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.0.tgz#67418ae40d34d6999c95ff56016759c718c82f78"
@@ -12666,6 +12751,11 @@ ini@2.0.0:
   resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5"
   integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==
 
+ini@4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/ini/-/ini-4.1.1.tgz#d95b3d843b1e906e56d6747d5447904ff50ce7a1"
+  integrity sha512-QQnnxNyfvmHFIsj7gkPcYymR8Jdw/o7mp5ZFihxn6h8Ci6fh3Dx4E1gPjpQEpIuPo9XVNY/ZUwh4BPMjGyL01g==
+
 ini@^1.3.2, ini@^1.3.4, ini@^1.3.8, ini@~1.3.0:
   version "1.3.8"
   resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c"
@@ -12743,6 +12833,11 @@ interpret@^2.2.0:
   resolved "https://registry.yarnpkg.com/interpret/-/interpret-2.2.0.tgz#1a78a0b5965c40a5416d007ad6f50ad27c417df9"
   integrity sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==
 
+interpret@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/interpret/-/interpret-3.1.1.tgz#5be0ceed67ca79c6c4bc5cf0d7ee843dcea110c4"
+  integrity sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==
+
 into-stream@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/into-stream/-/into-stream-3.1.0.tgz#96fb0a936c12babd6ff1752a17d05616abd094c6"
@@ -12973,6 +13068,14 @@ is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1:
   dependencies:
     is-extglob "^2.1.1"
 
+is-installed-globally@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-1.0.0.tgz#08952c43758c33d815692392f7f8437b9e436d5a"
+  integrity sha512-K55T22lfpQ63N4KEN57jZUAaAYqYHEe8veb/TycJRk9DdSCLLcovXz/mL6mOnhQaZsQGwPhuFopdQIlqGSEjiQ==
+  dependencies:
+    global-directory "^4.0.1"
+    is-path-inside "^4.0.0"
+
 is-installed-globally@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520"
@@ -13060,6 +13163,11 @@ is-path-inside@^3.0.2, is-path-inside@^3.0.3:
   resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283"
   integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==
 
+is-path-inside@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-4.0.0.tgz#805aeb62c47c1b12fc3fd13bfb3ed1e7430071db"
+  integrity sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==
+
 is-plain-obj@^1.0.0, is-plain-obj@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e"
@@ -14084,6 +14192,11 @@ json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1:
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
   integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==
 
+json5@2.2.3, json5@^2.2.1, json5@^2.2.2, json5@^2.2.3:
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
+  integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
+
 json5@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593"
@@ -14091,11 +14204,6 @@ json5@^1.0.2:
   dependencies:
     minimist "^1.2.0"
 
-json5@^2.2.1, json5@^2.2.2, json5@^2.2.3:
-  version "2.2.3"
-  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
-  integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
-
 jsonc-parser@3.2.0, jsonc-parser@^3.2.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/jsonc-parser/-/jsonc-parser-3.2.0.tgz#31ff3f4c2b9793f89c67212627c51c6394f88e76"
@@ -15441,6 +15549,13 @@ memdown@^5.1.0:
     ltgt "~2.2.0"
     safe-buffer "~5.2.0"
 
+memoize@10.0.0:
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/memoize/-/memoize-10.0.0.tgz#43fa66b2022363c7c50cf5dfab732a808a3d7147"
+  integrity sha512-H6cBLgsi6vMWOcCpvVCdFFnl3kerEXbrYh9q+lY6VXvQSmM6CkmV08VOwT+WE2tzIEqRPFfAq3fm4v/UIW6mSA==
+  dependencies:
+    mimic-function "^5.0.0"
+
 memory-pager@^1.0.2:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/memory-pager/-/memory-pager-1.5.0.tgz#d8751655d22d384682741c972f2c3d6dfa3e66b5"
@@ -15549,6 +15664,11 @@ mimic-fn@^4.0.0:
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-4.0.0.tgz#60a90550d5cb0b239cca65d893b1a53b29871ecc"
   integrity sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==
 
+mimic-function@^5.0.0:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/mimic-function/-/mimic-function-5.0.1.tgz#acbe2b3349f99b9deaca7fb70e48b83e94e67076"
+  integrity sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==
+
 mimic-response@^1.0.0, mimic-response@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
@@ -17412,15 +17532,20 @@ phin@^2.9.1:
   resolved "https://registry.yarnpkg.com/phin/-/phin-2.9.3.tgz#f9b6ac10a035636fb65dfc576aaaa17b8743125c"
   integrity sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==
 
+picocolors@1.0.1, picocolors@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1"
+  integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==
+
 picocolors@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"
   integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==
 
-picocolors@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1"
-  integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==
+picomatch@4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.2.tgz#77c742931e8f3b8820946c76cd0c1f13730d1dab"
+  integrity sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==
 
 picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3, picomatch@^2.3.1:
   version "2.3.1"
@@ -18385,7 +18510,7 @@ promise.series@^0.2.0:
   resolved "https://registry.yarnpkg.com/promise.series/-/promise.series-0.2.0.tgz#2cc7ebe959fc3a6619c04ab4dbdc9e452d864bbd"
   integrity sha512-VWQJyU2bcDTgZw8kpfBpB/ejZASlCrzwz5f2hjb/zlujOEB4oeiAhHygAWq8ubsX2GVkD4kCU5V2dwOTaCY5EQ==
 
-prompts@^2.0.1:
+prompts@2.4.2, prompts@^2.0.1:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069"
   integrity sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==
@@ -18952,6 +19077,11 @@ regenerator-transform@^0.15.1:
   dependencies:
     "@babel/runtime" "^7.8.4"
 
+regexp-tree@~0.1.1:
+  version "0.1.27"
+  resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.27.tgz#2198f0ef54518ffa743fe74d983b56ffd631b6cd"
+  integrity sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==
+
 regexp.prototype.flags@^1.4.3, regexp.prototype.flags@^1.5.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz#138f644a3350f981a858c44f6bb1a61ff59be334"
@@ -19492,6 +19622,13 @@ safe-regex-test@^1.0.3:
     es-errors "^1.3.0"
     is-regex "^1.1.4"
 
+safe-regex@2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/safe-regex/-/safe-regex-2.1.1.tgz#f7128f00d056e2fe5c11e81a1324dd974aadced2"
+  integrity sha512-rx+x8AMzKb5Q5lQ95Zoi6ZbJqwCLkqi3XuJXp5P3rT8OEc6sZCJG5AE5dU3lsgRr/F4Bs31jSlVN+j5KrsGu9A==
+  dependencies:
+    regexp-tree "~0.1.1"
+
 safe-stable-stringify@^2.1.0, safe-stable-stringify@^2.3.1:
   version "2.4.3"
   resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886"
@@ -19603,7 +19740,7 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@~2.3.1:
+"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.3, semver@~2.3.1:
   version "7.5.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e"
   integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==
@@ -20867,6 +21004,11 @@ tarn@^3.0.1, tarn@^3.0.2:
   resolved "https://registry.yarnpkg.com/tarn/-/tarn-3.0.2.tgz#73b6140fbb881b71559c4f8bfde3d9a4b3d27693"
   integrity sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==
 
+teamcity-service-messages@0.1.14:
+  version "0.1.14"
+  resolved "https://registry.yarnpkg.com/teamcity-service-messages/-/teamcity-service-messages-0.1.14.tgz#193d420a5e4aef8e5e50b8c39e7865e08fbb5d8a"
+  integrity sha512-29aQwaHqm8RMX74u2o/h1KbMLP89FjNiMxD9wbF2BbWOnbM+q+d1sCEC+MqCc4QW3NJykn77OMpTFw/xTHIc0w==
+
 tedious@^16.4.0:
   version "16.7.1"
   resolved "https://registry.yarnpkg.com/tedious/-/tedious-16.7.1.tgz#1190f30fd99a413f1dc9250dee4835cf0788b650"
@@ -21258,6 +21400,15 @@ ts-node@10.8.1:
     v8-compile-cache-lib "^3.0.1"
     yn "3.1.1"
 
+tsconfig-paths-webpack-plugin@4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/tsconfig-paths-webpack-plugin/-/tsconfig-paths-webpack-plugin-4.1.0.tgz#3c6892c5e7319c146eee1e7302ed9e6f2be4f763"
+  integrity sha512-xWFISjviPydmtmgeUAuXp4N1fky+VCtfhOkDUFIv5ea7p4wuTomI4QTrXvFBX2S4jZsmyTSrStQl+E+4w+RzxA==
+  dependencies:
+    chalk "^4.1.0"
+    enhanced-resolve "^5.7.0"
+    tsconfig-paths "^4.1.2"
+
 tsconfig-paths@4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.0.0.tgz#1082f5d99fd127b72397eef4809e4dd06d229b64"
@@ -22037,6 +22188,11 @@ walker@^1.0.8:
   dependencies:
     makeerror "1.0.12"
 
+watskeburt@4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/watskeburt/-/watskeburt-4.1.0.tgz#3c0227669be646a97424b631164b1afe3d4d5344"
+  integrity sha512-KkY5H51ajqy9HYYI+u9SIURcWnqeVVhdH0I+ab6aXPGHfZYxgRCwnR6Lm3+TYB6jJVt5jFqw4GAKmwf1zHmGQw==
+
 wcwidth@^1.0.0, wcwidth@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/wcwidth/-/wcwidth-1.0.1.tgz#f0b0dcf915bc5ff1528afadb2c0e17b532da2fe8"

From cc1e466db91082d04f5d7e24b23b41876fb024aa Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 30 Jul 2024 17:57:21 +0100
Subject: [PATCH 21/34] Add Oracle to table.spec.ts

---
 .../server/src/api/routes/tests/table.spec.ts   | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index 20c83549d2..a8bf9447e8 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -9,6 +9,7 @@ import {
   RelationshipType,
   Row,
   SaveTableRequest,
+  SourceName,
   Table,
   TableSourceType,
   User,
@@ -33,7 +34,8 @@ describe.each([
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-])("/tables (%s)", (_, dsProvider) => {
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+])("/tables (%s)", (name, dsProvider) => {
   const isInternal: boolean = !dsProvider
   let datasource: Datasource | undefined
   let config = setup.getConfig()
@@ -52,15 +54,20 @@ describe.each([
       jest.clearAllMocks()
     })
 
-    it.each([
+    let names = [
       "alphanum",
       "with spaces",
       "with-dashes",
       "with_underscores",
-      'with "double quotes"',
-      "with 'single quotes'",
       "with `backticks`",
-    ])("creates a table with name: %s", async name => {
+    ]
+
+    if (name !== DatabaseName.ORACLE) {
+      names.push(`with "double quotes"`)
+      names.push(`with 'single quotes'`)
+    }
+
+    it.each(names)("creates a table with name: %s", async name => {
       const table = await config.api.table.save(
         tableForDatasource(datasource, { name })
       )

From 7cc000a838d92daa94a065806eecbf7420fe3367 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 30 Jul 2024 18:22:20 +0100
Subject: [PATCH 22/34] Fixes for not contains in oracle.

---
 packages/backend-core/src/sql/sql.ts          | 27 ++++++++++---------
 .../src/api/routes/tests/search.spec.ts       | 14 +++++-----
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 8900a979c7..69d32fc5b9 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -3,10 +3,10 @@ import * as dbCore from "../db"
 import {
   getNativeSql,
   isExternalTable,
-  isValidISODateString,
-  isValidFilter,
-  sqlLog,
   isInvalidISODateString,
+  isValidFilter,
+  isValidISODateString,
+  sqlLog,
 } from "./utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
@@ -433,27 +433,30 @@ class InternalBuilder {
         const andOr = mode === filters?.containsAny ? " OR " : " AND "
         iterate(mode, (key, value) => {
           let statement = ""
+          const identifier = this.quotedIdentifier(key)
           for (let i in value) {
             if (typeof value[i] === "string") {
               value[i] = `%"${value[i].toLowerCase()}"%`
             } else {
               value[i] = `%${value[i]}%`
             }
-            const identifier = this.quotedIdentifier(key)
-            statement += statement ? andOr : ""
-            if (not) {
-              statement += `(NOT COALESCE(LOWER(${identifier}), '') LIKE ? OR ${identifier} IS NULL)`
-            } else {
-              statement += `COALESCE(LOWER(${identifier}), '') LIKE ?`
-            }
+            statement += `${
+              statement ? andOr : ""
+            }COALESCE(LOWER(${identifier}), '') LIKE ?`
           }
 
           if (statement === "") {
             return
           }
 
-          // @ts-ignore
-          query = query[rawFnc](statement, value)
+          if (not) {
+            query = query[rawFnc](
+              `(NOT (${statement}) OR ${identifier} IS NULL)`,
+              value
+            )
+          } else {
+            query = query[rawFnc](statement, value)
+          }
         })
       }
     }
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index e2df279603..4125e44999 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -40,13 +40,13 @@ import { structures } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 
 describe.each([
-  // ["in-memory", undefined],
-  // ["lucene", undefined],
-  // ["sqs", undefined],
-  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  ["in-memory", undefined],
+  ["lucene", undefined],
+  ["sqs", undefined],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
   [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"

From 4b7042be9fba0e082c4646e780078253537997c1 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 31 Jul 2024 12:00:50 +0100
Subject: [PATCH 23/34] Ignore Oracle on bulk upsert tests.

---
 .../server/src/api/routes/tests/row.spec.ts   | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 9cc53168d0..eba50147f4 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -65,14 +65,16 @@ async function waitForEvent(
 }
 
 describe.each([
-  ["internal", undefined],
-  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  // ["internal", undefined],
+  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/rows (%s)", (providerType, dsProvider) => {
   const isInternal = dsProvider === undefined
   const isMSSQL = providerType === DatabaseName.SQL_SERVER
+  const isOracle = providerType === DatabaseName.ORACLE
   const config = setup.getConfig()
 
   let table: Table
@@ -127,7 +129,8 @@ describe.each([
       primary: ["id"],
       schema: defaultSchema,
     }
-    return merge(req, ...overrides)
+    const merged = merge(req, ...overrides)
+    return merged
   }
 
   function defaultTable(
@@ -1369,9 +1372,10 @@ describe.each([
         expect(rows[2].description).toEqual("Row 3 description")
       })
 
-    // Upserting isn't yet supported in MSSQL, see:
+    // Upserting isn't yet supported in MSSQL or Oracle, see:
     //   https://github.com/knex/knex/pull/6050
     !isMSSQL &&
+      !isOracle &&
       !isInternal &&
       it("should be able to update existing rows with composite primary keys with bulkImport", async () => {
         const tableName = uuid.v4()
@@ -1438,9 +1442,10 @@ describe.each([
         expect(rows[2].description).toEqual("Row 3 description")
       })
 
-    // Upserting isn't yet supported in MSSQL, see:
+    // Upserting isn't yet supported in MSSQL/Oracle, see:
     //   https://github.com/knex/knex/pull/6050
     !isMSSQL &&
+      !isOracle &&
       !isInternal &&
       it("should be able to update existing rows an autoID primary key", async () => {
         const tableName = uuid.v4()

From 6b78e599f0cfb9ea53d0fc2d64bc8c3ebddf3171 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 31 Jul 2024 17:37:16 +0100
Subject: [PATCH 24/34] Working on getting Oracle auto column imports working.

---
 packages/server/src/api/routes/tests/row.spec.ts | 2 +-
 packages/server/src/integrations/oracle.ts       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 85a9b40c7d..de7714a876 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -864,7 +864,7 @@ describe.each([
     })
 
     !isInternal &&
-      it("can update a row on an external table with a primary key", async () => {
+      it.only("can update a row on an external table with a primary key", async () => {
         const tableName = uuid.v4().substring(0, 10)
         await client!.schema.createTable(tableName, table => {
           table.increments("id").primary()
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index 7895692076..691c5167a7 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -98,7 +98,7 @@ const SCHEMA: Integration = {
   },
 }
 
-const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
+const UNSUPPORTED_TYPES = ["BLOB", "NCLOB"]
 
 const OracleContraintTypes = {
   PRIMARY: "P",

From 0bd18a2832c886e433f6cc7d1ce36b992c8b7fbe Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 1 Aug 2024 09:34:56 +0100
Subject: [PATCH 25/34] wip trigger support

---
 .../server/src/integrations/base/types.ts     |  8 +++++++
 packages/server/src/integrations/oracle.ts    | 21 +++++++++++++++----
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/integrations/base/types.ts b/packages/server/src/integrations/base/types.ts
index 7144d20206..463d73444b 100644
--- a/packages/server/src/integrations/base/types.ts
+++ b/packages/server/src/integrations/base/types.ts
@@ -104,6 +104,14 @@ export interface OracleColumnsResponse {
   SEARCH_CONDITION: null | string
 }
 
+export interface OracleTriggersResponse {
+  TABLE_NAME: string
+  TRIGGER_NAME: string
+  TRIGGER_TYPE: string
+  TRIGGERING_EVENT: string
+  TRIGGER_BODY: string
+}
+
 /**
  * An oracle constraint
  */
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index 691c5167a7..956526e8cf 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -31,7 +31,12 @@ import oracledb, {
   ExecuteOptions,
   Result,
 } from "oracledb"
-import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
+import {
+  OracleTable,
+  OracleColumn,
+  OracleColumnsResponse,
+  OracleTriggersResponse,
+} from "./base/types"
 import { sql } from "@budibase/backend-core"
 
 const Sql = sql.Sql
@@ -111,7 +116,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
   private readonly config: OracleConfig
   private index: number = 1
 
-  private readonly COLUMNS_SQL = `
+  private static readonly COLUMNS_SQL = `
     SELECT
       tabs.table_name,
       cols.column_name,
@@ -139,6 +144,11 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       (cons.status = 'ENABLED'
         OR cons.status IS NULL)
   `
+
+  private static readonly TRIGGERS_SQL = `
+    SELECT table_name, trigger_name, trigger_type, triggering_event, trigger_body FROM all_triggers WHERE status = 'ENABLED';
+  `
+
   constructor(config: OracleConfig) {
     super(SqlClient.ORACLE)
     this.config = config
@@ -255,7 +265,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     entities: Record<string, Table>
   ): Promise<Schema> {
     const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
-      sql: this.COLUMNS_SQL,
+      sql: OracleIntegration.COLUMNS_SQL,
+    })
+    const triggersResponse = await this.internalQuery<OracleTriggersResponse>({
+      sql: OracleIntegration.TRIGGERS_SQL,
     })
     const oracleTables = this.mapColumns(columnsResponse)
 
@@ -325,7 +338,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
 
   async getTableNames() {
     const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
-      sql: this.COLUMNS_SQL,
+      sql: OracleIntegration.COLUMNS_SQL,
     })
     return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
   }

From f23f479eb990f00b130c68a44e66d869f1efafbd Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 2 Aug 2024 11:17:38 +0100
Subject: [PATCH 26/34] Fix autocolumn detection on schema import.

---
 .../server/src/api/routes/tests/row.spec.ts   |  2 +-
 .../server/src/integrations/base/types.ts     | 27 +++++-
 packages/server/src/integrations/oracle.ts    | 83 ++++++++++++++++++-
 3 files changed, 108 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 0c6c54e52f..9df8ab2ee2 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -864,7 +864,7 @@ describe.each([
     })
 
     !isInternal &&
-      it.only("can update a row on an external table with a primary key", async () => {
+      it("can update a row on an external table with a primary key", async () => {
         const tableName = uuid.v4().substring(0, 10)
         await client!.schema.createTable(tableName, table => {
           table.increments("id").primary()
diff --git a/packages/server/src/integrations/base/types.ts b/packages/server/src/integrations/base/types.ts
index 463d73444b..1d0dee97fa 100644
--- a/packages/server/src/integrations/base/types.ts
+++ b/packages/server/src/integrations/base/types.ts
@@ -104,11 +104,34 @@ export interface OracleColumnsResponse {
   SEARCH_CONDITION: null | string
 }
 
+export enum TriggeringEvent {
+  INSERT = "INSERT",
+  DELETE = "DELETE",
+  UPDATE = "UPDATE",
+  LOGON = "LOGON",
+  LOGOFF = "LOGOFF",
+  STARTUP = "STARTUP",
+  SHUTDOWN = "SHUTDOWN",
+  SERVERERROR = "SERVERERROR",
+  SCHEMA = "SCHEMA",
+  ALTER = "ALTER",
+  DROP = "DROP",
+}
+
+export enum TriggerType {
+  BEFORE_EACH_ROW = "BEFORE EACH ROW",
+  AFTER_EACH_ROW = "AFTER EACH ROW",
+  BEFORE_STATEMENT = "BEFORE STATEMENT",
+  AFTER_STATEMENT = "AFTER STATEMENT",
+  INSTEAD_OF = "INSTEAD OF",
+  COMPOUND = "COMPOUND",
+}
+
 export interface OracleTriggersResponse {
   TABLE_NAME: string
   TRIGGER_NAME: string
-  TRIGGER_TYPE: string
-  TRIGGERING_EVENT: string
+  TRIGGER_TYPE: TriggerType
+  TRIGGERING_EVENT: TriggeringEvent
   TRIGGER_BODY: string
 }
 
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index 956526e8cf..d1c0978b89 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -36,6 +36,8 @@ import {
   OracleColumn,
   OracleColumnsResponse,
   OracleTriggersResponse,
+  TriggeringEvent,
+  TriggerType,
 } from "./base/types"
 import { sql } from "@budibase/backend-core"
 
@@ -146,7 +148,15 @@ class OracleIntegration extends Sql implements DatasourcePlus {
   `
 
   private static readonly TRIGGERS_SQL = `
-    SELECT table_name, trigger_name, trigger_type, triggering_event, trigger_body FROM all_triggers WHERE status = 'ENABLED';
+    SELECT 
+      table_name, 
+      trigger_name, 
+      trigger_type, 
+      triggering_event, 
+      trigger_body 
+    FROM 
+      all_triggers 
+    WHERE status = 'ENABLED'
   `
 
   constructor(config: OracleConfig) {
@@ -221,6 +231,75 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     return oracleTables
   }
 
+  private getTriggersFor(
+    tableName: string,
+    triggersResponse: Result<OracleTriggersResponse>,
+    opts?: { event?: TriggeringEvent; type?: TriggerType }
+  ): OracleTriggersResponse[] {
+    const triggers: OracleTriggersResponse[] = []
+    for (const trigger of triggersResponse.rows || []) {
+      if (trigger.TABLE_NAME !== tableName) {
+        continue
+      }
+      if (opts?.event && opts.event !== trigger.TRIGGERING_EVENT) {
+        continue
+      }
+      if (opts?.type && opts.type !== trigger.TRIGGER_TYPE) {
+        continue
+      }
+      triggers.push(trigger)
+    }
+    return triggers
+  }
+
+  private markAutoIncrementColumns(
+    triggersResponse: Result<OracleTriggersResponse>,
+    tables: Record<string, Table>
+  ) {
+    for (const table of Object.values(tables)) {
+      const triggers = this.getTriggersFor(table.name, triggersResponse, {
+        type: TriggerType.BEFORE_EACH_ROW,
+        event: TriggeringEvent.INSERT,
+      })
+
+      // This is the trigger body Knex generates for an auto increment column
+      // called "id" on a table called "foo":
+      //
+      //   declare checking number := 1;
+      //   begin if (:new. "id" is null) then while checking >= 1 loop
+      //   select
+      //       "foo_seq".nextval into :new. "id"
+      //   from
+      //       dual;
+      //   select
+      //       count("id") into checking
+      //   from
+      //       "foo"
+      //   where
+      //       "id" = :new. "id";
+      //   end loop;
+      //   end if;
+      //   end;
+      for (const [columnName, schema] of Object.entries(table.schema)) {
+        const autoIncrementTriggers = triggers.filter(
+          trigger =>
+            // This is a bit heuristic, but I think it's the best we can do with
+            // the information we have. We're looking for triggers that run
+            // before each row is inserted, and that have a body that contains a
+            // call to a function that generates a new value for the column.  We
+            // also check that the column name is in the trigger body, to make
+            // sure we're not picking up triggers that don't affect the column.
+            trigger.TRIGGER_BODY.includes(`"${columnName}"`) &&
+            trigger.TRIGGER_BODY.includes(`.nextval`)
+        )
+
+        if (autoIncrementTriggers.length > 0) {
+          schema.autocolumn = true
+        }
+      }
+    }
+  }
+
   private static isSupportedColumn(column: OracleColumn) {
     return !UNSUPPORTED_TYPES.includes(column.type)
   }
@@ -331,6 +410,8 @@ class OracleIntegration extends Sql implements DatasourcePlus {
         })
     })
 
+    this.markAutoIncrementColumns(triggersResponse, tables)
+
     let externalTables = finaliseExternalTables(tables, entities)
     let errors = checkExternalTables(externalTables)
     return { tables: externalTables, errors }

From 2671b9d5ef551f5614519dcdec1c6a3ab2c64567 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 2 Aug 2024 16:58:12 +0100
Subject: [PATCH 27/34] More row.spec.ts fixes.

---
 packages/backend-core/src/sql/sql.ts | 31 ++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 7fa406e87b..aa5d4b7796 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -809,10 +809,33 @@ class InternalBuilder {
     const { body } = this.query
     let query = this.qualifiedKnex({ alias: false })
     const parsedBody = this.parseBody(body)
-    // make sure no null values in body for creation
-    for (let [key, value] of Object.entries(parsedBody)) {
-      if (value == null) {
-        delete parsedBody[key]
+
+    if (this.client === SqlClient.ORACLE) {
+      // Oracle doesn't seem to automatically insert nulls
+      // if we don't specify them, so we need to do that here
+      for (const [column, schema] of Object.entries(
+        this.query.meta.table.schema
+      )) {
+        if (
+          schema.constraints?.presence === true ||
+          schema.type === FieldType.FORMULA ||
+          schema.type === FieldType.AUTO ||
+          schema.type === FieldType.LINK
+        ) {
+          continue
+        }
+
+        const value = parsedBody[column]
+        if (value == null) {
+          parsedBody[column] = null
+        }
+      }
+    } else {
+      // make sure no null values in body for creation
+      for (let [key, value] of Object.entries(parsedBody)) {
+        if (value == null) {
+          delete parsedBody[key]
+        }
       }
     }
 

From bc797238aadfc5ea40d25763691362b4cef7f741 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 2 Aug 2024 17:17:33 +0100
Subject: [PATCH 28/34] row.spec.ts passing in full

---
 packages/backend-core/src/sql/sql.ts             | 10 +++++++++-
 packages/server/src/api/routes/tests/row.spec.ts | 13 +++++++------
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index aa5d4b7796..bafaef40e4 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -478,7 +478,15 @@ class InternalBuilder {
           }
         },
         (key: string[], array) => {
-          query = query[fnc](key, Array.isArray(array) ? array : [array])
+          if (this.client === SqlClient.ORACLE) {
+            const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})`
+            const binding = `(${array
+              .map((a: any) => `(${new Array(a.length).fill("?").join(",")})`)
+              .join(",")})`
+            query = query.whereRaw(`${keyStr} IN ${binding}`, array.flat())
+          } else {
+            query = query[fnc](key, Array.isArray(array) ? array : [array])
+          }
         }
       )
     }
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 770a60822d..cf94eb9f13 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -67,11 +67,11 @@ async function waitForEvent(
 }
 
 describe.each([
-  // ["internal", undefined],
-  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  ["internal", undefined],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
   [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/rows (%s)", (providerType, dsProvider) => {
   const isInternal = dsProvider === undefined
@@ -1409,9 +1409,10 @@ describe.each([
         await assertRowUsage(rowUsage + 3)
       })
 
-    // Upserting isn't yet supported in MSSQL, see:
+    // Upserting isn't yet supported in MSSQL / Oracle, see:
     //   https://github.com/knex/knex/pull/6050
     !isMSSQL &&
+      !isOracle &&
       it("should be able to update existing rows with bulkImport", async () => {
         const table = await config.api.table.save(
           saveTableRequest({

From 1ce5b534090259d02ee23727a99bf44065d72432 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 2 Aug 2024 17:20:52 +0100
Subject: [PATCH 29/34] Add Oracle to executeQuery.spec.ts

---
 .../automations/tests/executeQuery.spec.ts    | 28 ++++++-------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/packages/server/src/automations/tests/executeQuery.spec.ts b/packages/server/src/automations/tests/executeQuery.spec.ts
index e0bb7f7baa..20f906e695 100644
--- a/packages/server/src/automations/tests/executeQuery.spec.ts
+++ b/packages/server/src/automations/tests/executeQuery.spec.ts
@@ -1,27 +1,20 @@
-import { Datasource, Query, SourceName } from "@budibase/types"
+import { Datasource, Query } from "@budibase/types"
 import * as setup from "./utilities"
-import { DatabaseName, getDatasource } from "../../integrations/tests/utils"
-import knex, { Knex } from "knex"
+import {
+  DatabaseName,
+  getDatasource,
+  knexClient,
+} from "../../integrations/tests/utils"
+import { Knex } from "knex"
 import { generator } from "@budibase/backend-core/tests"
 
-function getKnexClientName(source: SourceName) {
-  switch (source) {
-    case SourceName.MYSQL:
-      return "mysql2"
-    case SourceName.SQL_SERVER:
-      return "mssql"
-    case SourceName.POSTGRES:
-      return "pg"
-  }
-  throw new Error(`Unsupported source: ${source}`)
-}
-
 describe.each(
   [
     DatabaseName.POSTGRES,
     DatabaseName.MYSQL,
     DatabaseName.SQL_SERVER,
     DatabaseName.MARIADB,
+    DatabaseName.ORACLE,
   ].map(name => [name, getDatasource(name)])
 )("execute query action (%s)", (_, dsProvider) => {
   let tableName: string
@@ -35,10 +28,7 @@ describe.each(
 
     const ds = await dsProvider
     datasource = await config.api.datasource.create(ds)
-    client = knex({
-      client: getKnexClientName(ds.source),
-      connection: ds.config,
-    })
+    client = await knexClient(ds)
   })
 
   beforeEach(async () => {

From 165e368a243e06fea7c888f95ae47371fb5a8f83 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 2 Aug 2024 17:22:18 +0100
Subject: [PATCH 30/34] Add Oracle to viewV2.spec.ts

---
 packages/server/src/api/routes/tests/viewV2.spec.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 2ff5c67852..8c0bc39234 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -33,6 +33,7 @@ describe.each([
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/v2/views (%s)", (name, dsProvider) => {
   const config = setup.getConfig()
   const isSqs = name === "sqs"

From eecd521a8ae7431f70bdcef1a7e0872ee8b6db40 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 5 Aug 2024 12:05:42 +0100
Subject: [PATCH 31/34] Fix generic-sql.spec.ts

---
 .../routes/tests/queries/generic-sql.spec.ts  | 300 +++++++++---------
 1 file changed, 153 insertions(+), 147 deletions(-)

diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
index 3ed19f5eee..a84b243e2d 100644
--- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
+++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
@@ -22,9 +22,13 @@ describe.each(
     DatabaseName.MYSQL,
     DatabaseName.SQL_SERVER,
     DatabaseName.MARIADB,
+    DatabaseName.ORACLE,
   ].map(name => [name, getDatasource(name)])
 )("queries (%s)", (dbName, dsProvider) => {
   const config = setup.getConfig()
+  const isOracle = dbName === DatabaseName.ORACLE
+  const isMsSQL = dbName === DatabaseName.SQL_SERVER
+
   let rawDatasource: Datasource
   let datasource: Datasource
   let client: Knex
@@ -97,7 +101,7 @@ describe.each(
         const query = await createQuery({
           name: "New Query",
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -106,7 +110,7 @@ describe.each(
           name: "New Query",
           parameters: [],
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
           schema: {},
           queryVerb: "read",
@@ -125,7 +129,7 @@ describe.each(
       it("should be able to update a query", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -135,7 +139,7 @@ describe.each(
           ...query,
           name: "Updated Query",
           fields: {
-            sql: "SELECT * FROM test_table WHERE id = 1",
+            sql: client("test_table").where({ id: 1 }).toString(),
           },
         })
 
@@ -144,7 +148,7 @@ describe.each(
           name: "Updated Query",
           parameters: [],
           fields: {
-            sql: "SELECT * FROM test_table WHERE id = 1",
+            sql: client("test_table").where({ id: 1 }).toString(),
           },
           schema: {},
           queryVerb: "read",
@@ -161,7 +165,7 @@ describe.each(
       it("should be able to delete a query", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -180,7 +184,7 @@ describe.each(
       it("should be able to list queries", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -191,7 +195,7 @@ describe.each(
       it("should strip sensitive fields for prod apps", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -212,7 +216,7 @@ describe.each(
         datasourceId: datasource._id!,
         queryVerb: "read",
         fields: {
-          sql: `SELECT * FROM test_table WHERE id = 1`,
+          sql: client("test_table").where({ id: 1 }).toString(),
         },
         parameters: [],
         transformer: "return data",
@@ -270,7 +274,7 @@ describe.each(
         name: "Test Query",
         queryVerb: "read",
         fields: {
-          sql: `SELECT * FROM ${tableName}`,
+          sql: client(tableName).select("*").toString(),
         },
         parameters: [],
         transformer: "return data",
@@ -284,11 +288,13 @@ describe.each(
         })
       )
 
+      await client(tableName).delete()
       await client.schema.alterTable(tableName, table => {
         table.string("data").alter()
       })
 
-      await client(tableName).update({
+      await client(tableName).insert({
+        name: "test",
         data: "string value",
       })
 
@@ -297,7 +303,7 @@ describe.each(
         name: "Test Query",
         queryVerb: "read",
         fields: {
-          sql: `SELECT * FROM ${tableName}`,
+          sql: client(tableName).select("*").toString(),
         },
         parameters: [],
         transformer: "return data",
@@ -311,6 +317,7 @@ describe.each(
         })
       )
     })
+
     it("should work with static variables", async () => {
       await config.api.datasource.update({
         ...datasource,
@@ -326,7 +333,7 @@ describe.each(
         datasourceId: datasource._id!,
         queryVerb: "read",
         fields: {
-          sql: `SELECT '{{ foo }}' as foo`,
+          sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
         },
         parameters: [],
         transformer: "return data",
@@ -337,16 +344,17 @@ describe.each(
 
       const response = await config.api.query.preview(request)
 
+      let key = isOracle ? "FOO" : "foo"
       expect(response.schema).toEqual({
-        foo: {
-          name: "foo",
+        [key]: {
+          name: key,
           type: "string",
         },
       })
 
       expect(response.rows).toEqual([
         {
-          foo: "bar",
+          [key]: "bar",
         },
       ])
     })
@@ -354,7 +362,7 @@ describe.each(
     it("should work with dynamic variables", async () => {
       const basedOnQuery = await createQuery({
         fields: {
-          sql: "SELECT name FROM test_table WHERE id = 1",
+          sql: client("test_table").select("name").where({ id: 1 }).toString(),
         },
       })
 
@@ -376,7 +384,7 @@ describe.each(
         datasourceId: datasource._id!,
         queryVerb: "read",
         fields: {
-          sql: `SELECT '{{ foo }}' as foo`,
+          sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
         },
         parameters: [],
         transformer: "return data",
@@ -385,16 +393,17 @@ describe.each(
         readable: true,
       })
 
+      let key = isOracle ? "FOO" : "foo"
       expect(preview.schema).toEqual({
-        foo: {
-          name: "foo",
+        [key]: {
+          name: key,
           type: "string",
         },
       })
 
       expect(preview.rows).toEqual([
         {
-          foo: "one",
+          [key]: "one",
         },
       ])
     })
@@ -402,7 +411,7 @@ describe.each(
     it("should handle the dynamic base query being deleted", async () => {
       const basedOnQuery = await createQuery({
         fields: {
-          sql: "SELECT name FROM test_table WHERE id = 1",
+          sql: client("test_table").select("name").where({ id: 1 }).toString(),
         },
       })
 
@@ -426,7 +435,7 @@ describe.each(
         datasourceId: datasource._id!,
         queryVerb: "read",
         fields: {
-          sql: `SELECT '{{ foo }}' as foo`,
+          sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
         },
         parameters: [],
         transformer: "return data",
@@ -435,16 +444,17 @@ describe.each(
         readable: true,
       })
 
+      let key = isOracle ? "FOO" : "foo"
       expect(preview.schema).toEqual({
-        foo: {
-          name: "foo",
+        [key]: {
+          name: key,
           type: "string",
         },
       })
 
       expect(preview.rows).toEqual([
         {
-          foo: datasource.source === SourceName.SQL_SERVER ? "" : null,
+          [key]: datasource.source === SourceName.SQL_SERVER ? "" : null,
         },
       ])
     })
@@ -455,7 +465,7 @@ describe.each(
       it("should be able to insert with bindings", async () => {
         const query = await createQuery({
           fields: {
-            sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+            sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
           },
           parameters: [
             {
@@ -488,7 +498,7 @@ describe.each(
       it("should not allow handlebars as parameters", async () => {
         const query = await createQuery({
           fields: {
-            sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+            sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
           },
           parameters: [
             {
@@ -516,46 +526,55 @@ describe.each(
         )
       })
 
-      it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
-        "should coerce %s into a date",
-        async datetimeStr => {
-          const date = new Date(datetimeStr)
-          const query = await createQuery({
-            fields: {
-              sql: `INSERT INTO test_table (name, birthday) VALUES ('foo', {{ birthday }})`,
-            },
-            parameters: [
-              {
-                name: "birthday",
-                default: "",
+      // Oracle doesn't automatically coerce strings into dates.
+      !isOracle &&
+        it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
+          "should coerce %s into a date",
+          async datetimeStr => {
+            const date = new Date(datetimeStr)
+            const query = await createQuery({
+              fields: {
+                sql: client("test_table")
+                  .insert({
+                    name: "foo",
+                    birthday: client.raw("{{ birthday }}"),
+                  })
+                  .toString(),
               },
-            ],
-            queryVerb: "create",
-          })
+              parameters: [
+                {
+                  name: "birthday",
+                  default: "",
+                },
+              ],
+              queryVerb: "create",
+            })
 
-          const result = await config.api.query.execute(query._id!, {
-            parameters: { birthday: datetimeStr },
-          })
+            const result = await config.api.query.execute(query._id!, {
+              parameters: { birthday: datetimeStr },
+            })
 
-          expect(result.data).toEqual([{ created: true }])
+            expect(result.data).toEqual([{ created: true }])
 
-          const rows = await client("test_table")
-            .where({ birthday: datetimeStr })
-            .select()
-          expect(rows).toHaveLength(1)
+            const rows = await client("test_table")
+              .where({ birthday: datetimeStr })
+              .select()
+            expect(rows).toHaveLength(1)
 
-          for (const row of rows) {
-            expect(new Date(row.birthday)).toEqual(date)
+            for (const row of rows) {
+              expect(new Date(row.birthday)).toEqual(date)
+            }
           }
-        }
-      )
+        )
 
       it.each(["2021,02,05", "202205-1500"])(
         "should not coerce %s as a date",
         async notDateStr => {
           const query = await createQuery({
             fields: {
-              sql: "INSERT INTO test_table (name) VALUES ({{ name }})",
+              sql: client("test_table")
+                .insert({ name: client.raw("{{ name }}") })
+                .toString(),
             },
             parameters: [
               {
@@ -586,7 +605,7 @@ describe.each(
       it("should execute a query", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table ORDER BY id",
+            sql: client("test_table").select("*").orderBy("id").toString(),
           },
         })
 
@@ -629,7 +648,7 @@ describe.each(
       it("should be able to transform a query", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table WHERE id = 1",
+            sql: client("test_table").where({ id: 1 }).select("*").toString(),
           },
           transformer: `
             data[0].id = data[0].id + 1;
@@ -652,7 +671,10 @@ describe.each(
       it("should coerce numeric bindings", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table WHERE id = {{ id }}",
+            sql: client("test_table")
+              .where({ id: client.raw("{{ id }}") })
+              .select("*")
+              .toString(),
           },
           parameters: [
             {
@@ -683,7 +705,10 @@ describe.each(
       it("should be able to update rows", async () => {
         const query = await createQuery({
           fields: {
-            sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
+            sql: client("test_table")
+              .update({ name: client.raw("{{ name }}") })
+              .where({ id: client.raw("{{ id }}") })
+              .toString(),
           },
           parameters: [
             {
@@ -698,19 +723,13 @@ describe.each(
           queryVerb: "update",
         })
 
-        const result = await config.api.query.execute(query._id!, {
+        await config.api.query.execute(query._id!, {
           parameters: {
             id: "1",
             name: "foo",
           },
         })
 
-        expect(result.data).toEqual([
-          {
-            updated: true,
-          },
-        ])
-
         const rows = await client("test_table").where({ id: 1 }).select()
         expect(rows).toEqual([
           { id: 1, name: "foo", birthday: null, number: null },
@@ -720,35 +739,34 @@ describe.each(
       it("should be able to execute an update that updates no rows", async () => {
         const query = await createQuery({
           fields: {
-            sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
+            sql: client("test_table")
+              .update({ name: "updated" })
+              .where({ id: 100 })
+              .toString(),
           },
           queryVerb: "update",
         })
 
-        const result = await config.api.query.execute(query._id!)
+        await config.api.query.execute(query._id!)
 
-        expect(result.data).toEqual([
-          {
-            updated: true,
-          },
-        ])
+        const rows = await client("test_table").select()
+        for (const row of rows) {
+          expect(row.name).not.toEqual("updated")
+        }
       })
 
       it("should be able to execute a delete that deletes no rows", async () => {
         const query = await createQuery({
           fields: {
-            sql: "DELETE FROM test_table WHERE id = 100",
+            sql: client("test_table").where({ id: 100 }).delete().toString(),
           },
           queryVerb: "delete",
         })
 
-        const result = await config.api.query.execute(query._id!)
+        await config.api.query.execute(query._id!)
 
-        expect(result.data).toEqual([
-          {
-            deleted: true,
-          },
-        ])
+        const rows = await client("test_table").select()
+        expect(rows).toHaveLength(5)
       })
     })
 
@@ -756,7 +774,10 @@ describe.each(
       it("should be able to delete rows", async () => {
         const query = await createQuery({
           fields: {
-            sql: "DELETE FROM test_table WHERE id = {{ id }}",
+            sql: client("test_table")
+              .where({ id: client.raw("{{ id }}") })
+              .delete()
+              .toString(),
           },
           parameters: [
             {
@@ -767,18 +788,12 @@ describe.each(
           queryVerb: "delete",
         })
 
-        const result = await config.api.query.execute(query._id!, {
+        await config.api.query.execute(query._id!, {
           parameters: {
             id: "1",
           },
         })
 
-        expect(result.data).toEqual([
-          {
-            deleted: true,
-          },
-        ])
-
         const rows = await client("test_table").where({ id: 1 }).select()
         expect(rows).toHaveLength(0)
       })
@@ -823,72 +838,63 @@ describe.each(
       })
     })
 
-    it("should be able to execute an update that updates no rows", async () => {
-      const query = await createQuery({
-        fields: {
-          sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
-        },
-        queryVerb: "update",
+    // this parameter really only impacts SQL queries
+    describe("confirm nullDefaultSupport", () => {
+      let queryParams: Partial<Query>
+      beforeAll(async () => {
+        queryParams = {
+          fields: {
+            sql: client("test_table")
+              .insert({
+                name: client.raw("{{ bindingName }}"),
+                number: client.raw("{{ bindingNumber }}"),
+              })
+              .toString(),
+          },
+          parameters: [
+            {
+              name: "bindingName",
+              default: "",
+            },
+            {
+              name: "bindingNumber",
+              default: "",
+            },
+          ],
+          queryVerb: "create",
+        }
       })
 
-      const result = await config.api.query.execute(query._id!, {})
+      it("should error for old queries", async () => {
+        const query = await createQuery(queryParams)
+        await config.api.query.save({ ...query, nullDefaultSupport: false })
+        let error: string | undefined
+        try {
+          await config.api.query.execute(query._id!, {
+            parameters: {
+              bindingName: "testing",
+            },
+          })
+        } catch (err: any) {
+          error = err.message
+        }
+        if (isMsSQL || isOracle) {
+          expect(error).toBeUndefined()
+        } else {
+          expect(error).toBeDefined()
+          expect(error).toContain("integer")
+        }
+      })
 
-      expect(result.data).toEqual([
-        {
-          updated: true,
-        },
-      ])
-    })
-  })
-
-  // this parameter really only impacts SQL queries
-  describe("confirm nullDefaultSupport", () => {
-    const queryParams = {
-      fields: {
-        sql: "INSERT INTO test_table (name, number) VALUES ({{ bindingName }}, {{ bindingNumber }})",
-      },
-      parameters: [
-        {
-          name: "bindingName",
-          default: "",
-        },
-        {
-          name: "bindingNumber",
-          default: "",
-        },
-      ],
-      queryVerb: "create",
-    }
-
-    it("should error for old queries", async () => {
-      const query = await createQuery(queryParams)
-      await config.api.query.save({ ...query, nullDefaultSupport: false })
-      let error: string | undefined
-      try {
-        await config.api.query.execute(query._id!, {
+      it("should not error for new queries", async () => {
+        const query = await createQuery(queryParams)
+        const results = await config.api.query.execute(query._id!, {
           parameters: {
             bindingName: "testing",
           },
         })
-      } catch (err: any) {
-        error = err.message
-      }
-      if (dbName === "mssql") {
-        expect(error).toBeUndefined()
-      } else {
-        expect(error).toBeDefined()
-        expect(error).toContain("integer")
-      }
-    })
-
-    it("should not error for new queries", async () => {
-      const query = await createQuery(queryParams)
-      const results = await config.api.query.execute(query._id!, {
-        parameters: {
-          bindingName: "testing",
-        },
+        expect(results).toEqual({ data: [{ created: true }] })
       })
-      expect(results).toEqual({ data: [{ created: true }] })
     })
   })
 })

From 2efa8dfca2e7835a566c0101995e0df16caff960 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 5 Aug 2024 12:10:03 +0100
Subject: [PATCH 32/34] Fix lint.

---
 packages/server/src/api/routes/tests/table.spec.ts  | 1 -
 packages/server/src/utilities/rowProcessor/index.ts | 1 -
 2 files changed, 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index 7db1eb0711..52d6c3d379 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -13,7 +13,6 @@ import {
   RelationshipType,
   Row,
   SaveTableRequest,
-  SourceName,
   Table,
   TableSchema,
   TableSourceType,
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 82676442dc..62a3b2dd74 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -28,7 +28,6 @@ import {
 import { isExternalTableID } from "../../integrations/utils"
 import { helpers } from "@budibase/shared-core"
 import { processString } from "@budibase/string-templates"
-import { DateTime } from "mssql"
 
 export * from "./utils"
 export * from "./attachments"

From d2f4817472fcf6a7638f3f78b36a6b21d7959d85 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 5 Aug 2024 12:12:24 +0100
Subject: [PATCH 33/34] Add Oracle to docker pulls in budibase_ci.yml.

---
 .github/workflows/budibase_ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 2e22e76ac6..a17ca352cc 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -175,6 +175,7 @@ jobs:
           docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} &
           docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} &
           docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} &
+          docker pull budibase/oracle-database:23.2-slim-faststart &
           docker pull minio/minio &
           docker pull redis &
           docker pull testcontainers/ryuk:0.5.1 &

From 7fe0e3188398a73d5c87a26e03cfc48287af75ce Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 5 Aug 2024 14:08:07 +0100
Subject: [PATCH 34/34] Fix sql.spec.ts

---
 packages/server/src/integrations/tests/sql.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index 47fc4228e9..c4b2a69f7d 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -210,7 +210,7 @@ describe("SQL query builder", () => {
     )
     expect(query).toEqual({
       bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4) order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
+      sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
     })
 
     query = new Sql(SqlClient.ORACLE, limit)._query(