wip

packages/backend-core/src/sql/sql.ts

@@ -109,6 +109,26 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters {
   return filters
 }
 
+// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
+// so when we use them we need to wrap them in to_char(). This function
+// converts a field name to the appropriate identifier.
+function convertClobs(client: SqlClient, table: Table, field: string): string {
+  const parts = field.split(".")
+  const col = parts.pop()!
+  const schema = table.schema[col]
+  let identifier = quotedIdentifier(client, field)
+  if (
+    schema.type === FieldType.STRING ||
+    schema.type === FieldType.LONGFORM ||
+    schema.type === FieldType.BB_REFERENCE_SINGLE ||
+    schema.type === FieldType.OPTIONS ||
+    schema.type === FieldType.BARCODEQR
+  ) {
+    identifier = `to_char(${identifier})`
+  }
+  return identifier
+}
+
 function generateSelectStatement(
   json: QueryJson,
   knex: Knex
@@ -372,7 +392,15 @@ class InternalBuilder {
       iterate(
         filters.oneOf,
         (key: string, array) => {
-          query = query[fnc](key, Array.isArray(array) ? array : [array])
+          if (this.client === SqlClient.ORACLE) {
+            key = convertClobs(this.client, table, key)
+            query = query.whereRaw(
+              `${key} IN (?)`,
+              Array.isArray(array) ? array : [array]
+            )
+          } else {
+            query = query[fnc](key, Array.isArray(array) ? array : [array])
+          }
         },
         (key: string[], array) => {
           query = query[fnc](key, Array.isArray(array) ? array : [array])
@@ -436,8 +464,9 @@ class InternalBuilder {
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
+          const identifier = convertClobs(this.client, table, key)
           query = query[fnc](
-            `COALESCE(${quotedIdentifier(this.client, key)}, -1) = ?`,
+            `(${identifier} IS NOT NULL AND ${identifier} = ?)`,
             [value]
           )
         } else {
@@ -460,8 +489,9 @@ class InternalBuilder {
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
+          const identifier = convertClobs(this.client, table, key)
           query = query[fnc](
-            `COALESCE(${quotedIdentifier(this.client, key)}, -1) != ?`,
+            `(${identifier} IS NOT NULL AND ${identifier} != ?)`,
             [value]
           )
         } else {
@@ -707,8 +737,11 @@ class InternalBuilder {
       }
       const ret = query.insert(parsedBody).onConflict(primary).merge()
       return ret
-    } else if (this.client === SqlClient.MS_SQL) {
-      // No upsert or onConflict support in MSSQL yet, see:
+    } else if (
+      this.client === SqlClient.MS_SQL ||
+      this.client === SqlClient.ORACLE
+    ) {
+      // No upsert or onConflict support in MSSQL/Oracle yet, see:
       //   https://github.com/knex/knex/pull/6050
       return query.insert(parsedBody)
     }
@@ -867,7 +900,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     const config: Knex.Config = {
       client: sqlClient,
     }
-    if (sqlClient === SqlClient.SQL_LITE) {
+    if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) {
       config.useNullAsDefault = true
     }
 

packages/server/__mocks__/oracledb.ts

@@ -1,21 +0,0 @@
-const executeMock = jest.fn(() => ({
-  rows: [
-    {
-      a: "string",
-      b: 1,
-    },
-  ],
-}))
-
-const closeMock = jest.fn()
-
-class Connection {
-  execute = executeMock
-  close = closeMock
-}
-
-module.exports = {
-  getConnection: jest.fn(() => new Connection()),
-  executeMock,
-  closeMock,
-}

packages/server/src/api/routes/tests/search.spec.ts

@@ -40,13 +40,13 @@ import { structures } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 
 describe.each([
-  //["in-memory", undefined],
-  //["lucene", undefined],
-  //["sqs", undefined],
-  //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  // ["in-memory", undefined],
+  // ["lucene", undefined],
+  // ["sqs", undefined],
+  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
   [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
@@ -292,7 +292,7 @@ describe.each([
     })
 
     describe("equal", () => {
-      it.only("successfully finds true row", async () => {
+      it("successfully finds true row", async () => {
         await expectQuery({ equal: { isTrue: true } }).toMatchExactly([
           { isTrue: true },
         ])
@@ -1577,12 +1577,15 @@ describe.each([
     })
   })
 
-  describe("bigints", () => {
+  describe.only("bigints", () => {
     const SMALL = "1"
     const MEDIUM = "10000000"
 
     // Our bigints are int64s in most datasources.
-    const BIG = "9223372036854775807"
+    let BIG = "9223372036854775807"
+    if (name === DatabaseName.ORACLE) {
+      // BIG = "9223372036854775808"
+    }
 
     beforeAll(async () => {
       table = await createTable({
@@ -2415,25 +2418,25 @@ describe.each([
 
   describe.each([
     "名前", // Japanese for "name"
-    "Benutzer-ID", // German for "user ID", includes a hyphen
-    "numéro", // French for "number", includes an accent
-    "år", // Swedish for "year", includes a ring above
-    "naïve", // English word borrowed from French, includes an umlaut
-    "الاسم", // Arabic for "name"
-    "оплата", // Russian for "payment"
-    "पता", // Hindi for "address"
-    "用戶名", // Chinese for "username"
-    "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
-    "preço", // Portuguese for "price", includes a cedilla
-    "사용자명", // Korean for "username"
-    "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
-    "файл", // Bulgarian for "file"
-    "δεδομένα", // Greek for "data"
-    "geändert_am", // German for "modified on", includes an umlaut
-    "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
-    "São_Paulo", // Portuguese, includes an underscore and a tilde
-    "età", // Italian for "age", includes an accent
-    "ชื่อผู้ใช้", // Thai for "username"
+    // "Benutzer-ID", // German for "user ID", includes a hyphen
+    // "numéro", // French for "number", includes an accent
+    // "år", // Swedish for "year", includes a ring above
+    // "naïve", // English word borrowed from French, includes an umlaut
+    // "الاسم", // Arabic for "name"
+    // "оплата", // Russian for "payment"
+    // "पता", // Hindi for "address"
+    // "用戶名", // Chinese for "username"
+    // "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
+    // "preço", // Portuguese for "price", includes a cedilla
+    // "사용자명", // Korean for "username"
+    // "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
+    // "файл", // Bulgarian for "file"
+    // "δεδομένα", // Greek for "data"
+    // "geändert_am", // German for "modified on", includes an umlaut
+    // "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
+    // "São_Paulo", // Portuguese, includes an underscore and a tilde
+    // "età", // Italian for "age", includes an accent
+    // "ชื่อผู้ใช้", // Thai for "username"
   ])("non-ascii column name: %s", name => {
     beforeAll(async () => {
       table = await createTable({

packages/server/src/integrations/oracle.ts

@@ -360,11 +360,20 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       this.index = 1
       connection = await this.getConnection()
 
-      const options: ExecuteOptions = { autoCommit: true }
+      const options: ExecuteOptions = {
+        autoCommit: true,
+        fetchTypeHandler: function (metaData) {
+          if (metaData.dbType === oracledb.CLOB) {
+            return { type: oracledb.STRING }
+          }
+          return undefined
+        },
+      }
       const bindings: BindParameters = query.bindings || []
 
       this.log(query.sql, bindings)
-      return await connection.execute<T>(query.sql, bindings, options)
+      const result = await connection.execute(query.sql, bindings, options)
+      return result as Result<T>
     } finally {
       if (connection) {
         try {

packages/server/src/integrations/tests/utils/oracle.ts

@@ -8,7 +8,7 @@ let ports: Promise<testContainerUtils.Port[]>
 
 export async function getDatasource(): Promise<Datasource> {
   if (!ports) {
-    let image = "oracle/database:19.3.0.0-ee"
+    let image = "oracle/database:19.3.0.0-ee-slim-faststart"
     if (process.arch.startsWith("arm")) {
       image = "samhuang78/oracle-database:19.3.0-ee-slim-faststart"
     }
@@ -17,7 +17,7 @@ export async function getDatasource(): Promise<Datasource> {
       new GenericContainer(image)
         .withExposedPorts(1521)
         .withEnvironment({ ORACLE_PASSWORD: "password" })
-        .withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(10000))
+        .withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(60000))
     )
   }
 
@@ -26,23 +26,25 @@ export async function getDatasource(): Promise<Datasource> {
     throw new Error("Oracle port not found")
   }
 
+  const host = "127.0.0.1"
+  const user = "SYSTEM"
+  const password = "password"
+
   const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.ORACLE,
     plus: true,
-    config: {
-      host: "127.0.0.1",
-      port,
-      database: "postgres",
-      user: "SYS",
-      password: "password",
-    },
+    config: { host, port, user, password, database: "FREEPDB1" },
   }
 
-  const database = generator.guid().replaceAll("-", "")
+  const newUser = "a" + generator.guid().replaceAll("-", "")
   const client = await knexClient(datasource)
-  await client.raw(`CREATE DATABASE "${database}"`)
-  datasource.config!.database = database
+  await client.raw(`CREATE USER ${newUser} IDENTIFIED BY password`)
+  await client.raw(
+    `GRANT CONNECT, RESOURCE, CREATE VIEW, CREATE SESSION TO ${newUser}`
+  )
+  await client.raw(`GRANT UNLIMITED TABLESPACE TO ${newUser}`)
+  datasource.config!.user = newUser
 
   return datasource
 }
@@ -55,8 +57,17 @@ export async function knexClient(ds: Datasource) {
     throw new Error("Datasource source is not Oracle")
   }
 
-  return knex({
+  const db = ds.config.database || "FREEPDB1"
+  const connectString = `${ds.config.host}:${ds.config.port}/${db}`
+
+  const c = knex({
     client: "oracledb",
-    connection: ds.config,
+    connection: {
+      connectString,
+      user: ds.config.user,
+      password: ds.config.password,
+    },
   })
+
+  return c
 }

packages/server/src/utilities/rowProcessor/index.ts

@@ -315,6 +315,13 @@ export async function outputProcessing<T extends Row[] | Row>(
           column.subtype
         )
       }
+    } else if (column.type === FieldType.BIGINT) {
+      for (const row of enriched) {
+        if (row[property] == null) {
+          continue
+        }
+        row[property] = row[property].toString()
+      }
     }
   }