Moving some stuff around to make way for other services using the sql layers.

hosting/docker-compose.dev.yaml

@@ -42,12 +42,13 @@ services:
   couchdb-service:
     container_name: budi-couchdb3-dev
     restart: on-failure
-    image: budibase/couchdb
+    image: budibase/couchdb:v3.2.1-sqs
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
     ports:
       - "${COUCH_DB_PORT}:5984"
+      - "${COUCH_DB_SQS_PORT}:4984"
     volumes:
       - couchdb_data:/data
 

packages/backend-core/package.json

@@ -54,7 +54,8 @@
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "^7.5.4",
     "tar-fs": "2.1.1",
-    "uuid": "^8.3.2"
+    "uuid": "^8.3.2",
+    "knex": "2.4.2"
   },
   "devDependencies": {
     "@shopify/jest-koa-mocks": "5.1.1",

packages/backend-core/src/constants/db.ts

@@ -67,3 +67,8 @@ export const APP_DEV = prefixed(DocumentType.APP_DEV)
 export const APP_DEV_PREFIX = APP_DEV
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"
 export const SQLITE_DESIGN_DOC_ID = "_design/sqlite"
+export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
+export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
+export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
+export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
+export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"

packages/backend-core/src/environment.ts

@@ -159,6 +159,9 @@ const environment = {
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
   HTTP_LOGGING: httpLogging(),
   ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR,
+  // Couch/search
+  SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
+  SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   // smtp
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
   SMTP_USER: process.env.SMTP_USER,

packages/backend-core/src/index.ts

@@ -34,6 +34,7 @@ export * as docUpdates from "./docUpdates"
 export * from "./utils/Duration"
 export * as docIds from "./docIds"
 export * as security from "./security"
+export * as sql from "./sql"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal
 // circular dependencies

packages/backend-core/src/sql/index.ts

@@ -0,0 +1,4 @@
+export * as utils from "./utils"
+
+export { default as Sql } from "./sql"
+export { default as SqlTable } from "./sqlTable"

packages/backend-core/src/sql/sql.ts

@@ -1,13 +1,7 @@
 import { Knex, knex } from "knex"
-import { db as dbCore } from "@budibase/backend-core"
+import * as dbCore from "../db"
-import { QueryOptions } from "../../definitions/datasource"
+import { isIsoDateString, isValidFilter, getNativeSql } from "./utils"
-import {
+import { SqlStatements } from "./sqlStatements"
-  isIsoDateString,
-  SqlClient,
-  isValidFilter,
-  getNativeSql,
-  SqlStatements,
-} from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
   BBReferenceFieldMetadata,
@@ -24,8 +18,10 @@ import {
   Table,
   TableSourceType,
   INTERNAL_TABLE_SOURCE_ID,
+  SqlClient,
+  QueryOptions,
 } from "@budibase/types"
-import environment from "../../environment"
+import environment from "../environment"
 import { helpers } from "@budibase/shared-core"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any

packages/backend-core/src/sql/sqlStatements.ts

@@ -0,0 +1,79 @@
+import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types"
+import { Knex } from "knex"
+
+export class SqlStatements {
+  client: string
+  table: Table
+  allOr: boolean | undefined
+  constructor(
+    client: string,
+    table: Table,
+    { allOr }: { allOr?: boolean } = {}
+  ) {
+    this.client = client
+    this.table = table
+    this.allOr = allOr
+  }
+
+  getField(key: string): FieldSchema | undefined {
+    const fieldName = key.split(".")[1]
+    return this.table.schema[fieldName]
+  }
+
+  between(
+    query: Knex.QueryBuilder,
+    key: string,
+    low: number | string,
+    high: number | string
+  ) {
+    // Use a between operator if we have 2 valid range values
+    const field = this.getField(key)
+    if (
+      field?.type === FieldType.BIGINT &&
+      this.client === SqlClient.SQL_LITE
+    ) {
+      query = query.whereRaw(
+        `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
+        [low, high]
+      )
+    } else {
+      const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
+      query = query[fnc](key, [low, high])
+    }
+    return query
+  }
+
+  lte(query: Knex.QueryBuilder, key: string, low: number | string) {
+    // Use just a single greater than operator if we only have a low
+    const field = this.getField(key)
+    if (
+      field?.type === FieldType.BIGINT &&
+      this.client === SqlClient.SQL_LITE
+    ) {
+      query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
+        low,
+      ])
+    } else {
+      const fnc = this.allOr ? "orWhere" : "where"
+      query = query[fnc](key, ">=", low)
+    }
+    return query
+  }
+
+  gte(query: Knex.QueryBuilder, key: string, high: number | string) {
+    const field = this.getField(key)
+    // Use just a single less than operator if we only have a high
+    if (
+      field?.type === FieldType.BIGINT &&
+      this.client === SqlClient.SQL_LITE
+    ) {
+      query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
+        high,
+      ])
+    } else {
+      const fnc = this.allOr ? "orWhere" : "where"
+      query = query[fnc](key, "<=", high)
+    }
+    return query
+  }
+}

packages/backend-core/src/sql/sqlTable.ts

@@ -9,8 +9,9 @@ import {
   SqlQuery,
   Table,
   TableSourceType,
+  SqlClient,
 } from "@budibase/types"
-import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
+import { breakExternalTableId, getNativeSql } from "./utils"
 import { helpers, utils } from "@budibase/shared-core"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder

packages/backend-core/src/sql/utils.ts

@@ -0,0 +1,134 @@
+import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types"
+import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
+import { Knex } from "knex"
+import { SEPARATOR } from "../db"
+
+const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
+const ROW_ID_REGEX = /^\[.*]$/g
+const ENCODED_SPACE = encodeURIComponent(" ")
+
+export function isExternalTableID(tableId: string) {
+  return tableId.includes(DocumentType.DATASOURCE)
+}
+
+export function isInternalTableID(tableId: string) {
+  return !isExternalTableID(tableId)
+}
+
+export function getNativeSql(
+  query: Knex.SchemaBuilder | Knex.QueryBuilder
+): SqlQuery | SqlQuery[] {
+  let sql = query.toSQL()
+  if (Array.isArray(sql)) {
+    return sql as SqlQuery[]
+  }
+  let native: Knex.SqlNative | undefined
+  if (sql.toNative) {
+    native = sql.toNative()
+  }
+  return {
+    sql: native?.sql || sql.sql,
+    bindings: native?.bindings || sql.bindings,
+  } as SqlQuery
+}
+
+export function isExternalTable(table: Table) {
+  if (
+    table?.sourceId &&
+    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
+    table?.sourceId !== DEFAULT_BB_DATASOURCE_ID
+  ) {
+    return true
+  } else if (table?.sourceType === TableSourceType.EXTERNAL) {
+    return true
+  } else if (table?._id && isExternalTableID(table._id)) {
+    return true
+  }
+  return false
+}
+
+export function buildExternalTableId(datasourceId: string, tableName: string) {
+  // encode spaces
+  if (tableName.includes(" ")) {
+    tableName = encodeURIComponent(tableName)
+  }
+  return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
+}
+
+export function breakExternalTableId(tableId: string | undefined) {
+  if (!tableId) {
+    return {}
+  }
+  const parts = tableId.split(DOUBLE_SEPARATOR)
+  let datasourceId = parts.shift()
+  // if they need joined
+  let tableName = parts.join(DOUBLE_SEPARATOR)
+  // if contains encoded spaces, decode it
+  if (tableName.includes(ENCODED_SPACE)) {
+    tableName = decodeURIComponent(tableName)
+  }
+  return { datasourceId, tableName }
+}
+
+export function generateRowIdField(keyProps: any[] = []) {
+  if (!Array.isArray(keyProps)) {
+    keyProps = [keyProps]
+  }
+  for (let index in keyProps) {
+    if (keyProps[index] instanceof Buffer) {
+      keyProps[index] = keyProps[index].toString()
+    }
+  }
+  // this conserves order and types
+  // we have to swap the double quotes to single quotes for use in HBS statements
+  // when using the literal helper the double quotes can break things
+  return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'"))
+}
+
+export function isRowId(field: any) {
+  return (
+    Array.isArray(field) ||
+    (typeof field === "string" && field.match(ROW_ID_REGEX) != null)
+  )
+}
+
+export function convertRowId(field: any) {
+  if (Array.isArray(field)) {
+    return field[0]
+  }
+  if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) {
+    return field.substring(1, field.length - 1)
+  }
+  return field
+}
+
+// should always return an array
+export function breakRowIdField(_id: string | { _id: string }): any[] {
+  if (!_id) {
+    return []
+  }
+  // have to replace on the way back as we swapped out the double quotes
+  // when encoding, but JSON can't handle the single quotes
+  const id = typeof _id === "string" ? _id : _id._id
+  const decoded: string = decodeURIComponent(id).replace(/'/g, '"')
+  try {
+    const parsed = JSON.parse(decoded)
+    return Array.isArray(parsed) ? parsed : [parsed]
+  } catch (err) {
+    // wasn't json - likely was handlebars for a many to many
+    return [_id]
+  }
+}
+
+export function isIsoDateString(str: string) {
+  const trimmedValue = str.trim()
+  if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) {
+    return false
+  }
+  let d = new Date(trimmedValue)
+  return d.toISOString() === trimmedValue
+}
+
+export function isValidFilter(value: any) {
+  return value != null && value !== ""
+}

packages/server/src/constants/index.ts

@@ -173,8 +173,8 @@ export const devClientVersion = "0.0.0"
 export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 export const MAX_AUTOMATION_RECURRING_ERRORS = 5
 export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber"
-export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
+export const DEFAULT_JOBS_TABLE_ID = constants.DEFAULT_JOBS_TABLE_ID
-export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
+export const DEFAULT_INVENTORY_TABLE_ID = constants.DEFAULT_INVENTORY_TABLE_ID
-export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
+export const DEFAULT_EXPENSES_TABLE_ID = constants.DEFAULT_EXPENSES_TABLE_ID
-export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
+export const DEFAULT_EMPLOYEE_TABLE_ID = constants.DEFAULT_EMPLOYEE_TABLE_ID
-export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
+export const DEFAULT_BB_DATASOURCE_ID = constants.DEFAULT_BB_DATASOURCE_ID

packages/server/src/definitions/datasource.ts

@@ -3,8 +3,3 @@
  * internal to the server and don't need to *
  * be exposed for use by other services.    *
  ********************************************/
-
-export interface QueryOptions {
-  disableReturning?: boolean
-  disableBindings?: boolean
-}

packages/server/src/integrations/microsoftSqlServer.ts

@@ -1,40 +1,41 @@
 import {
+  ConnectionInfo,
+  DatasourceFeature,
   DatasourceFieldType,
+  DatasourcePlus,
+  DatasourcePlusQueryResponse,
   Integration,
   Operation,
-  Table,
-  TableSchema,
   QueryJson,
   QueryType,
-  SqlQuery,
-  DatasourcePlus,
-  DatasourceFeature,
-  ConnectionInfo,
-  SourceName,
   Schema,
+  SourceName,
+  SqlClient,
+  SqlQuery,
+  Table,
+  TableSchema,
   TableSourceType,
-  DatasourcePlusQueryResponse,
 } from "@budibase/types"
 import {
-  getSqlQuery,
   buildExternalTableId,
-  generateColumnDefinition,
-  finaliseExternalTables,
-  SqlClient,
   checkExternalTables,
+  finaliseExternalTables,
+  generateColumnDefinition,
+  getSqlQuery,
   HOST_ADDRESS,
 } from "./utils"
-import Sql from "./base/sql"
+import { MSSQLColumn, MSSQLTablesResponse } from "./base/types"
-import { MSSQLTablesResponse, MSSQLColumn } from "./base/types"
 import { getReadableErrorMessage } from "./base/errorMapping"
 import sqlServer from "mssql"
-
+import { sql } from "@budibase/backend-core"
-const DEFAULT_SCHEMA = "dbo"
-
 import { ConfidentialClientApplication } from "@azure/msal-node"
 
 import { utils } from "@budibase/shared-core"
 
+const Sql = sql.Sql
+
+const DEFAULT_SCHEMA = "dbo"
+
 enum MSSQLConfigAuthType {
   AZURE_ACTIVE_DIRECTORY = "Azure Active Directory",
   NTLM = "NTLM",
@@ -590,8 +591,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       scriptParts.push(createTableStatement)
     }
 
-    const schema = scriptParts.join("\n")
+    return scriptParts.join("\n")
-    return schema
   }
 }
 

packages/server/src/integrations/mysql.ts

@@ -14,10 +14,10 @@ import {
   TableSourceType,
   DatasourcePlusQueryResponse,
   SqlQueryBinding,
+  SqlClient,
 } from "@budibase/types"
 import {
   getSqlQuery,
-  SqlClient,
   buildExternalTableId,
   generateColumnDefinition,
   finaliseExternalTables,
@@ -26,11 +26,13 @@ import {
 } from "./utils"
 import dayjs from "dayjs"
 import { NUMBER_REGEX } from "../utilities"
-import Sql from "./base/sql"
 import { MySQLColumn } from "./base/types"
 import { getReadableErrorMessage } from "./base/errorMapping"
+import { sql } from "@budibase/backend-core"
 import mysql from "mysql2/promise"
 
+const Sql = sql.Sql
+
 interface MySQLConfig extends mysql.ConnectionOptions {
   database: string
   rejectUnauthorized: boolean

packages/server/src/integrations/oracle.ts

@@ -14,6 +14,7 @@ import {
   TableSourceType,
   Row,
   DatasourcePlusQueryResponse,
+  SqlClient,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -21,10 +22,8 @@ import {
   generateColumnDefinition,
   finaliseExternalTables,
   getSqlQuery,
-  SqlClient,
   HOST_ADDRESS,
 } from "./utils"
-import Sql from "./base/sql"
 import {
   BindParameters,
   Connection,
@@ -33,6 +32,9 @@ import {
   Result,
 } from "oracledb"
 import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
+import { sql } from "@budibase/backend-core"
+
+const Sql = sql.Sql
 
 let oracledb: any
 try {

packages/server/src/integrations/postgres.ts

@@ -13,17 +13,16 @@ import {
   Schema,
   TableSourceType,
   DatasourcePlusQueryResponse,
+  SqlClient,
 } from "@budibase/types"
 import {
   getSqlQuery,
   buildExternalTableId,
   generateColumnDefinition,
   finaliseExternalTables,
-  SqlClient,
   checkExternalTables,
   HOST_ADDRESS,
 } from "./utils"
-import Sql from "./base/sql"
 import { PostgresColumn } from "./base/types"
 import { escapeDangerousCharacters } from "../utilities"
 
@@ -31,7 +30,7 @@ import { Client, ClientConfig, types } from "pg"
 import { getReadableErrorMessage } from "./base/errorMapping"
 import { exec } from "child_process"
 import { storeTempFile } from "../utilities/fileSystem"
-import { env } from "@budibase/backend-core"
+import { env, sql } from "@budibase/backend-core"
 
 // Return "date" and "timestamp" types as plain strings.
 // This lets us reference the original stored timezone.
@@ -43,6 +42,7 @@ if (types) {
 }
 
 const JSON_REGEX = /'{.*}'::json/s
+const Sql = sql.Sql
 
 interface PostgresConfig {
   host: string

packages/server/src/integrations/utils/index.ts

@@ -1,2 +1 @@
 export * from "./utils"
-export { SqlStatements } from "./sqlStatements"

packages/server/src/integrations/utils/sqlStatements.ts

@@ -1,80 +0,0 @@
-import { FieldType, Table, FieldSchema } from "@budibase/types"
-import { SqlClient } from "./utils"
-import { Knex } from "knex"
-
-export class SqlStatements {
-  client: string
-  table: Table
-  allOr: boolean | undefined
-  constructor(
-    client: string,
-    table: Table,
-    { allOr }: { allOr?: boolean } = {}
-  ) {
-    this.client = client
-    this.table = table
-    this.allOr = allOr
-  }
-
-  getField(key: string): FieldSchema | undefined {
-    const fieldName = key.split(".")[1]
-    return this.table.schema[fieldName]
-  }
-
-  between(
-    query: Knex.QueryBuilder,
-    key: string,
-    low: number | string,
-    high: number | string
-  ) {
-    // Use a between operator if we have 2 valid range values
-    const field = this.getField(key)
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(
-        `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
-        [low, high]
-      )
-    } else {
-      const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
-      query = query[fnc](key, [low, high])
-    }
-    return query
-  }
-
-  lte(query: Knex.QueryBuilder, key: string, low: number | string) {
-    // Use just a single greater than operator if we only have a low
-    const field = this.getField(key)
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
-        low,
-      ])
-    } else {
-      const fnc = this.allOr ? "orWhere" : "where"
-      query = query[fnc](key, ">=", low)
-    }
-    return query
-  }
-
-  gte(query: Knex.QueryBuilder, key: string, high: number | string) {
-    const field = this.getField(key)
-    // Use just a single less than operator if we only have a high
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
-        high,
-      ])
-    } else {
-      const fnc = this.allOr ? "orWhere" : "where"
-      query = query[fnc](key, "<=", high)
-    }
-    return query
-  }
-}

packages/server/src/integrations/utils/utils.ts

@@ -3,23 +3,16 @@ import {
   Table,
   Datasource,
   FieldType,
-  TableSourceType,
   FieldSchema,
 } from "@budibase/types"
-import { context, objectStore } from "@budibase/backend-core"
+import { context, objectStore, sql } from "@budibase/backend-core"
 import { v4 } from "uuid"
 import { parseStringPromise as xmlParser } from "xml2js"
 import { formatBytes } from "../../utilities"
 import bl from "bl"
 import env from "../../environment"
-import { DocumentType, SEPARATOR } from "../../db/utils"
+import { InvalidColumns } from "../../constants"
-import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
 import { helpers, utils } from "@budibase/shared-core"
-import { Knex } from "knex"
-
-const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
-const ROW_ID_REGEX = /^\[.*]$/g
-const ENCODED_SPACE = encodeURIComponent(" ")
 
 type PrimitiveTypes =
   | FieldType.STRING
@@ -109,13 +102,15 @@ const SQL_TYPE_MAP: Record<string, PrimitiveTypes> = {
   ...SQL_OPTIONS_TYPE_MAP,
 }
 
-export enum SqlClient {
+export const isExternalTableID = sql.utils.isExternalTableID
-  MS_SQL = "mssql",
+export const isExternalTable = sql.utils.isExternalTable
-  POSTGRES = "pg",
+export const buildExternalTableId = sql.utils.buildExternalTableId
-  MY_SQL = "mysql2",
+export const breakExternalTableId = sql.utils.breakExternalTableId
-  ORACLE = "oracledb",
+export const generateRowIdField = sql.utils.generateRowIdField
-  SQL_LITE = "sqlite3",
+export const isRowId = sql.utils.isRowId
-}
+export const convertRowId = sql.utils.convertRowId
+export const breakRowIdField = sql.utils.breakRowIdField
+export const isValidFilter = sql.utils.isValidFilter
 
 const isCloud = env.isProd() && !env.SELF_HOSTED
 const isSelfHost = env.isProd() && env.SELF_HOSTED
@@ -125,119 +120,6 @@ export const HOST_ADDRESS = isSelfHost
   ? ""
   : "localhost"
 
-export function isExternalTableID(tableId: string) {
-  return tableId.includes(DocumentType.DATASOURCE)
-}
-
-export function isInternalTableID(tableId: string) {
-  return !isExternalTableID(tableId)
-}
-
-export function getNativeSql(
-  query: Knex.SchemaBuilder | Knex.QueryBuilder
-): SqlQuery | SqlQuery[] {
-  let sql = query.toSQL()
-  if (Array.isArray(sql)) {
-    return sql as SqlQuery[]
-  }
-  let native: Knex.SqlNative | undefined
-  if (sql.toNative) {
-    native = sql.toNative()
-  }
-  return {
-    sql: native?.sql || sql.sql,
-    bindings: native?.bindings || sql.bindings,
-  } as SqlQuery
-}
-
-export function isExternalTable(table: Table) {
-  if (
-    table?.sourceId &&
-    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
-    table?.sourceId !== DEFAULT_BB_DATASOURCE_ID
-  ) {
-    return true
-  } else if (table?.sourceType === TableSourceType.EXTERNAL) {
-    return true
-  } else if (table?._id && isExternalTableID(table._id)) {
-    return true
-  }
-  return false
-}
-
-export function buildExternalTableId(datasourceId: string, tableName: string) {
-  // encode spaces
-  if (tableName.includes(" ")) {
-    tableName = encodeURIComponent(tableName)
-  }
-  return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
-}
-
-export function breakExternalTableId(tableId: string | undefined) {
-  if (!tableId) {
-    return {}
-  }
-  const parts = tableId.split(DOUBLE_SEPARATOR)
-  let datasourceId = parts.shift()
-  // if they need joined
-  let tableName = parts.join(DOUBLE_SEPARATOR)
-  // if contains encoded spaces, decode it
-  if (tableName.includes(ENCODED_SPACE)) {
-    tableName = decodeURIComponent(tableName)
-  }
-  return { datasourceId, tableName }
-}
-
-export function generateRowIdField(keyProps: any[] = []) {
-  if (!Array.isArray(keyProps)) {
-    keyProps = [keyProps]
-  }
-  for (let index in keyProps) {
-    if (keyProps[index] instanceof Buffer) {
-      keyProps[index] = keyProps[index].toString()
-    }
-  }
-  // this conserves order and types
-  // we have to swap the double quotes to single quotes for use in HBS statements
-  // when using the literal helper the double quotes can break things
-  return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'"))
-}
-
-export function isRowId(field: any) {
-  return (
-    Array.isArray(field) ||
-    (typeof field === "string" && field.match(ROW_ID_REGEX) != null)
-  )
-}
-
-export function convertRowId(field: any) {
-  if (Array.isArray(field)) {
-    return field[0]
-  }
-  if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) {
-    return field.substring(1, field.length - 1)
-  }
-  return field
-}
-
-// should always return an array
-export function breakRowIdField(_id: string | { _id: string }): any[] {
-  if (!_id) {
-    return []
-  }
-  // have to replace on the way back as we swapped out the double quotes
-  // when encoding, but JSON can't handle the single quotes
-  const id = typeof _id === "string" ? _id : _id._id
-  const decoded: string = decodeURIComponent(id).replace(/'/g, '"')
-  try {
-    const parsed = JSON.parse(decoded)
-    return Array.isArray(parsed) ? parsed : [parsed]
-  } catch (err) {
-    // wasn't json - likely was handlebars for a many to many
-    return [_id]
-  }
-}
-
 export function generateColumnDefinition(config: {
   externalType: string
   autocolumn: boolean
@@ -297,15 +179,6 @@ export function isSQL(datasource: Datasource) {
   return helpers.isSQL(datasource)
 }
 
-export function isIsoDateString(str: string) {
-  const trimmedValue = str.trim()
-  if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) {
-    return false
-  }
-  let d = new Date(trimmedValue)
-  return d.toISOString() === trimmedValue
-}
-
 /**
  * Looks for columns which need to be copied over into the new table definitions, like relationships,
  * options types and views.
@@ -451,34 +324,6 @@ export function checkExternalTables(
   return errors
 }
 
-/**
- * Checks if the provided input is an object, but specifically not a date type object.
- * Used during coercion of types and relationship handling, dates are considered valid
- * and can be used as a display field, but objects and arrays cannot.
- * @param testValue an unknown type which this function will attempt to extract
- * a valid primary display string from.
- */
-export function getPrimaryDisplay(testValue: unknown): string | undefined {
-  if (testValue instanceof Date) {
-    return testValue.toISOString()
-  }
-  if (
-    Array.isArray(testValue) &&
-    testValue[0] &&
-    typeof testValue[0] !== "object"
-  ) {
-    return testValue.join(", ")
-  }
-  if (typeof testValue === "object") {
-    return undefined
-  }
-  return testValue as string
-}
-
-export function isValidFilter(value: any) {
-  return value != null && value !== ""
-}
-
 export async function handleXml(response: any) {
   let data,
     rawXml = await response.text()
@@ -517,12 +362,6 @@ export async function handleFileResponse(
     const contentLength = response.headers.get("content-length")
     if (contentLength) {
       size = parseInt(contentLength, 10)
-    } else {
-      const chunks: Buffer[] = []
-      for await (const chunk of response.body) {
-        chunks.push(chunk)
-        size += chunk.length
-      }
     }
 
     await objectStore.streamUpload({
@@ -533,7 +372,7 @@ export async function handleFileResponse(
       type: response.headers["content-type"],
     })
   }
-  presignedUrl = await objectStore.getPresignedUrl(bucket, key)
+  presignedUrl = objectStore.getPresignedUrl(bucket, key)
   return {
     data: {
       size,

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -11,15 +11,14 @@ import {
   SortOrder,
   SortType,
   Table,
+  SqlClient,
 } from "@budibase/types"
-import SqlQueryBuilder from "../../../../integrations/base/sql"
-import { SqlClient } from "../../../../integrations/utils"
 import {
   buildInternalRelationships,
   sqlOutputProcessing,
 } from "../../../../api/controllers/row/utils"
 import sdk from "../../../index"
-import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
+import { context, sql, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
 import {
   CONSTANT_INTERNAL_ROW_COLS,
   SQS_DATASOURCE_INTERNAL,
@@ -104,7 +103,7 @@ export async function search(
 ): Promise<SearchResponse<Row>> {
   const { paginate, query, ...params } = options
 
-  const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
+  const builder = new sql.Sql(SqlClient.SQL_LITE)
   const allTables = await sdk.tables.getAllInternalTables()
   const allTablesMap = buildTableMap(allTables)
   if (!table) {

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -5,12 +5,12 @@ import {
   QueryJson,
   Row,
   SearchFilters,
+  SqlClient,
 } from "@budibase/types"
 import { getSQLClient } from "./utils"
 import { cloneDeep } from "lodash"
 import datasources from "../datasources"
 import { makeExternalQuery } from "../../../integrations/base/query"
-import { SqlClient } from "../../../integrations/utils"
 import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils"
 
 const WRITE_OPERATIONS: Operation[] = [

packages/server/src/sdk/app/rows/utils.ts

@@ -9,12 +9,13 @@ import {
   SourceName,
   Table,
   TableSchema,
+  SqlClient,
 } from "@budibase/types"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { isRelationshipColumn } from "../../../db/utils"
-import { SqlClient, isSQL } from "../../../integrations/utils"
+import { isSQL } from "../../../integrations/utils"
 
 const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
   [SourceName.POSTGRES]: SqlClient.POSTGRES,

packages/types/src/sdk/search.ts

@@ -117,6 +117,11 @@ export interface QueryJson {
   tableAliases?: Record<string, string>
 }
 
+export interface QueryOptions {
+  disableReturning?: boolean
+  disableBindings?: boolean
+}
+
 export type SqlQueryBinding = Knex.Value[]
 
 export interface SqlQuery {
@@ -128,3 +133,11 @@ export enum EmptyFilterOption {
   RETURN_ALL = "all",
   RETURN_NONE = "none",
 }
+
+export enum SqlClient {
+  MS_SQL = "mssql",
+  POSTGRES = "pg",
+  MY_SQL = "mysql2",
+  ORACLE = "oracledb",
+  SQL_LITE = "sqlite3",
+}

scripts/build.js

@@ -3,11 +3,11 @@
 const start = Date.now()
 
 const fs = require("fs")
-const { cp, readdir, copyFile, mkdir } = require('node:fs/promises');
+const { cp, readdir, copyFile, mkdir } = require("node:fs/promises")
 const path = require("path")
 
 const { build } = require("esbuild")
-const { compile } = require('svelte/compiler')
+const { compile } = require("svelte/compiler")
 
 const {
   default: TsconfigPathsPlugin,
@@ -15,13 +15,13 @@ const {
 const { nodeExternalsPlugin } = require("esbuild-node-externals")
 
 const svelteCompilePlugin = {
-  name: 'svelteCompile',
+  name: "svelteCompile",
   setup(build) {
     // Compiles `.svelte` files into JS classes so that they can be directly imported into our
     // Typescript packages
-    build.onLoad({ filter: /\.svelte$/ }, async (args) => {
+    build.onLoad({ filter: /\.svelte$/ }, async args => {
-      const source = await fs.promises.readFile(args.path, 'utf8')
+      const source = await fs.promises.readFile(args.path, "utf8")
-      const dir = path.dirname(args.path);
+      const dir = path.dirname(args.path)
 
       try {
         const { js } = compile(source, { css: "injected", generate: "ssr" })
@@ -31,15 +31,15 @@ const svelteCompilePlugin = {
           contents: js.code,
           // The loader this is passed to, basically how the above provided content is "treated",
           // the contents provided above will be transpiled and bundled like any other JS file.
-          loader: 'js',
+          loader: "js",
           // Where to resolve any imports present in the loaded file
-          resolveDir: dir
+          resolveDir: dir,
         }
       } catch (e) {
         return { errors: [JSON.stringify(e)] }
       }
     })
-  }
+  },
 }
 
 var { argv } = require("yargs")
@@ -75,7 +75,7 @@ async function runBuild(entry, outfile) {
       svelteCompilePlugin,
       TsconfigPathsPlugin({ tsconfig: tsconfigPathPluginContent }),
       nodeExternalsPlugin({
-        allowList: ["@budibase/frontend-core", "svelte"]
+        allowList: ["@budibase/frontend-core", "svelte"],
       }),
     ],
     preserveSymlinks: true,
@@ -90,25 +90,39 @@ async function runBuild(entry, outfile) {
       "bcryptjs",
       "graphql/*",
       "bson",
+      "better-sqlite3",
+      "sqlite3",
+      "mysql",
+      "mysql2",
+      "oracle",
+      "pg",
+      "pg-query-stream",
+      "pg-native",
     ],
   }
 
-  await mkdir('dist', { recursive: true });
+  await mkdir("dist", { recursive: true })
 
   const hbsFiles = (async () => {
-    const dir = await readdir('./', { recursive: true });
+    const dir = await readdir("./", { recursive: true })
-    const files = dir.filter(entry => entry.endsWith('.hbs') || entry.endsWith('ivm.bundle.js'));
+    const files = dir.filter(
-    const fileCopyPromises = files.map(file => copyFile(file, `dist/${path.basename(file)}`))
+      entry => entry.endsWith(".hbs") || entry.endsWith("ivm.bundle.js")
+    )
+    const fileCopyPromises = files.map(file =>
+      copyFile(file, `dist/${path.basename(file)}`)
+    )
 
     await Promise.all(fileCopyPromises)
   })()
 
   const oldClientVersions = (async () => {
     try {
-      await cp('./build/oldClientVersions', './dist/oldClientVersions', { recursive: true });
+      await cp("./build/oldClientVersions", "./dist/oldClientVersions", {
+        recursive: true,
+      })
     } catch (e) {
       if (e.code !== "EEXIST" && e.code !== "ENOENT") {
-        throw e;
+        throw e
       }
     }
   })()