Merge pull request #13735 from Budibase/feature/audit-log-sqs

Audit log SQS

hosting/docker-compose.build.yaml

@@ -29,6 +29,7 @@ services:
       BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
       BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
       PLUGINS_DIR: ${PLUGINS_DIR}
+      SQS_SEARCH_ENABLE: 1
     depends_on:
       - worker-service
       - redis-service
@@ -56,6 +57,7 @@ services:
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
+      SQS_SEARCH_ENABLE: 1
     depends_on:
       - redis-service
       - minio-service

hosting/docker-compose.dev.yaml

@@ -42,12 +42,13 @@ services:
   couchdb-service:
     container_name: budi-couchdb3-dev
     restart: on-failure
-    image: budibase/couchdb
+    image: budibase/couchdb:v3.2.1-sqs
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
     ports:
       - "${COUCH_DB_PORT}:5984"
+      - "${COUCH_DB_SQS_PORT}:4984"
     volumes:
       - couchdb_data:/data
 

packages/backend-core/package.json

@@ -54,7 +54,8 @@
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "^7.5.4",
     "tar-fs": "2.1.1",
-    "uuid": "^8.3.2"
+    "uuid": "^8.3.2",
+    "knex": "2.4.2"
   },
   "devDependencies": {
     "@shopify/jest-koa-mocks": "5.1.1",

packages/backend-core/src/constants/db.ts

@@ -65,5 +65,11 @@ export const StaticDatabases = {
 export const APP_PREFIX = prefixed(DocumentType.APP)
 export const APP_DEV = prefixed(DocumentType.APP_DEV)
 export const APP_DEV_PREFIX = APP_DEV
+export const SQS_DATASOURCE_INTERNAL = "internal"
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"
 export const SQLITE_DESIGN_DOC_ID = "_design/sqlite"
+export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
+export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
+export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
+export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
+export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"

packages/backend-core/src/environment.ts

@@ -159,6 +159,9 @@ const environment = {
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
   HTTP_LOGGING: httpLogging(),
   ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR,
+  // Couch/search
+  SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
+  SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   // smtp
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
   SMTP_USER: process.env.SMTP_USER,

packages/backend-core/src/index.ts

@@ -34,6 +34,7 @@ export * as docUpdates from "./docUpdates"
 export * from "./utils/Duration"
 export * as docIds from "./docIds"
 export * as security from "./security"
+export * as sql from "./sql"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal
 // circular dependencies

packages/backend-core/src/sql/designDoc.ts

@@ -0,0 +1,17 @@
+import { PreSaveSQLiteDefinition } from "@budibase/types"
+import { SQLITE_DESIGN_DOC_ID } from "../constants"
+
+// the table id property defines which property in the document
+// to use when splitting the documents into different sqlite tables
+export function base(tableIdProp: string): PreSaveSQLiteDefinition {
+  return {
+    _id: SQLITE_DESIGN_DOC_ID,
+    language: "sqlite",
+    sql: {
+      tables: {},
+      options: {
+        table_name: tableIdProp,
+      },
+    },
+  }
+}

packages/backend-core/src/sql/index.ts

@@ -0,0 +1,5 @@
+export * as utils from "./utils"
+
+export { default as Sql } from "./sql"
+export { default as SqlTable } from "./sqlTable"
+export * as designDoc from "./designDoc"

packages/backend-core/src/sql/sql.ts

@@ -1,13 +1,7 @@
 import { Knex, knex } from "knex"
-import { db as dbCore } from "@budibase/backend-core"
-import { QueryOptions } from "../../definitions/datasource"
-import {
-  isIsoDateString,
-  SqlClient,
-  isValidFilter,
-  getNativeSql,
-  SqlStatements,
-} from "../utils"
+import * as dbCore from "../db"
+import { isIsoDateString, isValidFilter, getNativeSql } from "./utils"
+import { SqlStatements } from "./sqlStatements"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
   BBReferenceFieldMetadata,
@@ -24,8 +18,11 @@ import {
   Table,
   TableSourceType,
   INTERNAL_TABLE_SOURCE_ID,
+  SqlClient,
+  QueryOptions,
+  JsonTypes,
 } from "@budibase/types"
-import environment from "../../environment"
+import environment from "../environment"
 import { helpers } from "@budibase/shared-core"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
@@ -45,6 +42,7 @@ function likeKey(client: string, key: string): string {
     case SqlClient.MY_SQL:
       start = end = "`"
       break
+    case SqlClient.SQL_LITE:
     case SqlClient.ORACLE:
     case SqlClient.POSTGRES:
       start = end = '"'
@@ -53,9 +51,6 @@ function likeKey(client: string, key: string): string {
       start = "["
       end = "]"
       break
-    case SqlClient.SQL_LITE:
-      start = end = "'"
-      break
     default:
       throw new Error("Unknown client generating like key")
   }
@@ -207,17 +202,20 @@ class InternalBuilder {
         const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${getTableAlias(tableName)}.${updatedKey}`, value)
+          const alias = getTableAlias(tableName)
+          fn(alias ? `${alias}.${updatedKey}` : updatedKey, value)
         }
         if (opts.relationship && isRelationshipField) {
           const [filterTableName, property] = updatedKey.split(".")
-          fn(`${getTableAlias(filterTableName)}.${property}`, value)
+          const alias = getTableAlias(filterTableName)
+          fn(alias ? `${alias}.${property}` : property, value)
         }
       }
     }
 
     const like = (key: string, value: any) => {
-      const fnc = allOr ? "orWhere" : "where"
+      const fuzzyOr = filters?.fuzzyOr
+      const fnc = fuzzyOr || allOr ? "orWhere" : "where"
       // postgres supports ilike, nothing else does
       if (this.client === SqlClient.POSTGRES) {
         query = query[fnc](key, "ilike", `%${value}%`)
@@ -788,11 +786,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     return results.length ? results : [{ [operation.toLowerCase()]: true }]
   }
 
-  convertJsonStringColumns(
+  convertJsonStringColumns<T extends Record<string, any>>(
     table: Table,
-    results: Record<string, any>[],
+    results: T[],
     aliases?: Record<string, string>
-  ): Record<string, any>[] {
+  ): T[] {
     const tableName = getTableName(table)
     for (const [name, field] of Object.entries(table.schema)) {
       if (!this._isJsonColumn(field)) {
@@ -801,11 +799,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
       const aliasedTableName = (tableName && aliases?.[tableName]) || tableName
       const fullName = `${aliasedTableName}.${name}`
       for (let row of results) {
-        if (typeof row[fullName] === "string") {
-          row[fullName] = JSON.parse(row[fullName])
+        if (typeof row[fullName as keyof T] === "string") {
+          row[fullName as keyof T] = JSON.parse(row[fullName])
         }
-        if (typeof row[name] === "string") {
-          row[name] = JSON.parse(row[name])
+        if (typeof row[name as keyof T] === "string") {
+          row[name as keyof T] = JSON.parse(row[name])
         }
       }
     }
@@ -816,9 +814,8 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     field: FieldSchema
   ): field is JsonFieldMetadata | BBReferenceFieldMetadata {
     return (
-      field.type === FieldType.JSON ||
-      (field.type === FieldType.BB_REFERENCE &&
-        !helpers.schema.isDeprecatedSingleUserColumn(field))
+      JsonTypes.includes(field.type) &&
+      !helpers.schema.isDeprecatedSingleUserColumn(field)
     )
   }
 

packages/backend-core/src/sql/sqlStatements.ts

@@ -0,0 +1,79 @@
+import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types"
+import { Knex } from "knex"
+
+export class SqlStatements {
+  client: string
+  table: Table
+  allOr: boolean | undefined
+  constructor(
+    client: string,
+    table: Table,
+    { allOr }: { allOr?: boolean } = {}
+  ) {
+    this.client = client
+    this.table = table
+    this.allOr = allOr
+  }
+
+  getField(key: string): FieldSchema | undefined {
+    const fieldName = key.split(".")[1]
+    return this.table.schema[fieldName]
+  }
+
+  between(
+    query: Knex.QueryBuilder,
+    key: string,
+    low: number | string,
+    high: number | string
+  ) {
+    // Use a between operator if we have 2 valid range values
+    const field = this.getField(key)
+    if (
+      field?.type === FieldType.BIGINT &&
+      this.client === SqlClient.SQL_LITE
+    ) {
+      query = query.whereRaw(
+        `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
+        [low, high]
+      )
+    } else {
+      const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
+      query = query[fnc](key, [low, high])
+    }
+    return query
+  }
+
+  lte(query: Knex.QueryBuilder, key: string, low: number | string) {
+    // Use just a single greater than operator if we only have a low
+    const field = this.getField(key)
+    if (
+      field?.type === FieldType.BIGINT &&
+      this.client === SqlClient.SQL_LITE
+    ) {
+      query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
+        low,
+      ])
+    } else {
+      const fnc = this.allOr ? "orWhere" : "where"
+      query = query[fnc](key, ">=", low)
+    }
+    return query
+  }
+
+  gte(query: Knex.QueryBuilder, key: string, high: number | string) {
+    const field = this.getField(key)
+    // Use just a single less than operator if we only have a high
+    if (
+      field?.type === FieldType.BIGINT &&
+      this.client === SqlClient.SQL_LITE
+    ) {
+      query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
+        high,
+      ])
+    } else {
+      const fnc = this.allOr ? "orWhere" : "where"
+      query = query[fnc](key, "<=", high)
+    }
+    return query
+  }
+}

packages/backend-core/src/sql/sqlTable.ts

@@ -9,8 +9,9 @@ import {
   SqlQuery,
   Table,
   TableSourceType,
+  SqlClient,
 } from "@budibase/types"
-import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
+import { breakExternalTableId, getNativeSql } from "./utils"
 import { helpers, utils } from "@budibase/shared-core"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder

packages/backend-core/src/sql/utils.ts

@@ -0,0 +1,134 @@
+import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types"
+import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
+import { Knex } from "knex"
+import { SEPARATOR } from "../db"
+
+const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
+const ROW_ID_REGEX = /^\[.*]$/g
+const ENCODED_SPACE = encodeURIComponent(" ")
+
+export function isExternalTableID(tableId: string) {
+  return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR)
+}
+
+export function isInternalTableID(tableId: string) {
+  return !isExternalTableID(tableId)
+}
+
+export function getNativeSql(
+  query: Knex.SchemaBuilder | Knex.QueryBuilder
+): SqlQuery | SqlQuery[] {
+  let sql = query.toSQL()
+  if (Array.isArray(sql)) {
+    return sql as SqlQuery[]
+  }
+  let native: Knex.SqlNative | undefined
+  if (sql.toNative) {
+    native = sql.toNative()
+  }
+  return {
+    sql: native?.sql || sql.sql,
+    bindings: native?.bindings || sql.bindings,
+  } as SqlQuery
+}
+
+export function isExternalTable(table: Table) {
+  if (
+    table?.sourceId &&
+    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
+    table?.sourceId !== DEFAULT_BB_DATASOURCE_ID
+  ) {
+    return true
+  } else if (table?.sourceType === TableSourceType.EXTERNAL) {
+    return true
+  } else if (table?._id && isExternalTableID(table._id)) {
+    return true
+  }
+  return false
+}
+
+export function buildExternalTableId(datasourceId: string, tableName: string) {
+  // encode spaces
+  if (tableName.includes(" ")) {
+    tableName = encodeURIComponent(tableName)
+  }
+  return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
+}
+
+export function breakExternalTableId(tableId: string | undefined) {
+  if (!tableId) {
+    return {}
+  }
+  const parts = tableId.split(DOUBLE_SEPARATOR)
+  let datasourceId = parts.shift()
+  // if they need joined
+  let tableName = parts.join(DOUBLE_SEPARATOR)
+  // if contains encoded spaces, decode it
+  if (tableName.includes(ENCODED_SPACE)) {
+    tableName = decodeURIComponent(tableName)
+  }
+  return { datasourceId, tableName }
+}
+
+export function generateRowIdField(keyProps: any[] = []) {
+  if (!Array.isArray(keyProps)) {
+    keyProps = [keyProps]
+  }
+  for (let index in keyProps) {
+    if (keyProps[index] instanceof Buffer) {
+      keyProps[index] = keyProps[index].toString()
+    }
+  }
+  // this conserves order and types
+  // we have to swap the double quotes to single quotes for use in HBS statements
+  // when using the literal helper the double quotes can break things
+  return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'"))
+}
+
+export function isRowId(field: any) {
+  return (
+    Array.isArray(field) ||
+    (typeof field === "string" && field.match(ROW_ID_REGEX) != null)
+  )
+}
+
+export function convertRowId(field: any) {
+  if (Array.isArray(field)) {
+    return field[0]
+  }
+  if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) {
+    return field.substring(1, field.length - 1)
+  }
+  return field
+}
+
+// should always return an array
+export function breakRowIdField(_id: string | { _id: string }): any[] {
+  if (!_id) {
+    return []
+  }
+  // have to replace on the way back as we swapped out the double quotes
+  // when encoding, but JSON can't handle the single quotes
+  const id = typeof _id === "string" ? _id : _id._id
+  const decoded: string = decodeURIComponent(id).replace(/'/g, '"')
+  try {
+    const parsed = JSON.parse(decoded)
+    return Array.isArray(parsed) ? parsed : [parsed]
+  } catch (err) {
+    // wasn't json - likely was handlebars for a many to many
+    return [_id]
+  }
+}
+
+export function isIsoDateString(str: string) {
+  const trimmedValue = str.trim()
+  if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) {
+    return false
+  }
+  let d = new Date(trimmedValue)
+  return d.toISOString() === trimmedValue
+}
+
+export function isValidFilter(value: any) {
+  return value != null && value !== ""
+}

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -4,6 +4,8 @@
   import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
   import { getUserBindings } from "dataBinding"
   import { makePropSafe } from "@budibase/string-templates"
+  import { search } from "@budibase/frontend-core"
+  import { tables } from "stores/builder"
 
   export let schema
   export let filters
@@ -15,11 +17,10 @@
   let drawer
 
   $: tempValue = filters || []
-  $: schemaFields = Object.entries(schema || {}).map(
-    ([fieldName, fieldSchema]) => ({
-      name: fieldName, // Using the key as name if not defined in the schema, for example in some autogenerated columns
-      ...fieldSchema,
-    })
+  $: schemaFields = search.getFields(
+    $tables.list,
+    Object.values(schema || {}),
+    { allowLinks: true }
   )
 
   $: text = getText(filters)

packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte

@@ -1,11 +1,11 @@
 <script>
   import { Button, ActionButton, Drawer } from "@budibase/bbui"
+  import { search } from "@budibase/frontend-core"
   import { createEventDispatcher } from "svelte"
   import ColumnDrawer from "./ColumnDrawer.svelte"
   import { cloneDeep } from "lodash/fp"
   import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
-  import { selectedScreen } from "stores/builder"
-  import { getFields } from "helpers/searchFields"
+  import { selectedScreen, tables } from "stores/builder"
 
   export let componentInstance
   export let value = []
@@ -25,9 +25,13 @@
     : enrichedSchemaFields?.map(field => field.name)
   $: sanitisedValue = getValidColumns(value, options)
   $: updateBoundValue(sanitisedValue)
-  $: enrichedSchemaFields = getFields(Object.values(schema || {}), {
-    allowLinks: true,
-  })
+  $: enrichedSchemaFields = search.getFields(
+    $tables.list,
+    Object.values(schema || {}),
+    {
+      allowLinks: true,
+    }
+  )
 
   $: {
     value = (value || []).filter(

packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte

@@ -4,6 +4,7 @@
 
   import { dataFilters } from "@budibase/shared-core"
   import { FilterBuilder } from "@budibase/frontend-core"
+  import { tables } from "stores/builder"
 
   import { createEventDispatcher, onMount } from "svelte"
 
@@ -58,6 +59,7 @@
 <FilterBuilder
   bind:filters={rawFilters}
   behaviourFilters={true}
+  tables={$tables.list}
   {schemaFields}
   {datasource}
   {allowBindings}

packages/builder/src/components/design/settings/controls/SearchFieldSelect.svelte

@@ -1,9 +1,9 @@
 <script>
   import { Multiselect } from "@budibase/bbui"
+  import { search } from "@budibase/frontend-core"
   import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
   import { selectedScreen, tables } from "stores/builder"
   import { createEventDispatcher } from "svelte"
-  import { getFields } from "helpers/searchFields"
 
   export let componentInstance = {}
   export let value = ""
@@ -20,10 +20,9 @@
     if (!ds?.tableId) {
       return base.map(field => field.name)
     }
-    const currentTable = $tables.list.find(table => table._id === ds.tableId)
-    return getFields(base, { allowLinks: currentTable?.sql }).map(
-      field => field.name
-    )
+    return search
+      .getFields($tables.list, base, { allowLinks: true })
+      .map(field => field.name)
   }
 
   function getSelectedOption(selectedOptions, allOptions) {

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -16,11 +16,13 @@
   import { LuceneUtils, Constants } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import FilterUsers from "./FilterUsers.svelte"
+  import { getFields } from "../utils/searchFields"
 
   const { OperatorOptions } = Constants
 
   export let schemaFields
   export let filters = []
+  export let tables = []
   export let datasource
   export let behaviourFilters = false
   export let allowBindings = false
@@ -45,12 +47,12 @@
 
   const context = getContext("context")
 
-  $: fieldOptions = (schemaFields ?? [])
-    .filter(field => getValidOperatorsForType(field).length)
-    .map(field => ({
-      label: field.displayName || field.name,
-      value: field.name,
-    }))
+  $: fieldOptions = getFields(tables, schemaFields || [], {
+    allowLinks: true,
+  }).map(field => ({
+    label: field.displayName || field.name,
+    value: field.name,
+  }))
 
   const addFilter = () => {
     filters = [

packages/frontend-core/src/constants.js

@@ -6,6 +6,15 @@ export { Feature as Features } from "@budibase/types"
 import { BpmCorrelationKey } from "@budibase/shared-core"
 import { FieldType, BBReferenceFieldSubType } from "@budibase/types"
 
+export const BannedSearchTypes = [
+  FieldType.LINK,
+  FieldType.ATTACHMENTS,
+  FieldType.FORMULA,
+  FieldType.JSON,
+  "jsonarray",
+  "queryarray",
+]
+
 // Cookie names
 export const Cookies = {
   Auth: "budibase:auth",

packages/frontend-core/src/utils/index.js

@@ -4,6 +4,7 @@ export * as CookieUtils from "./cookies"
 export * as RoleUtils from "./roles"
 export * as Utils from "./utils"
 export * as RowUtils from "./rows"
+export * as search from "./searchFields"
 export { memo, derivedMemo } from "./memo"
 export { createWebsocket } from "./websocket"
 export * from "./download"

packages/frontend-core/src/utils/searchFields.js

@@ -1,13 +1,12 @@
-import { tables } from "stores/builder"
-import { BannedSearchTypes } from "../constants/backend"
-import { get } from "svelte/store"
+import { BannedSearchTypes } from "../constants"
 
-export function getTableFields(linkField) {
-  const table = get(tables).list.find(table => table._id === linkField.tableId)
+export function getTableFields(tables, linkField) {
+  const table = tables.find(table => table._id === linkField.tableId)
+  // TODO: mdrury - add support for this with SQS at some point
   if (!table || !table.sql) {
     return []
   }
-  const linkFields = getFields(Object.values(table.schema), {
+  const linkFields = getFields(tables, Object.values(table.schema), {
     allowLinks: false,
   })
   return linkFields.map(field => ({
@@ -16,7 +15,11 @@ export function getTableFields(linkField) {
   }))
 }
 
-export function getFields(fields, { allowLinks } = { allowLinks: true }) {
+export function getFields(
+  tables,
+  fields,
+  { allowLinks } = { allowLinks: true }
+) {
   let filteredFields = fields.filter(
     field => !BannedSearchTypes.includes(field.type)
   )
@@ -24,7 +27,7 @@ export function getFields(fields, { allowLinks } = { allowLinks: true }) {
     const linkFields = fields.filter(field => field.type === "link")
     for (let linkField of linkFields) {
       // only allow one depth of SQL relationship filtering
-      filteredFields = filteredFields.concat(getTableFields(linkField))
+      filteredFields = filteredFields.concat(getTableFields(tables, linkField))
     }
   }
   const staticFormulaFields = fields.filter(

packages/pro

@@ -1 +1 @@
-Subproject commit d3c3077011a8e20ed3c48dcd6301caca4120b6ac
+Subproject commit 1879d8686b1d9392707595a02cdd4981923e7f99

packages/server/src/api/controllers/row/views.ts

@@ -4,8 +4,8 @@ import {
   SearchRowResponse,
   SearchViewRowRequest,
   RequiredKeys,
-  SearchFilters,
   RowSearchParams,
+  SearchFilterKey,
 } from "@budibase/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
@@ -45,10 +45,7 @@ export async function searchView(
 
     // Carry over filters for unused fields
     Object.keys(body.query).forEach(key => {
-      const operator = key as keyof Omit<
-        SearchFilters,
-        "allOr" | "onEmptyFilter"
-      >
+      const operator = key as SearchFilterKey
       Object.keys(body.query[operator] || {}).forEach(field => {
         if (!existingFields.includes(db.removeKeyNumbering(field))) {
           query[operator]![field] = body.query[operator]![field]

packages/server/src/constants/index.ts

@@ -173,8 +173,8 @@ export const devClientVersion = "0.0.0"
 export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 export const MAX_AUTOMATION_RECURRING_ERRORS = 5
 export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber"
-export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
-export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
-export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
-export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
-export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
+export const DEFAULT_JOBS_TABLE_ID = constants.DEFAULT_JOBS_TABLE_ID
+export const DEFAULT_INVENTORY_TABLE_ID = constants.DEFAULT_INVENTORY_TABLE_ID
+export const DEFAULT_EXPENSES_TABLE_ID = constants.DEFAULT_EXPENSES_TABLE_ID
+export const DEFAULT_EMPLOYEE_TABLE_ID = constants.DEFAULT_EMPLOYEE_TABLE_ID
+export const DEFAULT_BB_DATASOURCE_ID = constants.DEFAULT_BB_DATASOURCE_ID

packages/server/src/db/utils.ts

@@ -40,7 +40,6 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte
 export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
 export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
 export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
-export const SQS_DATASOURCE_INTERNAL = "internal"
 export const ViewName = dbCore.ViewName
 export const InternalTables = dbCore.InternalTable
 export const UNICODE_MAX = dbCore.UNICODE_MAX

packages/server/src/definitions/datasource.ts

@@ -3,8 +3,3 @@
  * internal to the server and don't need to *
  * be exposed for use by other services.    *
  ********************************************/
-
-export interface QueryOptions {
-  disableReturning?: boolean
-  disableBindings?: boolean
-}

packages/server/src/integrations/microsoftSqlServer.ts

@@ -1,40 +1,41 @@
 import {
+  ConnectionInfo,
+  DatasourceFeature,
   DatasourceFieldType,
+  DatasourcePlus,
+  DatasourcePlusQueryResponse,
   Integration,
   Operation,
-  Table,
-  TableSchema,
   QueryJson,
   QueryType,
-  SqlQuery,
-  DatasourcePlus,
-  DatasourceFeature,
-  ConnectionInfo,
-  SourceName,
   Schema,
+  SourceName,
+  SqlClient,
+  SqlQuery,
+  Table,
+  TableSchema,
   TableSourceType,
-  DatasourcePlusQueryResponse,
 } from "@budibase/types"
 import {
-  getSqlQuery,
   buildExternalTableId,
-  generateColumnDefinition,
-  finaliseExternalTables,
-  SqlClient,
   checkExternalTables,
+  finaliseExternalTables,
+  generateColumnDefinition,
+  getSqlQuery,
   HOST_ADDRESS,
 } from "./utils"
-import Sql from "./base/sql"
-import { MSSQLTablesResponse, MSSQLColumn } from "./base/types"
+import { MSSQLColumn, MSSQLTablesResponse } from "./base/types"
 import { getReadableErrorMessage } from "./base/errorMapping"
 import sqlServer from "mssql"
-
-const DEFAULT_SCHEMA = "dbo"
-
+import { sql } from "@budibase/backend-core"
 import { ConfidentialClientApplication } from "@azure/msal-node"
 
 import { utils } from "@budibase/shared-core"
 
+const Sql = sql.Sql
+
+const DEFAULT_SCHEMA = "dbo"
+
 enum MSSQLConfigAuthType {
   AZURE_ACTIVE_DIRECTORY = "Azure Active Directory",
   NTLM = "NTLM",
@@ -590,8 +591,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       scriptParts.push(createTableStatement)
     }
 
-    const schema = scriptParts.join("\n")
-    return schema
+    return scriptParts.join("\n")
   }
 }
 

packages/server/src/integrations/mysql.ts

@@ -14,10 +14,10 @@ import {
   TableSourceType,
   DatasourcePlusQueryResponse,
   SqlQueryBinding,
+  SqlClient,
 } from "@budibase/types"
 import {
   getSqlQuery,
-  SqlClient,
   buildExternalTableId,
   generateColumnDefinition,
   finaliseExternalTables,
@@ -26,11 +26,13 @@ import {
 } from "./utils"
 import dayjs from "dayjs"
 import { NUMBER_REGEX } from "../utilities"
-import Sql from "./base/sql"
 import { MySQLColumn } from "./base/types"
 import { getReadableErrorMessage } from "./base/errorMapping"
+import { sql } from "@budibase/backend-core"
 import mysql from "mysql2/promise"
 
+const Sql = sql.Sql
+
 interface MySQLConfig extends mysql.ConnectionOptions {
   database: string
   rejectUnauthorized: boolean

packages/server/src/integrations/oracle.ts

@@ -14,6 +14,7 @@ import {
   TableSourceType,
   Row,
   DatasourcePlusQueryResponse,
+  SqlClient,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -21,10 +22,8 @@ import {
   generateColumnDefinition,
   finaliseExternalTables,
   getSqlQuery,
-  SqlClient,
   HOST_ADDRESS,
 } from "./utils"
-import Sql from "./base/sql"
 import {
   BindParameters,
   Connection,
@@ -33,6 +32,9 @@ import {
   Result,
 } from "oracledb"
 import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
+import { sql } from "@budibase/backend-core"
+
+const Sql = sql.Sql
 
 let oracledb: any
 try {

packages/server/src/integrations/postgres.ts

@@ -13,17 +13,16 @@ import {
   Schema,
   TableSourceType,
   DatasourcePlusQueryResponse,
+  SqlClient,
 } from "@budibase/types"
 import {
   getSqlQuery,
   buildExternalTableId,
   generateColumnDefinition,
   finaliseExternalTables,
-  SqlClient,
   checkExternalTables,
   HOST_ADDRESS,
 } from "./utils"
-import Sql from "./base/sql"
 import { PostgresColumn } from "./base/types"
 import { escapeDangerousCharacters } from "../utilities"
 
@@ -31,7 +30,7 @@ import { Client, ClientConfig, types } from "pg"
 import { getReadableErrorMessage } from "./base/errorMapping"
 import { exec } from "child_process"
 import { storeTempFile } from "../utilities/fileSystem"
-import { env } from "@budibase/backend-core"
+import { env, sql } from "@budibase/backend-core"
 
 // Return "date" and "timestamp" types as plain strings.
 // This lets us reference the original stored timezone.
@@ -43,6 +42,7 @@ if (types) {
 }
 
 const JSON_REGEX = /'{.*}'::json/s
+const Sql = sql.Sql
 
 interface PostgresConfig {
   host: string

packages/server/src/integrations/tests/rest.spec.ts

@@ -664,6 +664,7 @@ describe("REST Integration", () => {
             }),
             get: (header: any) => {
               if (header === "content-type") return contentType
+              if (header === "content-length") return responseData.byteLength
               if (header === "content-disposition")
                 return `attachment; filename="${filename}"`
             },
@@ -709,6 +710,7 @@ describe("REST Integration", () => {
             }),
             get: (header: any) => {
               if (header === "content-type") return contentType
+              if (header === "content-length") return responseData.byteLength
               if (header === "content-disposition")
                 // eslint-disable-next-line no-useless-escape
                 return `attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`

packages/server/src/integrations/tests/sql.spec.ts

@@ -1,12 +1,14 @@
-import { SqlClient } from "../utils"
-import Sql from "../base/sql"
 import {
   FieldType,
   Operation,
   QueryJson,
   Table,
   TableSourceType,
+  SqlClient,
 } from "@budibase/types"
+import { sql } from "@budibase/backend-core"
+
+const Sql = sql.Sql
 
 const TABLE_NAME = "test"
 const TABLE: Table = {

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -6,13 +6,15 @@ import {
   SqlQuery,
   Table,
   TableSourceType,
+  SqlClient,
 } from "@budibase/types"
+import { sql } from "@budibase/backend-core"
 import { join } from "path"
-import Sql from "../base/sql"
-import { SqlClient } from "../utils"
 import { generator } from "@budibase/backend-core/tests"
 import sdk from "../../sdk"
 
+const Sql = sql.Sql
+
 // this doesn't exist strictly
 const TABLE: Table = {
   type: "table",

packages/server/src/integrations/utils/index.ts

@@ -1,2 +1 @@
 export * from "./utils"
-export { SqlStatements } from "./sqlStatements"

packages/server/src/integrations/utils/sqlStatements.ts

@@ -1,80 +0,0 @@
-import { FieldType, Table, FieldSchema } from "@budibase/types"
-import { SqlClient } from "./utils"
-import { Knex } from "knex"
-
-export class SqlStatements {
-  client: string
-  table: Table
-  allOr: boolean | undefined
-  constructor(
-    client: string,
-    table: Table,
-    { allOr }: { allOr?: boolean } = {}
-  ) {
-    this.client = client
-    this.table = table
-    this.allOr = allOr
-  }
-
-  getField(key: string): FieldSchema | undefined {
-    const fieldName = key.split(".")[1]
-    return this.table.schema[fieldName]
-  }
-
-  between(
-    query: Knex.QueryBuilder,
-    key: string,
-    low: number | string,
-    high: number | string
-  ) {
-    // Use a between operator if we have 2 valid range values
-    const field = this.getField(key)
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(
-        `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
-        [low, high]
-      )
-    } else {
-      const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
-      query = query[fnc](key, [low, high])
-    }
-    return query
-  }
-
-  lte(query: Knex.QueryBuilder, key: string, low: number | string) {
-    // Use just a single greater than operator if we only have a low
-    const field = this.getField(key)
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
-        low,
-      ])
-    } else {
-      const fnc = this.allOr ? "orWhere" : "where"
-      query = query[fnc](key, ">=", low)
-    }
-    return query
-  }
-
-  gte(query: Knex.QueryBuilder, key: string, high: number | string) {
-    const field = this.getField(key)
-    // Use just a single less than operator if we only have a high
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
-        high,
-      ])
-    } else {
-      const fnc = this.allOr ? "orWhere" : "where"
-      query = query[fnc](key, "<=", high)
-    }
-    return query
-  }
-}

packages/server/src/integrations/utils/utils.ts

@@ -3,23 +3,16 @@ import {
   Table,
   Datasource,
   FieldType,
-  TableSourceType,
   FieldSchema,
 } from "@budibase/types"
-import { context, objectStore } from "@budibase/backend-core"
+import { context, objectStore, sql } from "@budibase/backend-core"
 import { v4 } from "uuid"
 import { parseStringPromise as xmlParser } from "xml2js"
 import { formatBytes } from "../../utilities"
 import bl from "bl"
 import env from "../../environment"
-import { DocumentType, SEPARATOR } from "../../db/utils"
-import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
+import { InvalidColumns } from "../../constants"
 import { helpers, utils } from "@budibase/shared-core"
-import { Knex } from "knex"
-
-const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
-const ROW_ID_REGEX = /^\[.*]$/g
-const ENCODED_SPACE = encodeURIComponent(" ")
 
 type PrimitiveTypes =
   | FieldType.STRING
@@ -114,13 +107,15 @@ const SQL_TYPE_MAP: Record<string, PrimitiveTypes> = {
   ...SQL_OPTIONS_TYPE_MAP,
 }
 
-export enum SqlClient {
-  MS_SQL = "mssql",
-  POSTGRES = "pg",
-  MY_SQL = "mysql2",
-  ORACLE = "oracledb",
-  SQL_LITE = "sqlite3",
-}
+export const isExternalTableID = sql.utils.isExternalTableID
+export const isExternalTable = sql.utils.isExternalTable
+export const buildExternalTableId = sql.utils.buildExternalTableId
+export const breakExternalTableId = sql.utils.breakExternalTableId
+export const generateRowIdField = sql.utils.generateRowIdField
+export const isRowId = sql.utils.isRowId
+export const convertRowId = sql.utils.convertRowId
+export const breakRowIdField = sql.utils.breakRowIdField
+export const isValidFilter = sql.utils.isValidFilter
 
 const isCloud = env.isProd() && !env.SELF_HOSTED
 const isSelfHost = env.isProd() && env.SELF_HOSTED
@@ -130,119 +125,6 @@ export const HOST_ADDRESS = isSelfHost
   ? ""
   : "localhost"
 
-export function isExternalTableID(tableId: string) {
-  return tableId.includes(DocumentType.DATASOURCE)
-}
-
-export function isInternalTableID(tableId: string) {
-  return !isExternalTableID(tableId)
-}
-
-export function getNativeSql(
-  query: Knex.SchemaBuilder | Knex.QueryBuilder
-): SqlQuery | SqlQuery[] {
-  let sql = query.toSQL()
-  if (Array.isArray(sql)) {
-    return sql as SqlQuery[]
-  }
-  let native: Knex.SqlNative | undefined
-  if (sql.toNative) {
-    native = sql.toNative()
-  }
-  return {
-    sql: native?.sql || sql.sql,
-    bindings: native?.bindings || sql.bindings,
-  } as SqlQuery
-}
-
-export function isExternalTable(table: Table) {
-  if (
-    table?.sourceId &&
-    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
-    table?.sourceId !== DEFAULT_BB_DATASOURCE_ID
-  ) {
-    return true
-  } else if (table?.sourceType === TableSourceType.EXTERNAL) {
-    return true
-  } else if (table?._id && isExternalTableID(table._id)) {
-    return true
-  }
-  return false
-}
-
-export function buildExternalTableId(datasourceId: string, tableName: string) {
-  // encode spaces
-  if (tableName.includes(" ")) {
-    tableName = encodeURIComponent(tableName)
-  }
-  return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
-}
-
-export function breakExternalTableId(tableId: string | undefined) {
-  if (!tableId) {
-    return {}
-  }
-  const parts = tableId.split(DOUBLE_SEPARATOR)
-  let datasourceId = parts.shift()
-  // if they need joined
-  let tableName = parts.join(DOUBLE_SEPARATOR)
-  // if contains encoded spaces, decode it
-  if (tableName.includes(ENCODED_SPACE)) {
-    tableName = decodeURIComponent(tableName)
-  }
-  return { datasourceId, tableName }
-}
-
-export function generateRowIdField(keyProps: any[] = []) {
-  if (!Array.isArray(keyProps)) {
-    keyProps = [keyProps]
-  }
-  for (let index in keyProps) {
-    if (keyProps[index] instanceof Buffer) {
-      keyProps[index] = keyProps[index].toString()
-    }
-  }
-  // this conserves order and types
-  // we have to swap the double quotes to single quotes for use in HBS statements
-  // when using the literal helper the double quotes can break things
-  return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'"))
-}
-
-export function isRowId(field: any) {
-  return (
-    Array.isArray(field) ||
-    (typeof field === "string" && field.match(ROW_ID_REGEX) != null)
-  )
-}
-
-export function convertRowId(field: any) {
-  if (Array.isArray(field)) {
-    return field[0]
-  }
-  if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) {
-    return field.substring(1, field.length - 1)
-  }
-  return field
-}
-
-// should always return an array
-export function breakRowIdField(_id: string | { _id: string }): any[] {
-  if (!_id) {
-    return []
-  }
-  // have to replace on the way back as we swapped out the double quotes
-  // when encoding, but JSON can't handle the single quotes
-  const id = typeof _id === "string" ? _id : _id._id
-  const decoded: string = decodeURIComponent(id).replace(/'/g, '"')
-  try {
-    const parsed = JSON.parse(decoded)
-    return Array.isArray(parsed) ? parsed : [parsed]
-  } catch (err) {
-    // wasn't json - likely was handlebars for a many to many
-    return [_id]
-  }
-}
-
 export function generateColumnDefinition(config: {
   externalType: string
   autocolumn: boolean
@@ -302,15 +184,6 @@ export function isSQL(datasource: Datasource) {
   return helpers.isSQL(datasource)
 }
 
-export function isIsoDateString(str: string) {
-  const trimmedValue = str.trim()
-  if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) {
-    return false
-  }
-  let d = new Date(trimmedValue)
-  return d.toISOString() === trimmedValue
-}
-
 /**
  * Looks for columns which need to be copied over into the new table definitions, like relationships,
  * options types and views.
@@ -457,37 +330,8 @@ export function checkExternalTables(
   return errors
 }
 
-/**
- * Checks if the provided input is an object, but specifically not a date type object.
- * Used during coercion of types and relationship handling, dates are considered valid
- * and can be used as a display field, but objects and arrays cannot.
- * @param testValue an unknown type which this function will attempt to extract
- * a valid primary display string from.
- */
-export function getPrimaryDisplay(testValue: unknown): string | undefined {
-  if (testValue instanceof Date) {
-    return testValue.toISOString()
-  }
-  if (
-    Array.isArray(testValue) &&
-    testValue[0] &&
-    typeof testValue[0] !== "object"
-  ) {
-    return testValue.join(", ")
-  }
-  if (typeof testValue === "object") {
-    return undefined
-  }
-  return testValue as string
-}
-
-export function isValidFilter(value: any) {
-  return value != null && value !== ""
-}
-
 export async function handleXml(rawXml: string) {
-  let data
-  data =
+  let data =
     (await xmlParser(rawXml, {
       explicitArray: false,
       trim: true,
@@ -522,12 +366,6 @@ export async function handleFileResponse(
     const contentLength = response.headers.get("content-length")
     if (contentLength) {
       size = parseInt(contentLength, 10)
-    } else {
-      const chunks: Buffer[] = []
-      for await (const chunk of response.body) {
-        chunks.push(chunk)
-        size += chunk.length
-      }
     }
 
     await objectStore.streamUpload({
@@ -538,7 +376,7 @@ export async function handleFileResponse(
       type: response.headers["content-type"],
     })
   }
-  presignedUrl = await objectStore.getPresignedUrl(bucket, key)
+  presignedUrl = objectStore.getPresignedUrl(bucket, key)
   return {
     data: {
       size,

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -4,30 +4,33 @@ import {
   QueryJson,
   RelationshipFieldMetadata,
   Row,
-  SearchFilters,
   RowSearchParams,
+  SearchFilters,
   SearchResponse,
   SortDirection,
   SortOrder,
   SortType,
+  SqlClient,
   Table,
 } from "@budibase/types"
-import SqlQueryBuilder from "../../../../integrations/base/sql"
-import { SqlClient } from "../../../../integrations/utils"
 import {
   buildInternalRelationships,
   sqlOutputProcessing,
 } from "../../../../api/controllers/row/utils"
 import sdk from "../../../index"
-import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
 import {
-  CONSTANT_INTERNAL_ROW_COLS,
+  context,
+  sql,
+  SQLITE_DESIGN_DOC_ID,
   SQS_DATASOURCE_INTERNAL,
-} from "../../../../db/utils"
+} from "@budibase/backend-core"
+import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
 import AliasTables from "../sqlAlias"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
 
+const builder = new sql.Sql(SqlClient.SQL_LITE)
+
 function buildInternalFieldList(
   table: Table,
   tables: Table[],
@@ -97,13 +100,39 @@ function buildTableMap(tables: Table[]) {
   return tableMap
 }
 
+async function runSqlQuery(json: QueryJson, tables: Table[]) {
+  const alias = new AliasTables(tables.map(table => table.name))
+  return await alias.queryWithAliasing(json, async json => {
+    const query = builder._query(json, {
+      disableReturning: true,
+    })
+
+    if (Array.isArray(query)) {
+      throw new Error("SQS cannot currently handle multiple queries")
+    }
+
+    let sql = query.sql
+    let bindings = query.bindings
+
+    // quick hack for docIds
+    sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
+    sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
+
+    if (Array.isArray(query)) {
+      throw new Error("SQS cannot currently handle multiple queries")
+    }
+
+    const db = context.getAppDB()
+    return await db.sql<Row>(sql, bindings)
+  })
+}
+
 export async function search(
   options: RowSearchParams,
   table: Table
 ): Promise<SearchResponse<Row>> {
   const { paginate, query, ...params } = options
 
-  const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
   const allTables = await sdk.tables.getAllInternalTables()
   const allTablesMap = buildTableMap(allTables)
   if (!table) {
@@ -146,62 +175,72 @@ export async function search(
       },
     }
   }
+
+  if (params.bookmark && typeof params.bookmark !== "number") {
+    throw new Error("Unable to paginate with string based bookmarks")
+  }
+  const bookmark: number = (params.bookmark as number) || 1
+  const limit = params.limit
   if (paginate && params.limit) {
     request.paginate = {
-      limit: params.limit,
-      page: params.bookmark,
+      limit: params.limit + 1,
+      page: bookmark,
     }
   }
   try {
-    const alias = new AliasTables(allTables.map(table => table.name))
-    const rows = await alias.queryWithAliasing(request, async json => {
-      const query = builder._query(json, {
-        disableReturning: true,
-      })
+    const rows = await runSqlQuery(request, allTables)
 
-      if (Array.isArray(query)) {
-        throw new Error("SQS cannot currently handle multiple queries")
-      }
-
-      let sql = query.sql
-      let bindings = query.bindings
-
-      // quick hack for docIds
-      sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
-      sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
-
-      const db = context.getAppDB()
-      const rows = await db.sql<Row>(sql, bindings)
-      return rows
-    })
-
-    // process from the format of tableId.column to expected format
-    const processed = await sqlOutputProcessing(
-      rows,
-      table!,
-      allTablesMap,
-      relationships,
-      {
+    // process from the format of tableId.column to expected format also
+    // make sure JSON columns corrected
+    const processed = builder.convertJsonStringColumns<Row>(
+      table,
+      await sqlOutputProcessing(rows, table!, allTablesMap, relationships, {
         sqs: true,
-      }
+      })
     )
 
-    const output = {
-      rows: await outputProcessing<Row[]>(table, processed, {
-        preserveLinks: true,
-        squash: true,
-      }),
+    // check for pagination final row
+    let nextRow: Row | undefined
+    if (paginate && params.limit && processed.length > params.limit) {
+      nextRow = processed.pop()
     }
 
+    // get the rows
+    let finalRows = await outputProcessing<Row[]>(table, processed, {
+      preserveLinks: true,
+      squash: true,
+    })
+
+    // check if we need to pick specific rows out
     if (options.fields) {
       const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS]
-      output.rows = output.rows.map((r: any) => pick(r, fields))
+      finalRows = finalRows.map((r: any) => pick(r, fields))
     }
 
-    return output
+    // check for pagination
+    if (paginate && limit) {
+      const response: SearchResponse<Row> = {
+        rows: finalRows,
+      }
+      const prevLimit = request.paginate!.limit
+      request.paginate = {
+        limit: 1,
+        page: bookmark * prevLimit + 1,
+      }
+      const hasNextPage = !!nextRow
+      response.hasNextPage = hasNextPage
+      if (hasNextPage) {
+        response.bookmark = bookmark + 1
+      }
+      return response
+    } else {
+      return {
+        rows: finalRows,
+      }
+    }
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message
-    if (err.status === 404 && err.message?.includes(SQLITE_DESIGN_DOC_ID)) {
+    if (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID)) {
       await sdk.tables.sqs.syncDefinition()
       return search(options, table)
     }

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -5,13 +5,13 @@ import {
   QueryJson,
   Row,
   SearchFilters,
+  SqlClient,
 } from "@budibase/types"
+import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core"
 import { getSQLClient } from "./utils"
 import { cloneDeep } from "lodash"
 import datasources from "../datasources"
 import { makeExternalQuery } from "../../../integrations/base/query"
-import { SqlClient } from "../../../integrations/utils"
-import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils"
 
 const WRITE_OPERATIONS: Operation[] = [
   Operation.CREATE,

packages/server/src/sdk/app/rows/utils.ts

@@ -11,12 +11,13 @@ import {
   SourceName,
   Table,
   TableSchema,
+  SqlClient,
 } from "@budibase/types"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { isRelationshipColumn } from "../../../db/utils"
-import { isSQL, SqlClient } from "../../../integrations/utils"
+import { isSQL } from "../../../integrations/utils"
 
 const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
   [SourceName.POSTGRES]: SqlClient.POSTGRES,

packages/server/src/sdk/app/tables/getters.ts

@@ -15,6 +15,7 @@ import {
 } from "@budibase/types"
 import datasources from "../datasources"
 import sdk from "../../../sdk"
+import env from "../../../environment"
 
 export function processTable(table: Table): Table {
   if (!table) {
@@ -27,12 +28,16 @@ export function processTable(table: Table): Table {
       sourceType: TableSourceType.EXTERNAL,
     }
   } else {
-    return {
+    const processed: Table = {
       ...table,
       type: "table",
       sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
       sourceType: TableSourceType.INTERNAL,
     }
+    if (env.SQS_SEARCH_ENABLE) {
+      processed.sql = !!env.SQS_SEARCH_ENABLE
+    }
+    return processed
   }
 }
 

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -1,33 +1,20 @@
-import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
+import { context, sql, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
 import {
   FieldType,
   RelationshipFieldMetadata,
   SQLiteDefinition,
+  PreSaveSQLiteDefinition,
   SQLiteTable,
   SQLiteTables,
   SQLiteType,
   Table,
 } from "@budibase/types"
-import { cloneDeep } from "lodash"
 import tablesSdk from "../"
 import {
   CONSTANT_INTERNAL_ROW_COLS,
   generateJunctionTableID,
 } from "../../../../db/utils"
 
-type PreSaveSQLiteDefinition = Omit<SQLiteDefinition, "_rev">
-
-const BASIC_SQLITE_DOC: PreSaveSQLiteDefinition = {
-  _id: SQLITE_DESIGN_DOC_ID,
-  language: "sqlite",
-  sql: {
-    tables: {},
-    options: {
-      table_name: "tableId",
-    },
-  },
-}
-
 const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
   [FieldType.DATETIME]: SQLiteType.TEXT,
@@ -108,7 +95,7 @@ function mapTable(table: Table): SQLiteTables {
 // nothing exists, need to iterate though existing tables
 async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
   const tables = await tablesSdk.getAllInternalTables()
-  const definition = cloneDeep(BASIC_SQLITE_DOC)
+  const definition = sql.designDoc.base("tableId")
   for (let table of tables) {
     definition.sql.tables = {
       ...definition.sql.tables,

packages/types/src/documents/app/row.ts

@@ -118,6 +118,15 @@ export enum FieldType {
   BB_REFERENCE_SINGLE = "bb_reference_single",
 }
 
+export const JsonTypes = [
+  FieldType.ATTACHMENT_SINGLE,
+  FieldType.ATTACHMENTS,
+  // only BB_REFERENCE is JSON, it's an array, BB_REFERENCE_SINGLE is a string type
+  FieldType.BB_REFERENCE,
+  FieldType.JSON,
+  FieldType.ARRAY,
+]
+
 export interface RowAttachment {
   size: number
   name: string

packages/types/src/documents/app/sqlite.ts

@@ -29,3 +29,5 @@ export interface SQLiteDefinition {
     }
   }
 }
+
+export type PreSaveSQLiteDefinition = Omit<SQLiteDefinition, "_rev">

packages/types/src/documents/global/auditLogs.ts

@@ -2,6 +2,7 @@ import { Document } from "../document"
 import { Event } from "../../sdk"
 
 export const AuditLogSystemUser = "SYSTEM"
+export const AUDIT_LOG_TYPE = "auditLog"
 
 export type FallbackInfo = {
   appName?: string
@@ -15,5 +16,6 @@ export interface AuditLogDoc extends Document {
   timestamp: string
   metadata: any
   name: string
+  type?: "auditLog"
   fallback?: FallbackInfo
 }

packages/types/src/sdk/search.ts

@@ -19,6 +19,9 @@ export enum SearchFilterOperator {
 
 export interface SearchFilters {
   allOr?: boolean
+  // TODO: this is just around for now - we need a better way to do or/and
+  // allows just fuzzy to be or - all the fuzzy/like parameters
+  fuzzyOr?: boolean
   onEmptyFilter?: EmptyFilterOption
   [SearchFilterOperator.STRING]?: {
     [key: string]: string
@@ -61,6 +64,11 @@ export interface SearchFilters {
   }
 }
 
+export type SearchFilterKey = keyof Omit<
+  SearchFilters,
+  "allOr" | "onEmptyFilter" | "fuzzyOr"
+>
+
 export type SearchQueryFields = Omit<SearchFilters, "allOr" | "onEmptyFilter">
 
 export interface SortJson {
@@ -117,6 +125,11 @@ export interface QueryJson {
   tableAliases?: Record<string, string>
 }
 
+export interface QueryOptions {
+  disableReturning?: boolean
+  disableBindings?: boolean
+}
+
 export type SqlQueryBinding = Knex.Value[]
 
 export interface SqlQuery {
@@ -128,3 +141,11 @@ export enum EmptyFilterOption {
   RETURN_ALL = "all",
   RETURN_NONE = "none",
 }
+
+export enum SqlClient {
+  MS_SQL = "mssql",
+  POSTGRES = "pg",
+  MY_SQL = "mysql2",
+  ORACLE = "oracledb",
+  SQL_LITE = "sqlite3",
+}

packages/worker/package.json

@@ -70,7 +70,8 @@
     "pouchdb-all-dbs": "1.1.1",
     "server-destroy": "1.0.1",
     "undici": "^6.0.1",
-    "undici-types": "^6.0.1"
+    "undici-types": "^6.0.1",
+    "knex": "2.4.2"
   },
   "devDependencies": {
     "@swc/core": "1.3.71",

packages/worker/src/api/index.ts

@@ -4,8 +4,13 @@ const compress = require("koa-compress")
 
 import zlib from "zlib"
 import { routes } from "./routes"
-import { middleware as pro } from "@budibase/pro"
+import { middleware as pro, sdk } from "@budibase/pro"
 import { auth, middleware } from "@budibase/backend-core"
+import env from "../environment"
+
+if (env.SQS_SEARCH_ENABLE) {
+  sdk.auditLogs.useSQLSearch()
+}
 
 const PUBLIC_ENDPOINTS = [
   // deprecated single tenant sso callback

packages/worker/src/api/routes/global/tests/auditLogs.spec.ts

@@ -1,6 +1,7 @@
 import { mocks, structures } from "@budibase/backend-core/tests"
 import { context, events } from "@budibase/backend-core"
 import { Event, IdentityType } from "@budibase/types"
+import { auditLogs } from "@budibase/pro"
 import { TestConfiguration } from "../../../../tests"
 
 mocks.licenses.useAuditLogs()
@@ -12,10 +13,13 @@ const BASE_IDENTITY = {
 const USER_AUDIT_LOG_COUNT = 3
 const APP_ID = "app_1"
 
-describe("/api/global/auditlogs", () => {
+describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
   const config = new TestConfiguration()
 
   beforeAll(async () => {
+    if (method === "sql") {
+      auditLogs.useSQLSearch()
+    }
     await config.beforeAll()
   })
 

packages/worker/src/environment.ts

@@ -45,6 +45,7 @@ const environment = {
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
+  SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
   // smtp
   SMTP_USER: process.env.SMTP_USER,
   SMTP_PASSWORD: process.env.SMTP_PASSWORD,

scripts/build.js

@@ -3,11 +3,11 @@
 const start = Date.now()
 
 const fs = require("fs")
-const { cp, readdir, copyFile, mkdir } = require('node:fs/promises');
+const { cp, readdir, copyFile, mkdir } = require("node:fs/promises")
 const path = require("path")
 
 const { build } = require("esbuild")
-const { compile } = require('svelte/compiler')
+const { compile } = require("svelte/compiler")
 
 const {
   default: TsconfigPathsPlugin,
@@ -15,13 +15,13 @@ const {
 const { nodeExternalsPlugin } = require("esbuild-node-externals")
 
 const svelteCompilePlugin = {
-  name: 'svelteCompile',
+  name: "svelteCompile",
   setup(build) {
     // Compiles `.svelte` files into JS classes so that they can be directly imported into our
     // Typescript packages
-    build.onLoad({ filter: /\.svelte$/ }, async (args) => {
-      const source = await fs.promises.readFile(args.path, 'utf8')
-      const dir = path.dirname(args.path);
+    build.onLoad({ filter: /\.svelte$/ }, async args => {
+      const source = await fs.promises.readFile(args.path, "utf8")
+      const dir = path.dirname(args.path)
 
       try {
         const { js } = compile(source, { css: "injected", generate: "ssr" })
@@ -31,15 +31,15 @@ const svelteCompilePlugin = {
           contents: js.code,
           // The loader this is passed to, basically how the above provided content is "treated",
           // the contents provided above will be transpiled and bundled like any other JS file.
-          loader: 'js',
+          loader: "js",
           // Where to resolve any imports present in the loaded file
-          resolveDir: dir
+          resolveDir: dir,
         }
       } catch (e) {
         return { errors: [JSON.stringify(e)] }
       }
     })
-  }
+  },
 }
 
 var { argv } = require("yargs")
@@ -75,7 +75,7 @@ async function runBuild(entry, outfile) {
       svelteCompilePlugin,
       TsconfigPathsPlugin({ tsconfig: tsconfigPathPluginContent }),
       nodeExternalsPlugin({
-        allowList: ["@budibase/frontend-core", "svelte"]
+        allowList: ["@budibase/frontend-core", "svelte"],
       }),
     ],
     preserveSymlinks: true,
@@ -90,25 +90,39 @@ async function runBuild(entry, outfile) {
       "bcryptjs",
       "graphql/*",
       "bson",
+      "better-sqlite3",
+      "sqlite3",
+      "mysql",
+      "mysql2",
+      "oracle",
+      "pg",
+      "pg-query-stream",
+      "pg-native",
     ],
   }
 
-  await mkdir('dist', { recursive: true });
+  await mkdir("dist", { recursive: true })
 
   const hbsFiles = (async () => {
-    const dir = await readdir('./', { recursive: true });
-    const files = dir.filter(entry => entry.endsWith('.hbs') || entry.endsWith('ivm.bundle.js'));
-    const fileCopyPromises = files.map(file => copyFile(file, `dist/${path.basename(file)}`))
+    const dir = await readdir("./", { recursive: true })
+    const files = dir.filter(
+      entry => entry.endsWith(".hbs") || entry.endsWith("ivm.bundle.js")
+    )
+    const fileCopyPromises = files.map(file =>
+      copyFile(file, `dist/${path.basename(file)}`)
+    )
 
     await Promise.all(fileCopyPromises)
   })()
 
   const oldClientVersions = (async () => {
     try {
-      await cp('./build/oldClientVersions', './dist/oldClientVersions', { recursive: true });
+      await cp("./build/oldClientVersions", "./dist/oldClientVersions", {
+        recursive: true,
+      })
     } catch (e) {
       if (e.code !== "EEXIST" && e.code !== "ENOENT") {
-        throw e;
+        throw e
       }
     }
   })()

scripts/devDocker.sh

@@ -7,6 +7,5 @@ if [ ! -d "./packages/pro/src" ]; then
 fi
 
 yarn build:apps
-docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0
-
+docker compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0
 

yarn.lock

@@ -2050,6 +2050,7 @@
     ioredis "5.3.2"
     joi "17.6.0"
     jsonwebtoken "9.0.2"
+    knex "2.4.2"
     koa-passport "^6.0.0"
     koa-pino-logger "4.0.0"
     lodash "4.17.21"