From 0efa1f06ab3ab8216fb05888c0e534f270f2b37c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 May 2024 17:33:47 +0100 Subject: [PATCH] Moving some stuff around to make way for other services using the sql layers. --- hosting/docker-compose.dev.yaml | 3 +- packages/backend-core/package.json | 3 +- packages/backend-core/src/constants/db.ts | 5 + packages/backend-core/src/environment.ts | 3 + packages/backend-core/src/index.ts | 1 + packages/backend-core/src/sql/index.ts | 4 + .../base => backend-core/src/sql}/sql.ts | 16 +- .../backend-core/src/sql/sqlStatements.ts | 79 ++++++++ .../base => backend-core/src/sql}/sqlTable.ts | 3 +- packages/backend-core/src/sql/utils.ts | 134 +++++++++++++ packages/server/src/constants/index.ts | 10 +- packages/server/src/definitions/datasource.ts | 5 - .../src/integrations/microsoftSqlServer.ts | 38 ++-- packages/server/src/integrations/mysql.ts | 6 +- packages/server/src/integrations/oracle.ts | 6 +- packages/server/src/integrations/postgres.ts | 6 +- .../server/src/integrations/utils/index.ts | 1 - .../src/integrations/utils/sqlStatements.ts | 80 -------- .../server/src/integrations/utils/utils.ts | 185 ++---------------- .../server/src/sdk/app/rows/search/sqs.ts | 7 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 2 +- packages/server/src/sdk/app/rows/utils.ts | 3 +- packages/types/src/sdk/search.ts | 13 ++ scripts/build.js | 46 +++-- 24 files changed, 334 insertions(+), 325 deletions(-) create mode 100644 packages/backend-core/src/sql/index.ts rename packages/{server/src/integrations/base => backend-core/src/sql}/sql.ts (98%) create mode 100644 packages/backend-core/src/sql/sqlStatements.ts rename packages/{server/src/integrations/base => backend-core/src/sql}/sqlTable.ts (99%) create mode 100644 packages/backend-core/src/sql/utils.ts diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 9dba5d427c..77f6bd053b 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -42,12 +42,13 @@ services: couchdb-service: container_name: budi-couchdb3-dev restart: on-failure - image: budibase/couchdb + image: budibase/couchdb:v3.2.1-sqs environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" + - "${COUCH_DB_SQS_PORT}:4984" volumes: - couchdb_data:/data diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index ff35ccee22..f61059cc97 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -54,7 +54,8 @@ "sanitize-s3-objectkey": "0.0.1", "semver": "^7.5.4", "tar-fs": "2.1.1", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "knex": "2.4.2" }, "devDependencies": { "@shopify/jest-koa-mocks": "5.1.1", diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index c11c227b66..43ed1ad96a 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -67,3 +67,8 @@ export const APP_DEV = prefixed(DocumentType.APP_DEV) export const APP_DEV_PREFIX = APP_DEV export const BUDIBASE_DATASOURCE_TYPE = "budibase" export const SQLITE_DESIGN_DOC_ID = "_design/sqlite" +export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" +export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" +export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" +export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" +export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 20ff16739f..1e7da2f9a2 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -159,6 +159,9 @@ const environment = { process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", HTTP_LOGGING: httpLogging(), ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR, + // Couch/search + SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE, + SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, // smtp SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, SMTP_USER: process.env.SMTP_USER, diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 5ce35ee760..30c5fbdd7a 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -34,6 +34,7 @@ export * as docUpdates from "./docUpdates" export * from "./utils/Duration" export * as docIds from "./docIds" export * as security from "./security" +export * as sql from "./sql" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal // circular dependencies diff --git a/packages/backend-core/src/sql/index.ts b/packages/backend-core/src/sql/index.ts new file mode 100644 index 0000000000..058b54e159 --- /dev/null +++ b/packages/backend-core/src/sql/index.ts @@ -0,0 +1,4 @@ +export * as utils from "./utils" + +export { default as Sql } from "./sql" +export { default as SqlTable } from "./sqlTable" diff --git a/packages/server/src/integrations/base/sql.ts b/packages/backend-core/src/sql/sql.ts similarity index 98% rename from packages/server/src/integrations/base/sql.ts rename to packages/backend-core/src/sql/sql.ts index 85db642e47..554343dab7 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1,13 +1,7 @@ import { Knex, knex } from "knex" -import { db as dbCore } from "@budibase/backend-core" -import { QueryOptions } from "../../definitions/datasource" -import { - isIsoDateString, - SqlClient, - isValidFilter, - getNativeSql, - SqlStatements, -} from "../utils" +import * as dbCore from "../db" +import { isIsoDateString, isValidFilter, getNativeSql } from "./utils" +import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" import { BBReferenceFieldMetadata, @@ -24,8 +18,10 @@ import { Table, TableSourceType, INTERNAL_TABLE_SOURCE_ID, + SqlClient, + QueryOptions, } from "@budibase/types" -import environment from "../../environment" +import environment from "../environment" import { helpers } from "@budibase/shared-core" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any diff --git a/packages/backend-core/src/sql/sqlStatements.ts b/packages/backend-core/src/sql/sqlStatements.ts new file mode 100644 index 0000000000..a80defd8b8 --- /dev/null +++ b/packages/backend-core/src/sql/sqlStatements.ts @@ -0,0 +1,79 @@ +import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types" +import { Knex } from "knex" + +export class SqlStatements { + client: string + table: Table + allOr: boolean | undefined + constructor( + client: string, + table: Table, + { allOr }: { allOr?: boolean } = {} + ) { + this.client = client + this.table = table + this.allOr = allOr + } + + getField(key: string): FieldSchema | undefined { + const fieldName = key.split(".")[1] + return this.table.schema[fieldName] + } + + between( + query: Knex.QueryBuilder, + key: string, + low: number | string, + high: number | string + ) { + // Use a between operator if we have 2 valid range values + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, + [low, high] + ) + } else { + const fnc = this.allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [low, high]) + } + return query + } + + lte(query: Knex.QueryBuilder, key: string, low: number | string) { + // Use just a single greater than operator if we only have a low + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ + low, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, ">=", low) + } + return query + } + + gte(query: Knex.QueryBuilder, key: string, high: number | string) { + const field = this.getField(key) + // Use just a single less than operator if we only have a high + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ + high, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, "<=", high) + } + return query + } +} diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts similarity index 99% rename from packages/server/src/integrations/base/sqlTable.ts rename to packages/backend-core/src/sql/sqlTable.ts index a82a9fcea8..493729c04c 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -9,8 +9,9 @@ import { SqlQuery, Table, TableSourceType, + SqlClient, } from "@budibase/types" -import { breakExternalTableId, getNativeSql, SqlClient } from "../utils" +import { breakExternalTableId, getNativeSql } from "./utils" import { helpers, utils } from "@budibase/shared-core" import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts new file mode 100644 index 0000000000..a8b295d879 --- /dev/null +++ b/packages/backend-core/src/sql/utils.ts @@ -0,0 +1,134 @@ +import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types" +import { DEFAULT_BB_DATASOURCE_ID } from "../constants" +import { Knex } from "knex" +import { SEPARATOR } from "../db" + +const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` +const ROW_ID_REGEX = /^\[.*]$/g +const ENCODED_SPACE = encodeURIComponent(" ") + +export function isExternalTableID(tableId: string) { + return tableId.includes(DocumentType.DATASOURCE) +} + +export function isInternalTableID(tableId: string) { + return !isExternalTableID(tableId) +} + +export function getNativeSql( + query: Knex.SchemaBuilder | Knex.QueryBuilder +): SqlQuery | SqlQuery[] { + let sql = query.toSQL() + if (Array.isArray(sql)) { + return sql as SqlQuery[] + } + let native: Knex.SqlNative | undefined + if (sql.toNative) { + native = sql.toNative() + } + return { + sql: native?.sql || sql.sql, + bindings: native?.bindings || sql.bindings, + } as SqlQuery +} + +export function isExternalTable(table: Table) { + if ( + table?.sourceId && + table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) && + table?.sourceId !== DEFAULT_BB_DATASOURCE_ID + ) { + return true + } else if (table?.sourceType === TableSourceType.EXTERNAL) { + return true + } else if (table?._id && isExternalTableID(table._id)) { + return true + } + return false +} + +export function buildExternalTableId(datasourceId: string, tableName: string) { + // encode spaces + if (tableName.includes(" ")) { + tableName = encodeURIComponent(tableName) + } + return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}` +} + +export function breakExternalTableId(tableId: string | undefined) { + if (!tableId) { + return {} + } + const parts = tableId.split(DOUBLE_SEPARATOR) + let datasourceId = parts.shift() + // if they need joined + let tableName = parts.join(DOUBLE_SEPARATOR) + // if contains encoded spaces, decode it + if (tableName.includes(ENCODED_SPACE)) { + tableName = decodeURIComponent(tableName) + } + return { datasourceId, tableName } +} + +export function generateRowIdField(keyProps: any[] = []) { + if (!Array.isArray(keyProps)) { + keyProps = [keyProps] + } + for (let index in keyProps) { + if (keyProps[index] instanceof Buffer) { + keyProps[index] = keyProps[index].toString() + } + } + // this conserves order and types + // we have to swap the double quotes to single quotes for use in HBS statements + // when using the literal helper the double quotes can break things + return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'")) +} + +export function isRowId(field: any) { + return ( + Array.isArray(field) || + (typeof field === "string" && field.match(ROW_ID_REGEX) != null) + ) +} + +export function convertRowId(field: any) { + if (Array.isArray(field)) { + return field[0] + } + if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) { + return field.substring(1, field.length - 1) + } + return field +} + +// should always return an array +export function breakRowIdField(_id: string | { _id: string }): any[] { + if (!_id) { + return [] + } + // have to replace on the way back as we swapped out the double quotes + // when encoding, but JSON can't handle the single quotes + const id = typeof _id === "string" ? _id : _id._id + const decoded: string = decodeURIComponent(id).replace(/'/g, '"') + try { + const parsed = JSON.parse(decoded) + return Array.isArray(parsed) ? parsed : [parsed] + } catch (err) { + // wasn't json - likely was handlebars for a many to many + return [_id] + } +} + +export function isIsoDateString(str: string) { + const trimmedValue = str.trim() + if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) { + return false + } + let d = new Date(trimmedValue) + return d.toISOString() === trimmedValue +} + +export function isValidFilter(value: any) { + return value != null && value !== "" +} diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 37c275c8a3..bc255ecb2a 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -173,8 +173,8 @@ export const devClientVersion = "0.0.0" export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets export const MAX_AUTOMATION_RECURRING_ERRORS = 5 export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber" -export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" -export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" -export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" -export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" -export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" +export const DEFAULT_JOBS_TABLE_ID = constants.DEFAULT_JOBS_TABLE_ID +export const DEFAULT_INVENTORY_TABLE_ID = constants.DEFAULT_INVENTORY_TABLE_ID +export const DEFAULT_EXPENSES_TABLE_ID = constants.DEFAULT_EXPENSES_TABLE_ID +export const DEFAULT_EMPLOYEE_TABLE_ID = constants.DEFAULT_EMPLOYEE_TABLE_ID +export const DEFAULT_BB_DATASOURCE_ID = constants.DEFAULT_BB_DATASOURCE_ID diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index b90fa5db0c..acaec4c85d 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -3,8 +3,3 @@ * internal to the server and don't need to * * be exposed for use by other services. * ********************************************/ - -export interface QueryOptions { - disableReturning?: boolean - disableBindings?: boolean -} diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 5626d7eda3..af535891cf 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -1,40 +1,41 @@ import { + ConnectionInfo, + DatasourceFeature, DatasourceFieldType, + DatasourcePlus, + DatasourcePlusQueryResponse, Integration, Operation, - Table, - TableSchema, QueryJson, QueryType, - SqlQuery, - DatasourcePlus, - DatasourceFeature, - ConnectionInfo, - SourceName, Schema, + SourceName, + SqlClient, + SqlQuery, + Table, + TableSchema, TableSourceType, - DatasourcePlusQueryResponse, } from "@budibase/types" import { - getSqlQuery, buildExternalTableId, - generateColumnDefinition, - finaliseExternalTables, - SqlClient, checkExternalTables, + finaliseExternalTables, + generateColumnDefinition, + getSqlQuery, HOST_ADDRESS, } from "./utils" -import Sql from "./base/sql" -import { MSSQLTablesResponse, MSSQLColumn } from "./base/types" +import { MSSQLColumn, MSSQLTablesResponse } from "./base/types" import { getReadableErrorMessage } from "./base/errorMapping" import sqlServer from "mssql" - -const DEFAULT_SCHEMA = "dbo" - +import { sql } from "@budibase/backend-core" import { ConfidentialClientApplication } from "@azure/msal-node" import { utils } from "@budibase/shared-core" +const Sql = sql.Sql + +const DEFAULT_SCHEMA = "dbo" + enum MSSQLConfigAuthType { AZURE_ACTIVE_DIRECTORY = "Azure Active Directory", NTLM = "NTLM", @@ -590,8 +591,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { scriptParts.push(createTableStatement) } - const schema = scriptParts.join("\n") - return schema + return scriptParts.join("\n") } } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 19a63a44ad..ecb3c07fa4 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -14,10 +14,10 @@ import { TableSourceType, DatasourcePlusQueryResponse, SqlQueryBinding, + SqlClient, } from "@budibase/types" import { getSqlQuery, - SqlClient, buildExternalTableId, generateColumnDefinition, finaliseExternalTables, @@ -26,11 +26,13 @@ import { } from "./utils" import dayjs from "dayjs" import { NUMBER_REGEX } from "../utilities" -import Sql from "./base/sql" import { MySQLColumn } from "./base/types" import { getReadableErrorMessage } from "./base/errorMapping" +import { sql } from "@budibase/backend-core" import mysql from "mysql2/promise" +const Sql = sql.Sql + interface MySQLConfig extends mysql.ConnectionOptions { database: string rejectUnauthorized: boolean diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 8105edfef8..55e62260b8 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -14,6 +14,7 @@ import { TableSourceType, Row, DatasourcePlusQueryResponse, + SqlClient, } from "@budibase/types" import { buildExternalTableId, @@ -21,10 +22,8 @@ import { generateColumnDefinition, finaliseExternalTables, getSqlQuery, - SqlClient, HOST_ADDRESS, } from "./utils" -import Sql from "./base/sql" import { BindParameters, Connection, @@ -33,6 +32,9 @@ import { Result, } from "oracledb" import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types" +import { sql } from "@budibase/backend-core" + +const Sql = sql.Sql let oracledb: any try { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index e810986757..3711db6950 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -13,17 +13,16 @@ import { Schema, TableSourceType, DatasourcePlusQueryResponse, + SqlClient, } from "@budibase/types" import { getSqlQuery, buildExternalTableId, generateColumnDefinition, finaliseExternalTables, - SqlClient, checkExternalTables, HOST_ADDRESS, } from "./utils" -import Sql from "./base/sql" import { PostgresColumn } from "./base/types" import { escapeDangerousCharacters } from "../utilities" @@ -31,7 +30,7 @@ import { Client, ClientConfig, types } from "pg" import { getReadableErrorMessage } from "./base/errorMapping" import { exec } from "child_process" import { storeTempFile } from "../utilities/fileSystem" -import { env } from "@budibase/backend-core" +import { env, sql } from "@budibase/backend-core" // Return "date" and "timestamp" types as plain strings. // This lets us reference the original stored timezone. @@ -43,6 +42,7 @@ if (types) { } const JSON_REGEX = /'{.*}'::json/s +const Sql = sql.Sql interface PostgresConfig { host: string diff --git a/packages/server/src/integrations/utils/index.ts b/packages/server/src/integrations/utils/index.ts index a9c2019ba2..3eeaeaa90c 100644 --- a/packages/server/src/integrations/utils/index.ts +++ b/packages/server/src/integrations/utils/index.ts @@ -1,2 +1 @@ export * from "./utils" -export { SqlStatements } from "./sqlStatements" diff --git a/packages/server/src/integrations/utils/sqlStatements.ts b/packages/server/src/integrations/utils/sqlStatements.ts index 7a5482830b..e69de29bb2 100644 --- a/packages/server/src/integrations/utils/sqlStatements.ts +++ b/packages/server/src/integrations/utils/sqlStatements.ts @@ -1,80 +0,0 @@ -import { FieldType, Table, FieldSchema } from "@budibase/types" -import { SqlClient } from "./utils" -import { Knex } from "knex" - -export class SqlStatements { - client: string - table: Table - allOr: boolean | undefined - constructor( - client: string, - table: Table, - { allOr }: { allOr?: boolean } = {} - ) { - this.client = client - this.table = table - this.allOr = allOr - } - - getField(key: string): FieldSchema | undefined { - const fieldName = key.split(".")[1] - return this.table.schema[fieldName] - } - - between( - query: Knex.QueryBuilder, - key: string, - low: number | string, - high: number | string - ) { - // Use a between operator if we have 2 valid range values - const field = this.getField(key) - if ( - field?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, - [low, high] - ) - } else { - const fnc = this.allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [low, high]) - } - return query - } - - lte(query: Knex.QueryBuilder, key: string, low: number | string) { - // Use just a single greater than operator if we only have a low - const field = this.getField(key) - if ( - field?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ - low, - ]) - } else { - const fnc = this.allOr ? "orWhere" : "where" - query = query[fnc](key, ">=", low) - } - return query - } - - gte(query: Knex.QueryBuilder, key: string, high: number | string) { - const field = this.getField(key) - // Use just a single less than operator if we only have a high - if ( - field?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ - high, - ]) - } else { - const fnc = this.allOr ? "orWhere" : "where" - query = query[fnc](key, "<=", high) - } - return query - } -} diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 892d8ae034..9d95ee9eb5 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -3,23 +3,16 @@ import { Table, Datasource, FieldType, - TableSourceType, FieldSchema, } from "@budibase/types" -import { context, objectStore } from "@budibase/backend-core" +import { context, objectStore, sql } from "@budibase/backend-core" import { v4 } from "uuid" import { parseStringPromise as xmlParser } from "xml2js" import { formatBytes } from "../../utilities" import bl from "bl" import env from "../../environment" -import { DocumentType, SEPARATOR } from "../../db/utils" -import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants" +import { InvalidColumns } from "../../constants" import { helpers, utils } from "@budibase/shared-core" -import { Knex } from "knex" - -const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` -const ROW_ID_REGEX = /^\[.*]$/g -const ENCODED_SPACE = encodeURIComponent(" ") type PrimitiveTypes = | FieldType.STRING @@ -109,13 +102,15 @@ const SQL_TYPE_MAP: Record = { ...SQL_OPTIONS_TYPE_MAP, } -export enum SqlClient { - MS_SQL = "mssql", - POSTGRES = "pg", - MY_SQL = "mysql2", - ORACLE = "oracledb", - SQL_LITE = "sqlite3", -} +export const isExternalTableID = sql.utils.isExternalTableID +export const isExternalTable = sql.utils.isExternalTable +export const buildExternalTableId = sql.utils.buildExternalTableId +export const breakExternalTableId = sql.utils.breakExternalTableId +export const generateRowIdField = sql.utils.generateRowIdField +export const isRowId = sql.utils.isRowId +export const convertRowId = sql.utils.convertRowId +export const breakRowIdField = sql.utils.breakRowIdField +export const isValidFilter = sql.utils.isValidFilter const isCloud = env.isProd() && !env.SELF_HOSTED const isSelfHost = env.isProd() && env.SELF_HOSTED @@ -125,119 +120,6 @@ export const HOST_ADDRESS = isSelfHost ? "" : "localhost" -export function isExternalTableID(tableId: string) { - return tableId.includes(DocumentType.DATASOURCE) -} - -export function isInternalTableID(tableId: string) { - return !isExternalTableID(tableId) -} - -export function getNativeSql( - query: Knex.SchemaBuilder | Knex.QueryBuilder -): SqlQuery | SqlQuery[] { - let sql = query.toSQL() - if (Array.isArray(sql)) { - return sql as SqlQuery[] - } - let native: Knex.SqlNative | undefined - if (sql.toNative) { - native = sql.toNative() - } - return { - sql: native?.sql || sql.sql, - bindings: native?.bindings || sql.bindings, - } as SqlQuery -} - -export function isExternalTable(table: Table) { - if ( - table?.sourceId && - table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) && - table?.sourceId !== DEFAULT_BB_DATASOURCE_ID - ) { - return true - } else if (table?.sourceType === TableSourceType.EXTERNAL) { - return true - } else if (table?._id && isExternalTableID(table._id)) { - return true - } - return false -} - -export function buildExternalTableId(datasourceId: string, tableName: string) { - // encode spaces - if (tableName.includes(" ")) { - tableName = encodeURIComponent(tableName) - } - return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}` -} - -export function breakExternalTableId(tableId: string | undefined) { - if (!tableId) { - return {} - } - const parts = tableId.split(DOUBLE_SEPARATOR) - let datasourceId = parts.shift() - // if they need joined - let tableName = parts.join(DOUBLE_SEPARATOR) - // if contains encoded spaces, decode it - if (tableName.includes(ENCODED_SPACE)) { - tableName = decodeURIComponent(tableName) - } - return { datasourceId, tableName } -} - -export function generateRowIdField(keyProps: any[] = []) { - if (!Array.isArray(keyProps)) { - keyProps = [keyProps] - } - for (let index in keyProps) { - if (keyProps[index] instanceof Buffer) { - keyProps[index] = keyProps[index].toString() - } - } - // this conserves order and types - // we have to swap the double quotes to single quotes for use in HBS statements - // when using the literal helper the double quotes can break things - return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'")) -} - -export function isRowId(field: any) { - return ( - Array.isArray(field) || - (typeof field === "string" && field.match(ROW_ID_REGEX) != null) - ) -} - -export function convertRowId(field: any) { - if (Array.isArray(field)) { - return field[0] - } - if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) { - return field.substring(1, field.length - 1) - } - return field -} - -// should always return an array -export function breakRowIdField(_id: string | { _id: string }): any[] { - if (!_id) { - return [] - } - // have to replace on the way back as we swapped out the double quotes - // when encoding, but JSON can't handle the single quotes - const id = typeof _id === "string" ? _id : _id._id - const decoded: string = decodeURIComponent(id).replace(/'/g, '"') - try { - const parsed = JSON.parse(decoded) - return Array.isArray(parsed) ? parsed : [parsed] - } catch (err) { - // wasn't json - likely was handlebars for a many to many - return [_id] - } -} - export function generateColumnDefinition(config: { externalType: string autocolumn: boolean @@ -297,15 +179,6 @@ export function isSQL(datasource: Datasource) { return helpers.isSQL(datasource) } -export function isIsoDateString(str: string) { - const trimmedValue = str.trim() - if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) { - return false - } - let d = new Date(trimmedValue) - return d.toISOString() === trimmedValue -} - /** * Looks for columns which need to be copied over into the new table definitions, like relationships, * options types and views. @@ -451,34 +324,6 @@ export function checkExternalTables( return errors } -/** - * Checks if the provided input is an object, but specifically not a date type object. - * Used during coercion of types and relationship handling, dates are considered valid - * and can be used as a display field, but objects and arrays cannot. - * @param testValue an unknown type which this function will attempt to extract - * a valid primary display string from. - */ -export function getPrimaryDisplay(testValue: unknown): string | undefined { - if (testValue instanceof Date) { - return testValue.toISOString() - } - if ( - Array.isArray(testValue) && - testValue[0] && - typeof testValue[0] !== "object" - ) { - return testValue.join(", ") - } - if (typeof testValue === "object") { - return undefined - } - return testValue as string -} - -export function isValidFilter(value: any) { - return value != null && value !== "" -} - export async function handleXml(response: any) { let data, rawXml = await response.text() @@ -517,12 +362,6 @@ export async function handleFileResponse( const contentLength = response.headers.get("content-length") if (contentLength) { size = parseInt(contentLength, 10) - } else { - const chunks: Buffer[] = [] - for await (const chunk of response.body) { - chunks.push(chunk) - size += chunk.length - } } await objectStore.streamUpload({ @@ -533,7 +372,7 @@ export async function handleFileResponse( type: response.headers["content-type"], }) } - presignedUrl = await objectStore.getPresignedUrl(bucket, key) + presignedUrl = objectStore.getPresignedUrl(bucket, key) return { data: { size, diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index a94ce265c5..02a93537b9 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -11,15 +11,14 @@ import { SortOrder, SortType, Table, + SqlClient, } from "@budibase/types" -import SqlQueryBuilder from "../../../../integrations/base/sql" -import { SqlClient } from "../../../../integrations/utils" import { buildInternalRelationships, sqlOutputProcessing, } from "../../../../api/controllers/row/utils" import sdk from "../../../index" -import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" +import { context, sql, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" import { CONSTANT_INTERNAL_ROW_COLS, SQS_DATASOURCE_INTERNAL, @@ -104,7 +103,7 @@ export async function search( ): Promise> { const { paginate, query, ...params } = options - const builder = new SqlQueryBuilder(SqlClient.SQL_LITE) + const builder = new sql.Sql(SqlClient.SQL_LITE) const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) if (!table) { diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 0fc338ecbe..4c5c88155b 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -5,12 +5,12 @@ import { QueryJson, Row, SearchFilters, + SqlClient, } from "@budibase/types" import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" import datasources from "../datasources" import { makeExternalQuery } from "../../../integrations/base/query" -import { SqlClient } from "../../../integrations/utils" import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils" const WRITE_OPERATIONS: Operation[] = [ diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index a9df4f89cd..62039737ba 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -9,12 +9,13 @@ import { SourceName, Table, TableSchema, + SqlClient, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { isRelationshipColumn } from "../../../db/utils" -import { SqlClient, isSQL } from "../../../integrations/utils" +import { isSQL } from "../../../integrations/utils" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 40f411f02a..6314da9e9b 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -117,6 +117,11 @@ export interface QueryJson { tableAliases?: Record } +export interface QueryOptions { + disableReturning?: boolean + disableBindings?: boolean +} + export type SqlQueryBinding = Knex.Value[] export interface SqlQuery { @@ -128,3 +133,11 @@ export enum EmptyFilterOption { RETURN_ALL = "all", RETURN_NONE = "none", } + +export enum SqlClient { + MS_SQL = "mssql", + POSTGRES = "pg", + MY_SQL = "mysql2", + ORACLE = "oracledb", + SQL_LITE = "sqlite3", +} diff --git a/scripts/build.js b/scripts/build.js index 7fbd242d4d..7c400e59e9 100755 --- a/scripts/build.js +++ b/scripts/build.js @@ -3,11 +3,11 @@ const start = Date.now() const fs = require("fs") -const { cp, readdir, copyFile, mkdir } = require('node:fs/promises'); +const { cp, readdir, copyFile, mkdir } = require("node:fs/promises") const path = require("path") const { build } = require("esbuild") -const { compile } = require('svelte/compiler') +const { compile } = require("svelte/compiler") const { default: TsconfigPathsPlugin, @@ -15,13 +15,13 @@ const { const { nodeExternalsPlugin } = require("esbuild-node-externals") const svelteCompilePlugin = { - name: 'svelteCompile', + name: "svelteCompile", setup(build) { // Compiles `.svelte` files into JS classes so that they can be directly imported into our // Typescript packages - build.onLoad({ filter: /\.svelte$/ }, async (args) => { - const source = await fs.promises.readFile(args.path, 'utf8') - const dir = path.dirname(args.path); + build.onLoad({ filter: /\.svelte$/ }, async args => { + const source = await fs.promises.readFile(args.path, "utf8") + const dir = path.dirname(args.path) try { const { js } = compile(source, { css: "injected", generate: "ssr" }) @@ -31,15 +31,15 @@ const svelteCompilePlugin = { contents: js.code, // The loader this is passed to, basically how the above provided content is "treated", // the contents provided above will be transpiled and bundled like any other JS file. - loader: 'js', + loader: "js", // Where to resolve any imports present in the loaded file - resolveDir: dir + resolveDir: dir, } } catch (e) { return { errors: [JSON.stringify(e)] } } }) - } + }, } var { argv } = require("yargs") @@ -75,7 +75,7 @@ async function runBuild(entry, outfile) { svelteCompilePlugin, TsconfigPathsPlugin({ tsconfig: tsconfigPathPluginContent }), nodeExternalsPlugin({ - allowList: ["@budibase/frontend-core", "svelte"] + allowList: ["@budibase/frontend-core", "svelte"], }), ], preserveSymlinks: true, @@ -90,25 +90,39 @@ async function runBuild(entry, outfile) { "bcryptjs", "graphql/*", "bson", + "better-sqlite3", + "sqlite3", + "mysql", + "mysql2", + "oracle", + "pg", + "pg-query-stream", + "pg-native", ], } - await mkdir('dist', { recursive: true }); + await mkdir("dist", { recursive: true }) const hbsFiles = (async () => { - const dir = await readdir('./', { recursive: true }); - const files = dir.filter(entry => entry.endsWith('.hbs') || entry.endsWith('ivm.bundle.js')); - const fileCopyPromises = files.map(file => copyFile(file, `dist/${path.basename(file)}`)) + const dir = await readdir("./", { recursive: true }) + const files = dir.filter( + entry => entry.endsWith(".hbs") || entry.endsWith("ivm.bundle.js") + ) + const fileCopyPromises = files.map(file => + copyFile(file, `dist/${path.basename(file)}`) + ) await Promise.all(fileCopyPromises) })() const oldClientVersions = (async () => { try { - await cp('./build/oldClientVersions', './dist/oldClientVersions', { recursive: true }); + await cp("./build/oldClientVersions", "./dist/oldClientVersions", { + recursive: true, + }) } catch (e) { if (e.code !== "EEXIST" && e.code !== "ENOENT") { - throw e; + throw e } } })()