From 0efa1f06ab3ab8216fb05888c0e534f270f2b37c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 May 2024 17:33:47 +0100 Subject: [PATCH 01/33] Moving some stuff around to make way for other services using the sql layers. --- hosting/docker-compose.dev.yaml | 3 +- packages/backend-core/package.json | 3 +- packages/backend-core/src/constants/db.ts | 5 + packages/backend-core/src/environment.ts | 3 + packages/backend-core/src/index.ts | 1 + packages/backend-core/src/sql/index.ts | 4 + .../base => backend-core/src/sql}/sql.ts | 16 +- .../backend-core/src/sql/sqlStatements.ts | 79 ++++++++ .../base => backend-core/src/sql}/sqlTable.ts | 3 +- packages/backend-core/src/sql/utils.ts | 134 +++++++++++++ packages/server/src/constants/index.ts | 10 +- packages/server/src/definitions/datasource.ts | 5 - .../src/integrations/microsoftSqlServer.ts | 38 ++-- packages/server/src/integrations/mysql.ts | 6 +- packages/server/src/integrations/oracle.ts | 6 +- packages/server/src/integrations/postgres.ts | 6 +- .../server/src/integrations/utils/index.ts | 1 - .../src/integrations/utils/sqlStatements.ts | 80 -------- .../server/src/integrations/utils/utils.ts | 185 ++---------------- .../server/src/sdk/app/rows/search/sqs.ts | 7 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 2 +- packages/server/src/sdk/app/rows/utils.ts | 3 +- packages/types/src/sdk/search.ts | 13 ++ scripts/build.js | 46 +++-- 24 files changed, 334 insertions(+), 325 deletions(-) create mode 100644 packages/backend-core/src/sql/index.ts rename packages/{server/src/integrations/base => backend-core/src/sql}/sql.ts (98%) create mode 100644 packages/backend-core/src/sql/sqlStatements.ts rename packages/{server/src/integrations/base => backend-core/src/sql}/sqlTable.ts (99%) create mode 100644 packages/backend-core/src/sql/utils.ts diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 9dba5d427c..77f6bd053b 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -42,12 +42,13 @@ services: couchdb-service: container_name: budi-couchdb3-dev restart: on-failure - image: budibase/couchdb + image: budibase/couchdb:v3.2.1-sqs environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" + - "${COUCH_DB_SQS_PORT}:4984" volumes: - couchdb_data:/data diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index ff35ccee22..f61059cc97 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -54,7 +54,8 @@ "sanitize-s3-objectkey": "0.0.1", "semver": "^7.5.4", "tar-fs": "2.1.1", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "knex": "2.4.2" }, "devDependencies": { "@shopify/jest-koa-mocks": "5.1.1", diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index c11c227b66..43ed1ad96a 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -67,3 +67,8 @@ export const APP_DEV = prefixed(DocumentType.APP_DEV) export const APP_DEV_PREFIX = APP_DEV export const BUDIBASE_DATASOURCE_TYPE = "budibase" export const SQLITE_DESIGN_DOC_ID = "_design/sqlite" +export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" +export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" +export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" +export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" +export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 20ff16739f..1e7da2f9a2 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -159,6 +159,9 @@ const environment = { process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", HTTP_LOGGING: httpLogging(), ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR, + // Couch/search + SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE, + SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, // smtp SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, SMTP_USER: process.env.SMTP_USER, diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 5ce35ee760..30c5fbdd7a 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -34,6 +34,7 @@ export * as docUpdates from "./docUpdates" export * from "./utils/Duration" export * as docIds from "./docIds" export * as security from "./security" +export * as sql from "./sql" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal // circular dependencies diff --git a/packages/backend-core/src/sql/index.ts b/packages/backend-core/src/sql/index.ts new file mode 100644 index 0000000000..058b54e159 --- /dev/null +++ b/packages/backend-core/src/sql/index.ts @@ -0,0 +1,4 @@ +export * as utils from "./utils" + +export { default as Sql } from "./sql" +export { default as SqlTable } from "./sqlTable" diff --git a/packages/server/src/integrations/base/sql.ts b/packages/backend-core/src/sql/sql.ts similarity index 98% rename from packages/server/src/integrations/base/sql.ts rename to packages/backend-core/src/sql/sql.ts index 85db642e47..554343dab7 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1,13 +1,7 @@ import { Knex, knex } from "knex" -import { db as dbCore } from "@budibase/backend-core" -import { QueryOptions } from "../../definitions/datasource" -import { - isIsoDateString, - SqlClient, - isValidFilter, - getNativeSql, - SqlStatements, -} from "../utils" +import * as dbCore from "../db" +import { isIsoDateString, isValidFilter, getNativeSql } from "./utils" +import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" import { BBReferenceFieldMetadata, @@ -24,8 +18,10 @@ import { Table, TableSourceType, INTERNAL_TABLE_SOURCE_ID, + SqlClient, + QueryOptions, } from "@budibase/types" -import environment from "../../environment" +import environment from "../environment" import { helpers } from "@budibase/shared-core" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any diff --git a/packages/backend-core/src/sql/sqlStatements.ts b/packages/backend-core/src/sql/sqlStatements.ts new file mode 100644 index 0000000000..a80defd8b8 --- /dev/null +++ b/packages/backend-core/src/sql/sqlStatements.ts @@ -0,0 +1,79 @@ +import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types" +import { Knex } from "knex" + +export class SqlStatements { + client: string + table: Table + allOr: boolean | undefined + constructor( + client: string, + table: Table, + { allOr }: { allOr?: boolean } = {} + ) { + this.client = client + this.table = table + this.allOr = allOr + } + + getField(key: string): FieldSchema | undefined { + const fieldName = key.split(".")[1] + return this.table.schema[fieldName] + } + + between( + query: Knex.QueryBuilder, + key: string, + low: number | string, + high: number | string + ) { + // Use a between operator if we have 2 valid range values + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, + [low, high] + ) + } else { + const fnc = this.allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [low, high]) + } + return query + } + + lte(query: Knex.QueryBuilder, key: string, low: number | string) { + // Use just a single greater than operator if we only have a low + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ + low, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, ">=", low) + } + return query + } + + gte(query: Knex.QueryBuilder, key: string, high: number | string) { + const field = this.getField(key) + // Use just a single less than operator if we only have a high + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ + high, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, "<=", high) + } + return query + } +} diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts similarity index 99% rename from packages/server/src/integrations/base/sqlTable.ts rename to packages/backend-core/src/sql/sqlTable.ts index a82a9fcea8..493729c04c 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -9,8 +9,9 @@ import { SqlQuery, Table, TableSourceType, + SqlClient, } from "@budibase/types" -import { breakExternalTableId, getNativeSql, SqlClient } from "../utils" +import { breakExternalTableId, getNativeSql } from "./utils" import { helpers, utils } from "@budibase/shared-core" import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts new file mode 100644 index 0000000000..a8b295d879 --- /dev/null +++ b/packages/backend-core/src/sql/utils.ts @@ -0,0 +1,134 @@ +import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types" +import { DEFAULT_BB_DATASOURCE_ID } from "../constants" +import { Knex } from "knex" +import { SEPARATOR } from "../db" + +const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` +const ROW_ID_REGEX = /^\[.*]$/g +const ENCODED_SPACE = encodeURIComponent(" ") + +export function isExternalTableID(tableId: string) { + return tableId.includes(DocumentType.DATASOURCE) +} + +export function isInternalTableID(tableId: string) { + return !isExternalTableID(tableId) +} + +export function getNativeSql( + query: Knex.SchemaBuilder | Knex.QueryBuilder +): SqlQuery | SqlQuery[] { + let sql = query.toSQL() + if (Array.isArray(sql)) { + return sql as SqlQuery[] + } + let native: Knex.SqlNative | undefined + if (sql.toNative) { + native = sql.toNative() + } + return { + sql: native?.sql || sql.sql, + bindings: native?.bindings || sql.bindings, + } as SqlQuery +} + +export function isExternalTable(table: Table) { + if ( + table?.sourceId && + table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) && + table?.sourceId !== DEFAULT_BB_DATASOURCE_ID + ) { + return true + } else if (table?.sourceType === TableSourceType.EXTERNAL) { + return true + } else if (table?._id && isExternalTableID(table._id)) { + return true + } + return false +} + +export function buildExternalTableId(datasourceId: string, tableName: string) { + // encode spaces + if (tableName.includes(" ")) { + tableName = encodeURIComponent(tableName) + } + return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}` +} + +export function breakExternalTableId(tableId: string | undefined) { + if (!tableId) { + return {} + } + const parts = tableId.split(DOUBLE_SEPARATOR) + let datasourceId = parts.shift() + // if they need joined + let tableName = parts.join(DOUBLE_SEPARATOR) + // if contains encoded spaces, decode it + if (tableName.includes(ENCODED_SPACE)) { + tableName = decodeURIComponent(tableName) + } + return { datasourceId, tableName } +} + +export function generateRowIdField(keyProps: any[] = []) { + if (!Array.isArray(keyProps)) { + keyProps = [keyProps] + } + for (let index in keyProps) { + if (keyProps[index] instanceof Buffer) { + keyProps[index] = keyProps[index].toString() + } + } + // this conserves order and types + // we have to swap the double quotes to single quotes for use in HBS statements + // when using the literal helper the double quotes can break things + return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'")) +} + +export function isRowId(field: any) { + return ( + Array.isArray(field) || + (typeof field === "string" && field.match(ROW_ID_REGEX) != null) + ) +} + +export function convertRowId(field: any) { + if (Array.isArray(field)) { + return field[0] + } + if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) { + return field.substring(1, field.length - 1) + } + return field +} + +// should always return an array +export function breakRowIdField(_id: string | { _id: string }): any[] { + if (!_id) { + return [] + } + // have to replace on the way back as we swapped out the double quotes + // when encoding, but JSON can't handle the single quotes + const id = typeof _id === "string" ? _id : _id._id + const decoded: string = decodeURIComponent(id).replace(/'/g, '"') + try { + const parsed = JSON.parse(decoded) + return Array.isArray(parsed) ? parsed : [parsed] + } catch (err) { + // wasn't json - likely was handlebars for a many to many + return [_id] + } +} + +export function isIsoDateString(str: string) { + const trimmedValue = str.trim() + if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) { + return false + } + let d = new Date(trimmedValue) + return d.toISOString() === trimmedValue +} + +export function isValidFilter(value: any) { + return value != null && value !== "" +} diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 37c275c8a3..bc255ecb2a 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -173,8 +173,8 @@ export const devClientVersion = "0.0.0" export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets export const MAX_AUTOMATION_RECURRING_ERRORS = 5 export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber" -export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" -export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" -export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" -export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" -export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" +export const DEFAULT_JOBS_TABLE_ID = constants.DEFAULT_JOBS_TABLE_ID +export const DEFAULT_INVENTORY_TABLE_ID = constants.DEFAULT_INVENTORY_TABLE_ID +export const DEFAULT_EXPENSES_TABLE_ID = constants.DEFAULT_EXPENSES_TABLE_ID +export const DEFAULT_EMPLOYEE_TABLE_ID = constants.DEFAULT_EMPLOYEE_TABLE_ID +export const DEFAULT_BB_DATASOURCE_ID = constants.DEFAULT_BB_DATASOURCE_ID diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index b90fa5db0c..acaec4c85d 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -3,8 +3,3 @@ * internal to the server and don't need to * * be exposed for use by other services. * ********************************************/ - -export interface QueryOptions { - disableReturning?: boolean - disableBindings?: boolean -} diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 5626d7eda3..af535891cf 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -1,40 +1,41 @@ import { + ConnectionInfo, + DatasourceFeature, DatasourceFieldType, + DatasourcePlus, + DatasourcePlusQueryResponse, Integration, Operation, - Table, - TableSchema, QueryJson, QueryType, - SqlQuery, - DatasourcePlus, - DatasourceFeature, - ConnectionInfo, - SourceName, Schema, + SourceName, + SqlClient, + SqlQuery, + Table, + TableSchema, TableSourceType, - DatasourcePlusQueryResponse, } from "@budibase/types" import { - getSqlQuery, buildExternalTableId, - generateColumnDefinition, - finaliseExternalTables, - SqlClient, checkExternalTables, + finaliseExternalTables, + generateColumnDefinition, + getSqlQuery, HOST_ADDRESS, } from "./utils" -import Sql from "./base/sql" -import { MSSQLTablesResponse, MSSQLColumn } from "./base/types" +import { MSSQLColumn, MSSQLTablesResponse } from "./base/types" import { getReadableErrorMessage } from "./base/errorMapping" import sqlServer from "mssql" - -const DEFAULT_SCHEMA = "dbo" - +import { sql } from "@budibase/backend-core" import { ConfidentialClientApplication } from "@azure/msal-node" import { utils } from "@budibase/shared-core" +const Sql = sql.Sql + +const DEFAULT_SCHEMA = "dbo" + enum MSSQLConfigAuthType { AZURE_ACTIVE_DIRECTORY = "Azure Active Directory", NTLM = "NTLM", @@ -590,8 +591,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { scriptParts.push(createTableStatement) } - const schema = scriptParts.join("\n") - return schema + return scriptParts.join("\n") } } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 19a63a44ad..ecb3c07fa4 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -14,10 +14,10 @@ import { TableSourceType, DatasourcePlusQueryResponse, SqlQueryBinding, + SqlClient, } from "@budibase/types" import { getSqlQuery, - SqlClient, buildExternalTableId, generateColumnDefinition, finaliseExternalTables, @@ -26,11 +26,13 @@ import { } from "./utils" import dayjs from "dayjs" import { NUMBER_REGEX } from "../utilities" -import Sql from "./base/sql" import { MySQLColumn } from "./base/types" import { getReadableErrorMessage } from "./base/errorMapping" +import { sql } from "@budibase/backend-core" import mysql from "mysql2/promise" +const Sql = sql.Sql + interface MySQLConfig extends mysql.ConnectionOptions { database: string rejectUnauthorized: boolean diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 8105edfef8..55e62260b8 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -14,6 +14,7 @@ import { TableSourceType, Row, DatasourcePlusQueryResponse, + SqlClient, } from "@budibase/types" import { buildExternalTableId, @@ -21,10 +22,8 @@ import { generateColumnDefinition, finaliseExternalTables, getSqlQuery, - SqlClient, HOST_ADDRESS, } from "./utils" -import Sql from "./base/sql" import { BindParameters, Connection, @@ -33,6 +32,9 @@ import { Result, } from "oracledb" import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types" +import { sql } from "@budibase/backend-core" + +const Sql = sql.Sql let oracledb: any try { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index e810986757..3711db6950 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -13,17 +13,16 @@ import { Schema, TableSourceType, DatasourcePlusQueryResponse, + SqlClient, } from "@budibase/types" import { getSqlQuery, buildExternalTableId, generateColumnDefinition, finaliseExternalTables, - SqlClient, checkExternalTables, HOST_ADDRESS, } from "./utils" -import Sql from "./base/sql" import { PostgresColumn } from "./base/types" import { escapeDangerousCharacters } from "../utilities" @@ -31,7 +30,7 @@ import { Client, ClientConfig, types } from "pg" import { getReadableErrorMessage } from "./base/errorMapping" import { exec } from "child_process" import { storeTempFile } from "../utilities/fileSystem" -import { env } from "@budibase/backend-core" +import { env, sql } from "@budibase/backend-core" // Return "date" and "timestamp" types as plain strings. // This lets us reference the original stored timezone. @@ -43,6 +42,7 @@ if (types) { } const JSON_REGEX = /'{.*}'::json/s +const Sql = sql.Sql interface PostgresConfig { host: string diff --git a/packages/server/src/integrations/utils/index.ts b/packages/server/src/integrations/utils/index.ts index a9c2019ba2..3eeaeaa90c 100644 --- a/packages/server/src/integrations/utils/index.ts +++ b/packages/server/src/integrations/utils/index.ts @@ -1,2 +1 @@ export * from "./utils" -export { SqlStatements } from "./sqlStatements" diff --git a/packages/server/src/integrations/utils/sqlStatements.ts b/packages/server/src/integrations/utils/sqlStatements.ts index 7a5482830b..e69de29bb2 100644 --- a/packages/server/src/integrations/utils/sqlStatements.ts +++ b/packages/server/src/integrations/utils/sqlStatements.ts @@ -1,80 +0,0 @@ -import { FieldType, Table, FieldSchema } from "@budibase/types" -import { SqlClient } from "./utils" -import { Knex } from "knex" - -export class SqlStatements { - client: string - table: Table - allOr: boolean | undefined - constructor( - client: string, - table: Table, - { allOr }: { allOr?: boolean } = {} - ) { - this.client = client - this.table = table - this.allOr = allOr - } - - getField(key: string): FieldSchema | undefined { - const fieldName = key.split(".")[1] - return this.table.schema[fieldName] - } - - between( - query: Knex.QueryBuilder, - key: string, - low: number | string, - high: number | string - ) { - // Use a between operator if we have 2 valid range values - const field = this.getField(key) - if ( - field?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, - [low, high] - ) - } else { - const fnc = this.allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [low, high]) - } - return query - } - - lte(query: Knex.QueryBuilder, key: string, low: number | string) { - // Use just a single greater than operator if we only have a low - const field = this.getField(key) - if ( - field?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ - low, - ]) - } else { - const fnc = this.allOr ? "orWhere" : "where" - query = query[fnc](key, ">=", low) - } - return query - } - - gte(query: Knex.QueryBuilder, key: string, high: number | string) { - const field = this.getField(key) - // Use just a single less than operator if we only have a high - if ( - field?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ - high, - ]) - } else { - const fnc = this.allOr ? "orWhere" : "where" - query = query[fnc](key, "<=", high) - } - return query - } -} diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 892d8ae034..9d95ee9eb5 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -3,23 +3,16 @@ import { Table, Datasource, FieldType, - TableSourceType, FieldSchema, } from "@budibase/types" -import { context, objectStore } from "@budibase/backend-core" +import { context, objectStore, sql } from "@budibase/backend-core" import { v4 } from "uuid" import { parseStringPromise as xmlParser } from "xml2js" import { formatBytes } from "../../utilities" import bl from "bl" import env from "../../environment" -import { DocumentType, SEPARATOR } from "../../db/utils" -import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants" +import { InvalidColumns } from "../../constants" import { helpers, utils } from "@budibase/shared-core" -import { Knex } from "knex" - -const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` -const ROW_ID_REGEX = /^\[.*]$/g -const ENCODED_SPACE = encodeURIComponent(" ") type PrimitiveTypes = | FieldType.STRING @@ -109,13 +102,15 @@ const SQL_TYPE_MAP: Record = { ...SQL_OPTIONS_TYPE_MAP, } -export enum SqlClient { - MS_SQL = "mssql", - POSTGRES = "pg", - MY_SQL = "mysql2", - ORACLE = "oracledb", - SQL_LITE = "sqlite3", -} +export const isExternalTableID = sql.utils.isExternalTableID +export const isExternalTable = sql.utils.isExternalTable +export const buildExternalTableId = sql.utils.buildExternalTableId +export const breakExternalTableId = sql.utils.breakExternalTableId +export const generateRowIdField = sql.utils.generateRowIdField +export const isRowId = sql.utils.isRowId +export const convertRowId = sql.utils.convertRowId +export const breakRowIdField = sql.utils.breakRowIdField +export const isValidFilter = sql.utils.isValidFilter const isCloud = env.isProd() && !env.SELF_HOSTED const isSelfHost = env.isProd() && env.SELF_HOSTED @@ -125,119 +120,6 @@ export const HOST_ADDRESS = isSelfHost ? "" : "localhost" -export function isExternalTableID(tableId: string) { - return tableId.includes(DocumentType.DATASOURCE) -} - -export function isInternalTableID(tableId: string) { - return !isExternalTableID(tableId) -} - -export function getNativeSql( - query: Knex.SchemaBuilder | Knex.QueryBuilder -): SqlQuery | SqlQuery[] { - let sql = query.toSQL() - if (Array.isArray(sql)) { - return sql as SqlQuery[] - } - let native: Knex.SqlNative | undefined - if (sql.toNative) { - native = sql.toNative() - } - return { - sql: native?.sql || sql.sql, - bindings: native?.bindings || sql.bindings, - } as SqlQuery -} - -export function isExternalTable(table: Table) { - if ( - table?.sourceId && - table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) && - table?.sourceId !== DEFAULT_BB_DATASOURCE_ID - ) { - return true - } else if (table?.sourceType === TableSourceType.EXTERNAL) { - return true - } else if (table?._id && isExternalTableID(table._id)) { - return true - } - return false -} - -export function buildExternalTableId(datasourceId: string, tableName: string) { - // encode spaces - if (tableName.includes(" ")) { - tableName = encodeURIComponent(tableName) - } - return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}` -} - -export function breakExternalTableId(tableId: string | undefined) { - if (!tableId) { - return {} - } - const parts = tableId.split(DOUBLE_SEPARATOR) - let datasourceId = parts.shift() - // if they need joined - let tableName = parts.join(DOUBLE_SEPARATOR) - // if contains encoded spaces, decode it - if (tableName.includes(ENCODED_SPACE)) { - tableName = decodeURIComponent(tableName) - } - return { datasourceId, tableName } -} - -export function generateRowIdField(keyProps: any[] = []) { - if (!Array.isArray(keyProps)) { - keyProps = [keyProps] - } - for (let index in keyProps) { - if (keyProps[index] instanceof Buffer) { - keyProps[index] = keyProps[index].toString() - } - } - // this conserves order and types - // we have to swap the double quotes to single quotes for use in HBS statements - // when using the literal helper the double quotes can break things - return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'")) -} - -export function isRowId(field: any) { - return ( - Array.isArray(field) || - (typeof field === "string" && field.match(ROW_ID_REGEX) != null) - ) -} - -export function convertRowId(field: any) { - if (Array.isArray(field)) { - return field[0] - } - if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) { - return field.substring(1, field.length - 1) - } - return field -} - -// should always return an array -export function breakRowIdField(_id: string | { _id: string }): any[] { - if (!_id) { - return [] - } - // have to replace on the way back as we swapped out the double quotes - // when encoding, but JSON can't handle the single quotes - const id = typeof _id === "string" ? _id : _id._id - const decoded: string = decodeURIComponent(id).replace(/'/g, '"') - try { - const parsed = JSON.parse(decoded) - return Array.isArray(parsed) ? parsed : [parsed] - } catch (err) { - // wasn't json - likely was handlebars for a many to many - return [_id] - } -} - export function generateColumnDefinition(config: { externalType: string autocolumn: boolean @@ -297,15 +179,6 @@ export function isSQL(datasource: Datasource) { return helpers.isSQL(datasource) } -export function isIsoDateString(str: string) { - const trimmedValue = str.trim() - if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) { - return false - } - let d = new Date(trimmedValue) - return d.toISOString() === trimmedValue -} - /** * Looks for columns which need to be copied over into the new table definitions, like relationships, * options types and views. @@ -451,34 +324,6 @@ export function checkExternalTables( return errors } -/** - * Checks if the provided input is an object, but specifically not a date type object. - * Used during coercion of types and relationship handling, dates are considered valid - * and can be used as a display field, but objects and arrays cannot. - * @param testValue an unknown type which this function will attempt to extract - * a valid primary display string from. - */ -export function getPrimaryDisplay(testValue: unknown): string | undefined { - if (testValue instanceof Date) { - return testValue.toISOString() - } - if ( - Array.isArray(testValue) && - testValue[0] && - typeof testValue[0] !== "object" - ) { - return testValue.join(", ") - } - if (typeof testValue === "object") { - return undefined - } - return testValue as string -} - -export function isValidFilter(value: any) { - return value != null && value !== "" -} - export async function handleXml(response: any) { let data, rawXml = await response.text() @@ -517,12 +362,6 @@ export async function handleFileResponse( const contentLength = response.headers.get("content-length") if (contentLength) { size = parseInt(contentLength, 10) - } else { - const chunks: Buffer[] = [] - for await (const chunk of response.body) { - chunks.push(chunk) - size += chunk.length - } } await objectStore.streamUpload({ @@ -533,7 +372,7 @@ export async function handleFileResponse( type: response.headers["content-type"], }) } - presignedUrl = await objectStore.getPresignedUrl(bucket, key) + presignedUrl = objectStore.getPresignedUrl(bucket, key) return { data: { size, diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index a94ce265c5..02a93537b9 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -11,15 +11,14 @@ import { SortOrder, SortType, Table, + SqlClient, } from "@budibase/types" -import SqlQueryBuilder from "../../../../integrations/base/sql" -import { SqlClient } from "../../../../integrations/utils" import { buildInternalRelationships, sqlOutputProcessing, } from "../../../../api/controllers/row/utils" import sdk from "../../../index" -import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" +import { context, sql, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" import { CONSTANT_INTERNAL_ROW_COLS, SQS_DATASOURCE_INTERNAL, @@ -104,7 +103,7 @@ export async function search( ): Promise> { const { paginate, query, ...params } = options - const builder = new SqlQueryBuilder(SqlClient.SQL_LITE) + const builder = new sql.Sql(SqlClient.SQL_LITE) const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) if (!table) { diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 0fc338ecbe..4c5c88155b 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -5,12 +5,12 @@ import { QueryJson, Row, SearchFilters, + SqlClient, } from "@budibase/types" import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" import datasources from "../datasources" import { makeExternalQuery } from "../../../integrations/base/query" -import { SqlClient } from "../../../integrations/utils" import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils" const WRITE_OPERATIONS: Operation[] = [ diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index a9df4f89cd..62039737ba 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -9,12 +9,13 @@ import { SourceName, Table, TableSchema, + SqlClient, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { isRelationshipColumn } from "../../../db/utils" -import { SqlClient, isSQL } from "../../../integrations/utils" +import { isSQL } from "../../../integrations/utils" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 40f411f02a..6314da9e9b 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -117,6 +117,11 @@ export interface QueryJson { tableAliases?: Record } +export interface QueryOptions { + disableReturning?: boolean + disableBindings?: boolean +} + export type SqlQueryBinding = Knex.Value[] export interface SqlQuery { @@ -128,3 +133,11 @@ export enum EmptyFilterOption { RETURN_ALL = "all", RETURN_NONE = "none", } + +export enum SqlClient { + MS_SQL = "mssql", + POSTGRES = "pg", + MY_SQL = "mysql2", + ORACLE = "oracledb", + SQL_LITE = "sqlite3", +} diff --git a/scripts/build.js b/scripts/build.js index 7fbd242d4d..7c400e59e9 100755 --- a/scripts/build.js +++ b/scripts/build.js @@ -3,11 +3,11 @@ const start = Date.now() const fs = require("fs") -const { cp, readdir, copyFile, mkdir } = require('node:fs/promises'); +const { cp, readdir, copyFile, mkdir } = require("node:fs/promises") const path = require("path") const { build } = require("esbuild") -const { compile } = require('svelte/compiler') +const { compile } = require("svelte/compiler") const { default: TsconfigPathsPlugin, @@ -15,13 +15,13 @@ const { const { nodeExternalsPlugin } = require("esbuild-node-externals") const svelteCompilePlugin = { - name: 'svelteCompile', + name: "svelteCompile", setup(build) { // Compiles `.svelte` files into JS classes so that they can be directly imported into our // Typescript packages - build.onLoad({ filter: /\.svelte$/ }, async (args) => { - const source = await fs.promises.readFile(args.path, 'utf8') - const dir = path.dirname(args.path); + build.onLoad({ filter: /\.svelte$/ }, async args => { + const source = await fs.promises.readFile(args.path, "utf8") + const dir = path.dirname(args.path) try { const { js } = compile(source, { css: "injected", generate: "ssr" }) @@ -31,15 +31,15 @@ const svelteCompilePlugin = { contents: js.code, // The loader this is passed to, basically how the above provided content is "treated", // the contents provided above will be transpiled and bundled like any other JS file. - loader: 'js', + loader: "js", // Where to resolve any imports present in the loaded file - resolveDir: dir + resolveDir: dir, } } catch (e) { return { errors: [JSON.stringify(e)] } } }) - } + }, } var { argv } = require("yargs") @@ -75,7 +75,7 @@ async function runBuild(entry, outfile) { svelteCompilePlugin, TsconfigPathsPlugin({ tsconfig: tsconfigPathPluginContent }), nodeExternalsPlugin({ - allowList: ["@budibase/frontend-core", "svelte"] + allowList: ["@budibase/frontend-core", "svelte"], }), ], preserveSymlinks: true, @@ -90,25 +90,39 @@ async function runBuild(entry, outfile) { "bcryptjs", "graphql/*", "bson", + "better-sqlite3", + "sqlite3", + "mysql", + "mysql2", + "oracle", + "pg", + "pg-query-stream", + "pg-native", ], } - await mkdir('dist', { recursive: true }); + await mkdir("dist", { recursive: true }) const hbsFiles = (async () => { - const dir = await readdir('./', { recursive: true }); - const files = dir.filter(entry => entry.endsWith('.hbs') || entry.endsWith('ivm.bundle.js')); - const fileCopyPromises = files.map(file => copyFile(file, `dist/${path.basename(file)}`)) + const dir = await readdir("./", { recursive: true }) + const files = dir.filter( + entry => entry.endsWith(".hbs") || entry.endsWith("ivm.bundle.js") + ) + const fileCopyPromises = files.map(file => + copyFile(file, `dist/${path.basename(file)}`) + ) await Promise.all(fileCopyPromises) })() const oldClientVersions = (async () => { try { - await cp('./build/oldClientVersions', './dist/oldClientVersions', { recursive: true }); + await cp("./build/oldClientVersions", "./dist/oldClientVersions", { + recursive: true, + }) } catch (e) { if (e.code !== "EEXIST" && e.code !== "ENOENT") { - throw e; + throw e } } })() From d7f3109a22120221777c5a5e08e93748c4bf13c5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 17 May 2024 12:35:31 +0100 Subject: [PATCH 02/33] Making sqlite design doc generation more accessible. --- packages/backend-core/src/sql/designDoc.ts | 17 +++++++++++++++++ packages/backend-core/src/sql/index.ts | 1 + .../server/src/sdk/app/tables/internal/sqs.ts | 19 +++---------------- packages/types/src/documents/app/sqlite.ts | 2 ++ 4 files changed, 23 insertions(+), 16 deletions(-) create mode 100644 packages/backend-core/src/sql/designDoc.ts diff --git a/packages/backend-core/src/sql/designDoc.ts b/packages/backend-core/src/sql/designDoc.ts new file mode 100644 index 0000000000..dc334496a0 --- /dev/null +++ b/packages/backend-core/src/sql/designDoc.ts @@ -0,0 +1,17 @@ +import { PreSaveSQLiteDefinition } from "@budibase/types" +import { SQLITE_DESIGN_DOC_ID } from "../constants" + +// the table id property defines which property in the document +// to use when splitting the documents into different sqlite tables +export function base(tableIdProp: string): PreSaveSQLiteDefinition { + return { + _id: SQLITE_DESIGN_DOC_ID, + language: "sqlite", + sql: { + tables: {}, + options: { + table_name: tableIdProp, + }, + }, + } +} diff --git a/packages/backend-core/src/sql/index.ts b/packages/backend-core/src/sql/index.ts index 058b54e159..16b718d2e6 100644 --- a/packages/backend-core/src/sql/index.ts +++ b/packages/backend-core/src/sql/index.ts @@ -2,3 +2,4 @@ export * as utils from "./utils" export { default as Sql } from "./sql" export { default as SqlTable } from "./sqlTable" +export * as designDoc from "./designDoc" diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 5ecfd9692e..a6aa778f02 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -1,33 +1,20 @@ -import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" +import { context, sql, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" import { FieldType, RelationshipFieldMetadata, SQLiteDefinition, + PreSaveSQLiteDefinition, SQLiteTable, SQLiteTables, SQLiteType, Table, } from "@budibase/types" -import { cloneDeep } from "lodash" import tablesSdk from "../" import { CONSTANT_INTERNAL_ROW_COLS, generateJunctionTableID, } from "../../../../db/utils" -type PreSaveSQLiteDefinition = Omit - -const BASIC_SQLITE_DOC: PreSaveSQLiteDefinition = { - _id: SQLITE_DESIGN_DOC_ID, - language: "sqlite", - sql: { - tables: {}, - options: { - table_name: "tableId", - }, - }, -} - const FieldTypeMap: Record = { [FieldType.BOOLEAN]: SQLiteType.NUMERIC, [FieldType.DATETIME]: SQLiteType.TEXT, @@ -107,7 +94,7 @@ function mapTable(table: Table): SQLiteTables { // nothing exists, need to iterate though existing tables async function buildBaseDefinition(): Promise { const tables = await tablesSdk.getAllInternalTables() - const definition = cloneDeep(BASIC_SQLITE_DOC) + const definition = sql.designDoc.base("tableId") for (let table of tables) { definition.sql.tables = { ...definition.sql.tables, diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts index c380a2d6b4..5636fef15b 100644 --- a/packages/types/src/documents/app/sqlite.ts +++ b/packages/types/src/documents/app/sqlite.ts @@ -29,3 +29,5 @@ export interface SQLiteDefinition { } } } + +export type PreSaveSQLiteDefinition = Omit From d1b6a542fae645f2710ea8d1c51d414b37f9406f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 17 May 2024 20:26:34 +0100 Subject: [PATCH 03/33] Getting audit log searching with sqs working - a bit more work than expected but fixed quite a few SQS bugs along the way. --- packages/backend-core/src/constants/db.ts | 1 + packages/backend-core/src/sql/sql.ts | 11 ++++-- packages/pro | 2 +- packages/server/src/db/utils.ts | 1 - .../server/src/sdk/app/rows/search/sqs.ts | 38 ++++++++++++++----- packages/server/src/sdk/app/rows/sqlAlias.ts | 2 +- .../types/src/documents/global/auditLogs.ts | 2 + packages/types/src/sdk/search.ts | 3 ++ 8 files changed, 43 insertions(+), 17 deletions(-) diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index 43ed1ad96a..2fd713119b 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -65,6 +65,7 @@ export const StaticDatabases = { export const APP_PREFIX = prefixed(DocumentType.APP) export const APP_DEV = prefixed(DocumentType.APP_DEV) export const APP_DEV_PREFIX = APP_DEV +export const SQS_DATASOURCE_INTERNAL = "internal" export const BUDIBASE_DATASOURCE_TYPE = "budibase" export const SQLITE_DESIGN_DOC_ID = "_design/sqlite" export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 554343dab7..9052c3b04c 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -50,7 +50,7 @@ function likeKey(client: string, key: string): string { end = "]" break case SqlClient.SQL_LITE: - start = end = "'" + start = end = "" break default: throw new Error("Unknown client generating like key") @@ -198,17 +198,20 @@ class InternalBuilder { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { - fn(`${getTableAlias(tableName)}.${updatedKey}`, value) + const alias = getTableAlias(tableName) + fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) } if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") - fn(`${getTableAlias(filterTableName)}.${property}`, value) + const alias = getTableAlias(filterTableName) + fn(alias ? `${alias}.${property}` : property, value) } } } const like = (key: string, value: any) => { - const fnc = allOr ? "orWhere" : "where" + const fuzzyOr = filters?.fuzzyOr + const fnc = fuzzyOr || allOr ? "orWhere" : "where" // postgres supports ilike, nothing else does if (this.client === SqlClient.POSTGRES) { query = query[fnc](key, "ilike", `%${value}%`) diff --git a/packages/pro b/packages/pro index d3c3077011..a6492c51ea 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit d3c3077011a8e20ed3c48dcd6301caca4120b6ac +Subproject commit a6492c51ea691c3ff969a7b92d4c66f919c06417 diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index ce8d0accbb..b1c02b1764 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -40,7 +40,6 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}` export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}` export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}` -export const SQS_DATASOURCE_INTERNAL = "internal" export const ViewName = dbCore.ViewName export const InternalTables = dbCore.InternalTable export const UNICODE_MAX = dbCore.UNICODE_MAX diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 02a93537b9..c6c76e7959 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -18,11 +18,13 @@ import { sqlOutputProcessing, } from "../../../../api/controllers/row/utils" import sdk from "../../../index" -import { context, sql, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" import { - CONSTANT_INTERNAL_ROW_COLS, + context, + sql, + SQLITE_DESIGN_DOC_ID, SQS_DATASOURCE_INTERNAL, -} from "../../../../db/utils" +} from "@budibase/backend-core" +import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" import AliasTables from "../sqlAlias" import { outputProcessing } from "../../../../utilities/rowProcessor" @@ -146,10 +148,16 @@ export async function search( }, } } + + if (typeof params.bookmark !== "number") { + throw new Error("Unable to paginate with string based bookmarks") + } + const bookmark: number = params.bookmark || 1 + const limit = params.limit if (paginate && params.limit) { request.paginate = { limit: params.limit, - page: params.bookmark, + page: bookmark, } } try { @@ -185,12 +193,22 @@ export async function search( } ) - return { - // final row processing for response - rows: await outputProcessing(table, processed, { - preserveLinks: true, - squash: true, - }), + const finalRows = await outputProcessing(table, processed, { + preserveLinks: true, + squash: true, + }) + if (paginate && limit) { + return { + // final row processing for response + rows: finalRows, + bookmark: bookmark + 1, + // TODO: need to work out if next page available + hasNextPage: false, + } + } else { + return { + rows: finalRows, + } } } catch (err: any) { const msg = typeof err === "string" ? err : err.message diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 4c5c88155b..3d3ecd76e5 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -7,11 +7,11 @@ import { SearchFilters, SqlClient, } from "@budibase/types" +import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" import datasources from "../datasources" import { makeExternalQuery } from "../../../integrations/base/query" -import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils" const WRITE_OPERATIONS: Operation[] = [ Operation.CREATE, diff --git a/packages/types/src/documents/global/auditLogs.ts b/packages/types/src/documents/global/auditLogs.ts index 091c7b8418..084bec9e38 100644 --- a/packages/types/src/documents/global/auditLogs.ts +++ b/packages/types/src/documents/global/auditLogs.ts @@ -2,6 +2,7 @@ import { Document } from "../document" import { Event } from "../../sdk" export const AuditLogSystemUser = "SYSTEM" +export const AUDIT_LOG_TYPE = "auditLog" export type FallbackInfo = { appName?: string @@ -15,5 +16,6 @@ export interface AuditLogDoc extends Document { timestamp: string metadata: any name: string + type?: "auditLog" fallback?: FallbackInfo } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 6314da9e9b..e59a4980e3 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -19,6 +19,9 @@ export enum SearchFilterOperator { export interface SearchFilters { allOr?: boolean + // TODO: this is just around for now - we need a better way to do or/and + // allows just fuzzy to be or - all the fuzzy/like parameters + fuzzyOr?: boolean onEmptyFilter?: EmptyFilterOption [SearchFilterOperator.STRING]?: { [key: string]: string From 93e48245490cf1c40830116affb4ca571292c24a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 May 2024 15:25:55 +0100 Subject: [PATCH 04/33] Fixing some error scenarios --- packages/server/src/sdk/app/rows/search/sqs.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index c6c76e7959..12f242a90f 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -149,10 +149,10 @@ export async function search( } } - if (typeof params.bookmark !== "number") { + if (params.bookmark && typeof params.bookmark !== "number") { throw new Error("Unable to paginate with string based bookmarks") } - const bookmark: number = params.bookmark || 1 + const bookmark: number = (params.bookmark as number) || 1 const limit = params.limit if (paginate && params.limit) { request.paginate = { @@ -212,7 +212,7 @@ export async function search( } } catch (err: any) { const msg = typeof err === "string" ? err : err.message - if (err.status === 404 && err.message?.includes(SQLITE_DESIGN_DOC_ID)) { + if (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID)) { await sdk.tables.sqs.syncDefinition() return search(options, table) } From 40dbe52c2eabe32b0111a84985df92416c6517df Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 May 2024 18:22:46 +0100 Subject: [PATCH 05/33] Fixing some issues with test cases. --- packages/server/src/integrations/tests/sql.spec.ts | 6 ++++-- packages/server/src/integrations/tests/sqlAlias.spec.ts | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index 5de9cc4fbc..64bbaffb96 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -1,12 +1,14 @@ -import { SqlClient } from "../utils" -import Sql from "../base/sql" import { FieldType, Operation, QueryJson, Table, TableSourceType, + SqlClient, } from "@budibase/types" +import { sql } from "@budibase/backend-core" + +const Sql = sql.Sql const TABLE_NAME = "test" const TABLE: Table = { diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index fda2a091fa..8b7ac0e947 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -6,13 +6,15 @@ import { SqlQuery, Table, TableSourceType, + SqlClient, } from "@budibase/types" +import { sql } from "@budibase/backend-core" import { join } from "path" -import Sql from "../base/sql" -import { SqlClient } from "../utils" import { generator } from "@budibase/backend-core/tests" import sdk from "../../sdk" +const Sql = sql.Sql + // this doesn't exist strictly const TABLE: Table = { type: "table", From 082092b3928c31934920c9d57ecdec1a62cfb74a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 May 2024 12:39:36 +0100 Subject: [PATCH 06/33] Handling pagination for SQS API. --- .../server/src/sdk/app/rows/search/sqs.ts | 69 ++-- yarn.lock | 341 ++++++++++++++++++ 2 files changed, 382 insertions(+), 28 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 12f242a90f..bee5d41ed1 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -4,14 +4,14 @@ import { QueryJson, RelationshipFieldMetadata, Row, - SearchFilters, RowSearchParams, + SearchFilters, SearchResponse, SortDirection, SortOrder, SortType, - Table, SqlClient, + Table, } from "@budibase/types" import { buildInternalRelationships, @@ -99,13 +99,35 @@ function buildTableMap(tables: Table[]) { return tableMap } +async function runSqlQuery(json: QueryJson, tables: Table[]) { + const builder = new sql.Sql(SqlClient.SQL_LITE) + const alias = new AliasTables(tables.map(table => table.name)) + return await alias.queryWithAliasing(json, async json => { + const query = builder._query(json, { + disableReturning: true, + }) + + if (Array.isArray(query)) { + throw new Error("SQS cannot currently handle multiple queries") + } + + let sql = query.sql + let bindings = query.bindings + + // quick hack for docIds + sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") + sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") + const db = context.getAppDB() + return await db.sql(sql, bindings) + }) +} + export async function search( options: RowSearchParams, table: Table ): Promise> { const { paginate, query, ...params } = options - const builder = new sql.Sql(SqlClient.SQL_LITE) const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) if (!table) { @@ -161,26 +183,7 @@ export async function search( } } try { - const alias = new AliasTables(allTables.map(table => table.name)) - const rows = await alias.queryWithAliasing(request, async json => { - const query = builder._query(json, { - disableReturning: true, - }) - - if (Array.isArray(query)) { - throw new Error("SQS cannot currently handle multiple queries") - } - - let sql = query.sql - let bindings = query.bindings - - // quick hack for docIds - sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") - sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") - - const db = context.getAppDB() - return await db.sql(sql, bindings) - }) + const rows = await runSqlQuery(request, allTables) // process from the format of tableId.column to expected format const processed = await sqlOutputProcessing( @@ -198,13 +201,23 @@ export async function search( squash: true, }) if (paginate && limit) { - return { - // final row processing for response + const response: SearchResponse = { rows: finalRows, - bookmark: bookmark + 1, - // TODO: need to work out if next page available - hasNextPage: false, } + const prevLimit = request.paginate!.limit + request.paginate = { + limit: 1, + page: bookmark * prevLimit + 1, + } + // check if there is another row + const nextRow = await runSqlQuery(request, allTables) + // check if there is a row found + const hasNextPage = Array.isArray(nextRow) && nextRow.length >= 1 + response.hasNextPage = hasNextPage + if (hasNextPage) { + response.bookmark = bookmark + 1 + } + return response } else { return { rows: finalRows, diff --git a/yarn.lock b/yarn.lock index 36ce2ce75e..71ab88e601 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2050,6 +2050,7 @@ ioredis "5.3.2" joi "17.6.0" jsonwebtoken "9.0.2" + knex "2.4.2" koa-passport "^6.0.0" koa-pino-logger "4.0.0" lodash "4.17.21" @@ -2499,11 +2500,31 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" +"@eslint/eslintrc@^2.1.4": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.1.4.tgz#388a269f0f25c1b6adc317b5a2c55714894c70ad" + integrity sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ== + dependencies: + ajv "^6.12.4" + debug "^4.3.2" + espree "^9.6.0" + globals "^13.19.0" + ignore "^5.2.0" + import-fresh "^3.2.1" + js-yaml "^4.1.0" + minimatch "^3.1.2" + strip-json-comments "^3.1.1" + "@eslint/js@8.52.0": version "8.52.0" resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.52.0.tgz#78fe5f117840f69dc4a353adf9b9cd926353378c" integrity sha512-mjZVbpaeMZludF2fsWLD0Z9gCref1Tk4i9+wddjRvpUNqqcndPkBD09N/Mapey0b3jaXbLm2kICwFv2E64QinA== +"@eslint/js@8.57.0": + version "8.57.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.0.tgz#a5417ae8427873f1dd08b70b3574b453e67b5f7f" + integrity sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g== + "@fastify/busboy@^2.0.0": version "2.1.0" resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.0.tgz#0709e9f4cb252351c609c6e6d8d6779a8d25edff" @@ -2676,6 +2697,15 @@ debug "^4.1.1" minimatch "^3.0.5" +"@humanwhocodes/config-array@^0.11.14": + version "0.11.14" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" + integrity sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg== + dependencies: + "@humanwhocodes/object-schema" "^2.0.2" + debug "^4.3.1" + minimatch "^3.0.5" + "@humanwhocodes/module-importer@^1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" @@ -2686,6 +2716,11 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz#e5211452df060fa8522b55c7b3c0c4d1981cb044" integrity sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw== +"@humanwhocodes/object-schema@^2.0.2": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz#4a2868d75d6d6963e423bcf90b7fd1be343409d3" + integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA== + "@hutson/parse-repository-url@^3.0.0": version "3.0.2" resolved "https://registry.yarnpkg.com/@hutson/parse-repository-url/-/parse-repository-url-3.0.2.tgz#98c23c950a3d9b6c8f0daed06da6c3af06981340" @@ -3216,16 +3251,35 @@ "@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/trace-mapping" "^0.3.9" +"@jridgewell/gen-mapping@^0.3.5": + version "0.3.5" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36" + integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== + dependencies: + "@jridgewell/set-array" "^1.2.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.24" + "@jridgewell/resolve-uri@3.1.0", "@jridgewell/resolve-uri@^3.0.3": version "3.1.0" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== + "@jridgewell/set-array@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== +"@jridgewell/set-array@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280" + integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A== + "@jridgewell/source-map@^0.3.2": version "0.3.2" resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.2.tgz#f45351aaed4527a298512ec72f81040c998580fb" @@ -3234,6 +3288,14 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" +"@jridgewell/source-map@^0.3.3": + version "0.3.6" + resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.6.tgz#9d71ca886e32502eb9362c9a74a46787c36df81a" + integrity sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ== + dependencies: + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + "@jridgewell/sourcemap-codec@1.4.14": version "1.4.14" resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" @@ -3260,6 +3322,14 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" +"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": + version "0.3.25" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@js-joda/core@^5.5.3": version "5.6.1" resolved "https://registry.yarnpkg.com/@js-joda/core/-/core-5.6.1.tgz#03e2453d877b61c3f593cf031fd18b375bd548b6" @@ -3926,6 +3996,15 @@ "@rollup/pluginutils" "^5.0.1" magic-string "^0.30.3" +"@rollup/plugin-terser@^0.4.4": + version "0.4.4" + resolved "https://registry.yarnpkg.com/@rollup/plugin-terser/-/plugin-terser-0.4.4.tgz#15dffdb3f73f121aa4fbb37e7ca6be9aeea91962" + integrity sha512-XHeJC5Bgvs8LfukDwWZp7yeqin6ns8RTl2B9avbejt6tZqsqvVoWI7ZTQrcNsfKEDWBTnTxM8nMDkO2IFFbd0A== + dependencies: + serialize-javascript "^6.0.1" + smob "^1.0.0" + terser "^5.17.4" + "@rollup/plugin-typescript@8.3.0": version "8.3.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-typescript/-/plugin-typescript-8.3.0.tgz#bc1077fa5897b980fc27e376c4e377882c63e68b" @@ -3965,66 +4044,146 @@ resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.12.0.tgz#38c3abd1955a3c21d492af6b1a1dca4bb1d894d6" integrity sha512-+ac02NL/2TCKRrJu2wffk1kZ+RyqxVUlbjSagNgPm94frxtr+XDL12E5Ll1enWskLrtrZ2r8L3wED1orIibV/w== +"@rollup/rollup-android-arm-eabi@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.17.2.tgz#1a32112822660ee104c5dd3a7c595e26100d4c2d" + integrity sha512-NM0jFxY8bB8QLkoKxIQeObCaDlJKewVlIEkuyYKm5An1tdVZ966w2+MPQ2l8LBZLjR+SgyV+nRkTIunzOYBMLQ== + "@rollup/rollup-android-arm64@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.12.0.tgz#3822e929f415627609e53b11cec9a4be806de0e2" integrity sha512-OBqcX2BMe6nvjQ0Nyp7cC90cnumt8PXmO7Dp3gfAju/6YwG0Tj74z1vKrfRz7qAv23nBcYM8BCbhrsWqO7PzQQ== +"@rollup/rollup-android-arm64@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.17.2.tgz#5aeef206d65ff4db423f3a93f71af91b28662c5b" + integrity sha512-yeX/Usk7daNIVwkq2uGoq2BYJKZY1JfyLTaHO/jaiSwi/lsf8fTFoQW/n6IdAsx5tx+iotu2zCJwz8MxI6D/Bw== + "@rollup/rollup-darwin-arm64@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.12.0.tgz#6c082de71f481f57df6cfa3701ab2a7afde96f69" integrity sha512-X64tZd8dRE/QTrBIEs63kaOBG0b5GVEd3ccoLtyf6IdXtHdh8h+I56C2yC3PtC9Ucnv0CpNFJLqKFVgCYe0lOQ== +"@rollup/rollup-darwin-arm64@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.17.2.tgz#6b66aaf003c70454c292cd5f0236ebdc6ffbdf1a" + integrity sha512-kcMLpE6uCwls023+kknm71ug7MZOrtXo+y5p/tsg6jltpDtgQY1Eq5sGfHcQfb+lfuKwhBmEURDga9N0ol4YPw== + "@rollup/rollup-darwin-x64@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.12.0.tgz#c34ca0d31f3c46a22c9afa0e944403eea0edcfd8" integrity sha512-cc71KUZoVbUJmGP2cOuiZ9HSOP14AzBAThn3OU+9LcA1+IUqswJyR1cAJj3Mg55HbjZP6OLAIscbQsQLrpgTOg== +"@rollup/rollup-darwin-x64@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.17.2.tgz#f64fc51ed12b19f883131ccbcea59fc68cbd6c0b" + integrity sha512-AtKwD0VEx0zWkL0ZjixEkp5tbNLzX+FCqGG1SvOu993HnSz4qDI6S4kGzubrEJAljpVkhRSlg5bzpV//E6ysTQ== + "@rollup/rollup-linux-arm-gnueabihf@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.12.0.tgz#48e899c1e438629c072889b824a98787a7c2362d" integrity sha512-a6w/Y3hyyO6GlpKL2xJ4IOh/7d+APaqLYdMf86xnczU3nurFTaVN9s9jOXQg97BE4nYm/7Ga51rjec5nfRdrvA== +"@rollup/rollup-linux-arm-gnueabihf@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.17.2.tgz#1a7641111be67c10111f7122d1e375d1226cbf14" + integrity sha512-3reX2fUHqN7sffBNqmEyMQVj/CKhIHZd4y631duy0hZqI8Qoqf6lTtmAKvJFYa6bhU95B1D0WgzHkmTg33In0A== + +"@rollup/rollup-linux-arm-musleabihf@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.17.2.tgz#c93fd632923e0fee25aacd2ae414288d0b7455bb" + integrity sha512-uSqpsp91mheRgw96xtyAGP9FW5ChctTFEoXP0r5FAzj/3ZRv3Uxjtc7taRQSaQM/q85KEKjKsZuiZM3GyUivRg== + "@rollup/rollup-linux-arm64-gnu@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.12.0.tgz#788c2698a119dc229062d40da6ada8a090a73a68" integrity sha512-0fZBq27b+D7Ar5CQMofVN8sggOVhEtzFUwOwPppQt0k+VR+7UHMZZY4y+64WJ06XOhBTKXtQB/Sv0NwQMXyNAA== +"@rollup/rollup-linux-arm64-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.17.2.tgz#fa531425dd21d058a630947527b4612d9d0b4a4a" + integrity sha512-EMMPHkiCRtE8Wdk3Qhtciq6BndLtstqZIroHiiGzB3C5LDJmIZcSzVtLRbwuXuUft1Cnv+9fxuDtDxz3k3EW2A== + "@rollup/rollup-linux-arm64-musl@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.12.0.tgz#3882a4e3a564af9e55804beeb67076857b035ab7" integrity sha512-eTvzUS3hhhlgeAv6bfigekzWZjaEX9xP9HhxB0Dvrdbkk5w/b+1Sxct2ZuDxNJKzsRStSq1EaEkVSEe7A7ipgQ== +"@rollup/rollup-linux-arm64-musl@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.17.2.tgz#8acc16f095ceea5854caf7b07e73f7d1802ac5af" + integrity sha512-NMPylUUZ1i0z/xJUIx6VUhISZDRT+uTWpBcjdv0/zkp7b/bQDF+NfnfdzuTiB1G6HTodgoFa93hp0O1xl+/UbA== + +"@rollup/rollup-linux-powerpc64le-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.17.2.tgz#94e69a8499b5cf368911b83a44bb230782aeb571" + integrity sha512-T19My13y8uYXPw/L/k0JYaX1fJKFT/PWdXiHr8mTbXWxjVF1t+8Xl31DgBBvEKclw+1b00Chg0hxE2O7bTG7GQ== + "@rollup/rollup-linux-riscv64-gnu@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.12.0.tgz#0c6ad792e1195c12bfae634425a3d2aa0fe93ab7" integrity sha512-ix+qAB9qmrCRiaO71VFfY8rkiAZJL8zQRXveS27HS+pKdjwUfEhqo2+YF2oI+H/22Xsiski+qqwIBxVewLK7sw== +"@rollup/rollup-linux-riscv64-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.17.2.tgz#7ef1c781c7e59e85a6ce261cc95d7f1e0b56db0f" + integrity sha512-BOaNfthf3X3fOWAB+IJ9kxTgPmMqPPH5f5k2DcCsRrBIbWnaJCgX2ll77dV1TdSy9SaXTR5iDXRL8n7AnoP5cg== + +"@rollup/rollup-linux-s390x-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.17.2.tgz#f15775841c3232fca9b78cd25a7a0512c694b354" + integrity sha512-W0UP/x7bnn3xN2eYMql2T/+wpASLE5SjObXILTMPUBDB/Fg/FxC+gX4nvCfPBCbNhz51C+HcqQp2qQ4u25ok6g== + "@rollup/rollup-linux-x64-gnu@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.12.0.tgz#9d62485ea0f18d8674033b57aa14fb758f6ec6e3" integrity sha512-TenQhZVOtw/3qKOPa7d+QgkeM6xY0LtwzR8OplmyL5LrgTWIXpTQg2Q2ycBf8jm+SFW2Wt/DTn1gf7nFp3ssVA== +"@rollup/rollup-linux-x64-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.17.2.tgz#b521d271798d037ad70c9f85dd97d25f8a52e811" + integrity sha512-Hy7pLwByUOuyaFC6mAr7m+oMC+V7qyifzs/nW2OJfC8H4hbCzOX07Ov0VFk/zP3kBsELWNFi7rJtgbKYsav9QQ== + "@rollup/rollup-linux-x64-musl@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.12.0.tgz#50e8167e28b33c977c1f813def2b2074d1435e05" integrity sha512-LfFdRhNnW0zdMvdCb5FNuWlls2WbbSridJvxOvYWgSBOYZtgBfW9UGNJG//rwMqTX1xQE9BAodvMH9tAusKDUw== +"@rollup/rollup-linux-x64-musl@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.17.2.tgz#9254019cc4baac35800991315d133cc9fd1bf385" + integrity sha512-h1+yTWeYbRdAyJ/jMiVw0l6fOOm/0D1vNLui9iPuqgRGnXA0u21gAqOyB5iHjlM9MMfNOm9RHCQ7zLIzT0x11Q== + "@rollup/rollup-win32-arm64-msvc@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.12.0.tgz#68d233272a2004429124494121a42c4aebdc5b8e" integrity sha512-JPDxovheWNp6d7AHCgsUlkuCKvtu3RB55iNEkaQcf0ttsDU/JZF+iQnYcQJSk/7PtT4mjjVG8N1kpwnI9SLYaw== +"@rollup/rollup-win32-arm64-msvc@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.17.2.tgz#27f65a89f6f52ee9426ec11e3571038e4671790f" + integrity sha512-tmdtXMfKAjy5+IQsVtDiCfqbynAQE/TQRpWdVataHmhMb9DCoJxp9vLcCBjEQWMiUYxO1QprH/HbY9ragCEFLA== + "@rollup/rollup-win32-ia32-msvc@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.12.0.tgz#366ca62221d1689e3b55a03f4ae12ae9ba595d40" integrity sha512-fjtuvMWRGJn1oZacG8IPnzIV6GF2/XG+h71FKn76OYFqySXInJtseAqdprVTDTyqPxQOG9Exak5/E9Z3+EJ8ZA== +"@rollup/rollup-win32-ia32-msvc@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.17.2.tgz#a2fbf8246ed0bb014f078ca34ae6b377a90cb411" + integrity sha512-7II/QCSTAHuE5vdZaQEwJq2ZACkBpQDOmQsE6D6XUbnBHW8IAhm4eTufL6msLJorzrHDFv3CF8oCA/hSIRuZeQ== + "@rollup/rollup-win32-x64-msvc@4.12.0": version "4.12.0" resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.12.0.tgz#9ffdf9ed133a7464f4ae187eb9e1294413fab235" integrity sha512-ZYmr5mS2wd4Dew/JjT0Fqi2NPB/ZhZ2VvPp7SmvPZb4Y1CG/LRcS6tcRo2cYU7zLK5A7cdbhWnnWmUjoI4qapg== +"@rollup/rollup-win32-x64-msvc@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.17.2.tgz#5a2d08b81e8064b34242d5cc9973ef8dd1e60503" + integrity sha512-TGGO7v7qOq4CYmSBVEYpI1Y5xDuCEnbVC5Vth8mOsW0gDSzxNrVERPc790IGHsrT2dQSimgMr9Ub3Y1Jci5/8w== + "@roxi/routify@2.18.0": version "2.18.0" resolved "https://registry.yarnpkg.com/@roxi/routify/-/routify-2.18.0.tgz#8f88bedd936312d0dbe44cbc11ab179b1f938ec2" @@ -6945,6 +7104,18 @@ atomic-sleep@^1.0.0: resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b" integrity sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ== +atrament@^4.3.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/atrament/-/atrament-4.4.0.tgz#01898c90d08941b0d64494f13a52f79e0c0ce5d1" + integrity sha512-kUlJvSrr8QF4eSQzkkBE77FsGUWXIKf1Zl0qv7BWki8B5vPYMVKDn+a8ayPqTX9IvJx6466uLnYFIksX91kFzw== + dependencies: + "@rollup/plugin-terser" "^0.4.4" + eslint "^8.56.0" + eslint-config-airbnb-base "^15.0.0" + eslint-plugin-import "^2.26.0" + rollup "^4.9.4" + rollup-plugin-web-worker-loader "^1.6.1" + available-typed-arrays@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" @@ -8369,6 +8540,11 @@ configstore@^5.0.1: write-file-atomic "^3.0.0" xdg-basedir "^4.0.0" +confusing-browser-globals@^1.0.10: + version "1.0.11" + resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz#ae40e9b57cdd3915408a2805ebd3a5585608dc81" + integrity sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA== + console-control-strings@^1.0.0, console-control-strings@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" @@ -10025,6 +10201,13 @@ es-get-iterator@^1.1.2: isarray "^2.0.5" stop-iteration-iterator "^1.0.0" +es-object-atoms@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.0.0.tgz#ddb55cd47ac2e240701260bc2a8e31ecb643d941" + integrity sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw== + dependencies: + es-errors "^1.3.0" + es-set-tostringtag@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz#338d502f6f674301d710b80c8592de8a15f09cd8" @@ -10164,6 +10347,16 @@ eslint-compat-utils@^0.1.2: resolved "https://registry.yarnpkg.com/eslint-compat-utils/-/eslint-compat-utils-0.1.2.tgz#f45e3b5ced4c746c127cf724fb074cd4e730d653" integrity sha512-Jia4JDldWnFNIru1Ehx1H5s9/yxiRHY/TimCuUc0jNexew3cF1gI6CYZil1ociakfWO3rRqFjl1mskBblB3RYg== +eslint-config-airbnb-base@^15.0.0: + version "15.0.0" + resolved "https://registry.yarnpkg.com/eslint-config-airbnb-base/-/eslint-config-airbnb-base-15.0.0.tgz#6b09add90ac79c2f8d723a2580e07f3925afd236" + integrity sha512-xaX3z4ZZIcFLvh2oUNvcX5oEofXda7giYmuplVxoOg5A7EXJMrUyqRgR+mhDhPK8LZ4PttFOBvCYDbX3sUoUig== + dependencies: + confusing-browser-globals "^1.0.10" + object.assign "^4.1.2" + object.entries "^1.1.5" + semver "^6.3.0" + eslint-import-resolver-node@^0.3.9: version "0.3.9" resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz#d4eaac52b8a2e7c3cd1903eb00f7e053356118ac" @@ -10180,6 +10373,29 @@ eslint-module-utils@^2.8.0: dependencies: debug "^3.2.7" +eslint-plugin-import@^2.26.0: + version "2.29.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz#d45b37b5ef5901d639c15270d74d46d161150643" + integrity sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw== + dependencies: + array-includes "^3.1.7" + array.prototype.findlastindex "^1.2.3" + array.prototype.flat "^1.3.2" + array.prototype.flatmap "^1.3.2" + debug "^3.2.7" + doctrine "^2.1.0" + eslint-import-resolver-node "^0.3.9" + eslint-module-utils "^2.8.0" + hasown "^2.0.0" + is-core-module "^2.13.1" + is-glob "^4.0.3" + minimatch "^3.1.2" + object.fromentries "^2.0.7" + object.groupby "^1.0.1" + object.values "^1.1.7" + semver "^6.3.1" + tsconfig-paths "^3.15.0" + eslint-plugin-import@^2.29.0: version "2.29.0" resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.29.0.tgz#8133232e4329ee344f2f612885ac3073b0b7e155" @@ -10303,6 +10519,50 @@ eslint@^8.52.0: strip-ansi "^6.0.1" text-table "^0.2.0" +eslint@^8.56.0: + version "8.57.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.57.0.tgz#c786a6fd0e0b68941aaf624596fb987089195668" + integrity sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ== + dependencies: + "@eslint-community/eslint-utils" "^4.2.0" + "@eslint-community/regexpp" "^4.6.1" + "@eslint/eslintrc" "^2.1.4" + "@eslint/js" "8.57.0" + "@humanwhocodes/config-array" "^0.11.14" + "@humanwhocodes/module-importer" "^1.0.1" + "@nodelib/fs.walk" "^1.2.8" + "@ungap/structured-clone" "^1.2.0" + ajv "^6.12.4" + chalk "^4.0.0" + cross-spawn "^7.0.2" + debug "^4.3.2" + doctrine "^3.0.0" + escape-string-regexp "^4.0.0" + eslint-scope "^7.2.2" + eslint-visitor-keys "^3.4.3" + espree "^9.6.1" + esquery "^1.4.2" + esutils "^2.0.2" + fast-deep-equal "^3.1.3" + file-entry-cache "^6.0.1" + find-up "^5.0.0" + glob-parent "^6.0.2" + globals "^13.19.0" + graphemer "^1.4.0" + ignore "^5.2.0" + imurmurhash "^0.1.4" + is-glob "^4.0.0" + is-path-inside "^3.0.3" + js-yaml "^4.1.0" + json-stable-stringify-without-jsonify "^1.0.1" + levn "^0.4.1" + lodash.merge "^4.6.2" + minimatch "^3.1.2" + natural-compare "^1.4.0" + optionator "^0.9.3" + strip-ansi "^6.0.1" + text-table "^0.2.0" + esm@^3.2.25: version "3.2.25" resolved "https://registry.yarnpkg.com/esm/-/esm-3.2.25.tgz#342c18c29d56157688ba5ce31f8431fbb795cc10" @@ -16072,6 +16332,16 @@ object-sizeof@2.6.1: dependencies: buffer "^6.0.3" +object.assign@^4.1.2: + version "4.1.5" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.5.tgz#3a833f9ab7fdb80fc9e8d2300c803d216d8fdbb0" + integrity sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ== + dependencies: + call-bind "^1.0.5" + define-properties "^1.2.1" + has-symbols "^1.0.3" + object-keys "^1.1.1" + object.assign@^4.1.4: version "4.1.4" resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.4.tgz#9673c7c7c351ab8c4d0b516f4343ebf4dfb7799f" @@ -16082,6 +16352,15 @@ object.assign@^4.1.4: has-symbols "^1.0.3" object-keys "^1.1.1" +object.entries@^1.1.5: + version "1.1.8" + resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.8.tgz#bffe6f282e01f4d17807204a24f8edd823599c41" + integrity sha512-cmopxi8VwRIAw/fkijJohSfpef5PdN0pMQJN6VC/ZKvn0LIknWD8KtgY6KlQdEc4tIjcQ3HxSMmnvtzIscdaYQ== + dependencies: + call-bind "^1.0.7" + define-properties "^1.2.1" + es-object-atoms "^1.0.0" + object.fromentries@^2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.7.tgz#71e95f441e9a0ea6baf682ecaaf37fa2a8d7e616" @@ -18824,6 +19103,11 @@ rollup-plugin-visualizer@^5.12.0: source-map "^0.7.4" yargs "^17.5.1" +rollup-plugin-web-worker-loader@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/rollup-plugin-web-worker-loader/-/rollup-plugin-web-worker-loader-1.6.1.tgz#9d7a27575b64b0780fe4e8b3bc87470d217e485f" + integrity sha512-4QywQSz1NXFHKdyiou16mH3ijpcfLtLGOrAqvAqu1Gx+P8+zj+3gwC2BSL/VW1d+LW4nIHC8F7d7OXhs9UdR2A== + rollup-pluginutils@^1.3.1: version "1.5.2" resolved "https://registry.yarnpkg.com/rollup-pluginutils/-/rollup-pluginutils-1.5.2.tgz#1e156e778f94b7255bfa1b3d0178be8f5c552408" @@ -18853,6 +19137,31 @@ rollup@^3.27.1: optionalDependencies: fsevents "~2.3.2" +rollup@^4.9.4: + version "4.17.2" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.17.2.tgz#26d1785d0144122277fdb20ab3a24729ae68301f" + integrity sha512-/9ClTJPByC0U4zNLowV1tMBe8yMEAxewtR3cUNX5BoEpGH3dQEWpJLr6CLp0fPdYRF/fzVOgvDb1zXuakwF5kQ== + dependencies: + "@types/estree" "1.0.5" + optionalDependencies: + "@rollup/rollup-android-arm-eabi" "4.17.2" + "@rollup/rollup-android-arm64" "4.17.2" + "@rollup/rollup-darwin-arm64" "4.17.2" + "@rollup/rollup-darwin-x64" "4.17.2" + "@rollup/rollup-linux-arm-gnueabihf" "4.17.2" + "@rollup/rollup-linux-arm-musleabihf" "4.17.2" + "@rollup/rollup-linux-arm64-gnu" "4.17.2" + "@rollup/rollup-linux-arm64-musl" "4.17.2" + "@rollup/rollup-linux-powerpc64le-gnu" "4.17.2" + "@rollup/rollup-linux-riscv64-gnu" "4.17.2" + "@rollup/rollup-linux-s390x-gnu" "4.17.2" + "@rollup/rollup-linux-x64-gnu" "4.17.2" + "@rollup/rollup-linux-x64-musl" "4.17.2" + "@rollup/rollup-win32-arm64-msvc" "4.17.2" + "@rollup/rollup-win32-ia32-msvc" "4.17.2" + "@rollup/rollup-win32-x64-msvc" "4.17.2" + fsevents "~2.3.2" + rollup@^4.9.6: version "4.12.0" resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.12.0.tgz#0b6d1e5f3d46bbcf244deec41a7421dc54cc45b5" @@ -19082,6 +19391,13 @@ serialize-javascript@^4.0.0: dependencies: randombytes "^2.1.0" +serialize-javascript@^6.0.1: + version "6.0.2" + resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.2.tgz#defa1e055c83bf6d59ea805d8da862254eb6a6c2" + integrity sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g== + dependencies: + randombytes "^2.1.0" + server-destroy@1.0.1, server-destroy@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/server-destroy/-/server-destroy-1.0.1.tgz#f13bf928e42b9c3e79383e61cc3998b5d14e6cdd" @@ -19280,6 +19596,11 @@ smart-buffer@^4.2.0: resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== +smob@^1.0.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/smob/-/smob-1.5.0.tgz#85d79a1403abf128d24d3ebc1cdc5e1a9548d3ab" + integrity sha512-g6T+p7QO8npa+/hNx9ohv1E5pVCmWrVCUzUXJyLdMmftX6ER0oiWY/w9knEonLpnOp6b6FenKnMfR8gqwWdwig== + snowflake-promise@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/snowflake-promise/-/snowflake-promise-4.5.0.tgz#ceba611d27b3792966bc752c545760e0ce168c1c" @@ -20355,6 +20676,16 @@ terser@^5.0.0: commander "^2.20.0" source-map-support "~0.5.20" +terser@^5.17.4: + version "5.31.0" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.31.0.tgz#06eef86f17007dbad4593f11a574c7f5eb02c6a1" + integrity sha512-Q1JFAoUKE5IMfI4Z/lkE/E6+SwgzO+x4tq4v1AyBLRj8VSYvRO6A/rQrPg1yud4g0En9EKI1TvFRF2tQFcoUkg== + dependencies: + "@jridgewell/source-map" "^0.3.3" + acorn "^8.8.2" + commander "^2.20.0" + source-map-support "~0.5.20" + test-exclude@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e" @@ -20703,6 +21034,16 @@ tsconfig-paths@^3.10.1, tsconfig-paths@^3.14.2: minimist "^1.2.6" strip-bom "^3.0.0" +tsconfig-paths@^3.15.0: + version "3.15.0" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz#5299ec605e55b1abb23ec939ef15edaf483070d4" + integrity sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg== + dependencies: + "@types/json5" "^0.0.29" + json5 "^1.0.2" + minimist "^1.2.6" + strip-bom "^3.0.0" + tsconfig-paths@^4.1.2, tsconfig-paths@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz#ef78e19039133446d244beac0fd6a1632e2d107c" From 540cd2f2f5068cfcc55a008ca912aff57b4c96ac Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 May 2024 13:15:27 +0100 Subject: [PATCH 07/33] Setting knex as dependency for worker (audit logs). --- packages/worker/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/worker/package.json b/packages/worker/package.json index 0c2a9f3b72..95410668da 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -70,7 +70,8 @@ "pouchdb-all-dbs": "1.1.1", "server-destroy": "1.0.1", "undici": "^6.0.1", - "undici-types": "^6.0.1" + "undici-types": "^6.0.1", + "knex": "2.4.2" }, "devDependencies": { "@swc/core": "1.3.71", From ffcd9e498d9f26266e7e7408e79b0cabf2404cae Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 May 2024 18:13:54 +0100 Subject: [PATCH 08/33] Some work to allow toggling between lucene search and SQL search for audit logs. --- packages/pro | 2 +- packages/worker/src/api/index.ts | 7 ++++++- packages/worker/src/environment.ts | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/pro b/packages/pro index a6492c51ea..06e908dc1f 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit a6492c51ea691c3ff969a7b92d4c66f919c06417 +Subproject commit 06e908dc1fd8a3d03eca5a0e92894612770cfbe8 diff --git a/packages/worker/src/api/index.ts b/packages/worker/src/api/index.ts index 4936c104e1..84627c56c6 100644 --- a/packages/worker/src/api/index.ts +++ b/packages/worker/src/api/index.ts @@ -4,8 +4,13 @@ const compress = require("koa-compress") import zlib from "zlib" import { routes } from "./routes" -import { middleware as pro } from "@budibase/pro" +import { middleware as pro, sdk } from "@budibase/pro" import { auth, middleware } from "@budibase/backend-core" +import env from "../environment" + +if (env.SQS_SEARCH_ENABLE) { + sdk.auditLogs.useSQLSearch() +} const PUBLIC_ENDPOINTS = [ // deprecated single tenant sso callback diff --git a/packages/worker/src/environment.ts b/packages/worker/src/environment.ts index f5c01573db..70fb911ee1 100644 --- a/packages/worker/src/environment.ts +++ b/packages/worker/src/environment.ts @@ -45,6 +45,7 @@ const environment = { DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, + SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, // smtp SMTP_USER: process.env.SMTP_USER, SMTP_PASSWORD: process.env.SMTP_PASSWORD, From aa9f836572ecfa7b3b4f0468a599c91a3b4e1fe0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 May 2024 18:14:04 +0100 Subject: [PATCH 09/33] Support dev build with SQS. --- hosting/docker-compose.build.yaml | 2 ++ scripts/devDocker.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml index dbc3613599..253dda0232 100644 --- a/hosting/docker-compose.build.yaml +++ b/hosting/docker-compose.build.yaml @@ -29,6 +29,7 @@ services: BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL} BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} + SQS_SEARCH_ENABLE: 1 depends_on: - worker-service - redis-service @@ -56,6 +57,7 @@ services: INTERNAL_API_KEY: ${INTERNAL_API_KEY} REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} + SQS_SEARCH_ENABLE: 1 depends_on: - redis-service - minio-service diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh index 5e01e5813a..e5981b7187 100755 --- a/scripts/devDocker.sh +++ b/scripts/devDocker.sh @@ -7,6 +7,6 @@ if [ ! -d "./packages/pro/src" ]; then fi yarn build --scope @budibase/server --scope @budibase/worker -docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0 +DOCKER_BUILDKIT=1 docker compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0 From 429d415cc9a6a453dd969b1e04c6688d1118da2d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 May 2024 18:14:19 +0100 Subject: [PATCH 10/33] Quick fix to allow data UI to show relational fields. --- .../backend/DataTable/buttons/TableFilterButton.svelte | 8 ++------ packages/builder/src/helpers/searchFields.js | 1 + 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index decf77069f..9fab2e5533 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -4,6 +4,7 @@ import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte" import { getUserBindings } from "dataBinding" import { makePropSafe } from "@budibase/string-templates" + import { getFields } from "helpers/searchFields" export let schema export let filters @@ -15,12 +16,7 @@ let drawer $: tempValue = filters || [] - $: schemaFields = Object.entries(schema || {}).map( - ([fieldName, fieldSchema]) => ({ - name: fieldName, // Using the key as name if not defined in the schema, for example in some autogenerated columns - ...fieldSchema, - }) - ) + $: schemaFields = getFields(Object.values(schema || {}), { allowLinks: true }) $: text = getText(filters) $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0 diff --git a/packages/builder/src/helpers/searchFields.js b/packages/builder/src/helpers/searchFields.js index 73e4f1aaa2..4cbe394e04 100644 --- a/packages/builder/src/helpers/searchFields.js +++ b/packages/builder/src/helpers/searchFields.js @@ -4,6 +4,7 @@ import { get } from "svelte/store" export function getTableFields(linkField) { const table = get(tables).list.find(table => table._id === linkField.tableId) + // TODO: mdrury - add support for this with SQS at some point if (!table || !table.sql) { return [] } From 2ed510dc90ca9ad3bef8ae0001282c7073e21819 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 22 May 2024 16:02:11 +0100 Subject: [PATCH 11/33] Fixing build issue. --- packages/server/src/api/controllers/row/views.ts | 7 ++----- packages/types/src/sdk/search.ts | 5 +++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 5237ca712a..13f6ce8eb0 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -4,8 +4,8 @@ import { SearchRowResponse, SearchViewRowRequest, RequiredKeys, - SearchFilters, RowSearchParams, + SearchFilterKey, } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" @@ -45,10 +45,7 @@ export async function searchView( // Carry over filters for unused fields Object.keys(body.query).forEach(key => { - const operator = key as keyof Omit< - SearchFilters, - "allOr" | "onEmptyFilter" - > + const operator = key as SearchFilterKey Object.keys(body.query[operator] || {}).forEach(field => { if (!existingFields.includes(db.removeKeyNumbering(field))) { query[operator]![field] = body.query[operator]![field] diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index e59a4980e3..62c5054c7a 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -64,6 +64,11 @@ export interface SearchFilters { } } +export type SearchFilterKey = keyof Omit< + SearchFilters, + "allOr" | "onEmptyFilter" | "fuzzyOr" +> + export type SearchQueryFields = Omit export interface SortJson { From e90aa582428ec478954d6e29c6e065f23b92f2c8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 22 May 2024 16:09:19 +0100 Subject: [PATCH 12/33] Fixing REST test failure (?). --- packages/server/src/integrations/tests/rest.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/integrations/tests/rest.spec.ts b/packages/server/src/integrations/tests/rest.spec.ts index 144aefa576..ac5fa33148 100644 --- a/packages/server/src/integrations/tests/rest.spec.ts +++ b/packages/server/src/integrations/tests/rest.spec.ts @@ -655,6 +655,7 @@ describe("REST Integration", () => { }), get: (header: any) => { if (header === "content-type") return contentType + if (header === "content-length") return responseData.byteLength if (header === "content-disposition") return `attachment; filename="${filename}"` }, @@ -700,6 +701,7 @@ describe("REST Integration", () => { }), get: (header: any) => { if (header === "content-type") return contentType + if (header === "content-length") return responseData.byteLength if (header === "content-disposition") // eslint-disable-next-line no-useless-escape return `attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf` From 782bc55842b2937d9cb150f06b69d6e2effa7e5d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 22 May 2024 17:04:09 +0100 Subject: [PATCH 13/33] Updating pro. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 06e908dc1f..f50401c8c7 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 06e908dc1fd8a3d03eca5a0e92894612770cfbe8 +Subproject commit f50401c8c7ef3ae6156748cfc94bc9241dc79fa8 From 0ebb55eea0f5f5703232f90e0d5899eae65222cc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 22 May 2024 17:10:27 +0100 Subject: [PATCH 14/33] Updating pro (again). --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f50401c8c7..35489bafaa 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f50401c8c7ef3ae6156748cfc94bc9241dc79fa8 +Subproject commit 35489bafaa7b65702b0d342594ab2e5f5f2b187c From 77d049722f69d59893d1a08c9ac62775fd028c1a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 22 May 2024 17:37:56 +0100 Subject: [PATCH 15/33] Updating pro and updating test cases to try both lucene and SQS. --- packages/pro | 2 +- .../worker/src/api/routes/global/tests/auditLogs.spec.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/pro b/packages/pro index 35489bafaa..78b08caab8 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 35489bafaa7b65702b0d342594ab2e5f5f2b187c +Subproject commit 78b08caab8d83e03c5a4dde07e2e823dbb5d184f diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index 19e3cd64b4..92a366bf4e 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -1,6 +1,7 @@ import { mocks, structures } from "@budibase/backend-core/tests" import { context, events } from "@budibase/backend-core" import { Event, IdentityType } from "@budibase/types" +import { auditLogs } from "@budibase/pro" import { TestConfiguration } from "../../../../tests" mocks.licenses.useAuditLogs() @@ -12,10 +13,13 @@ const BASE_IDENTITY = { const USER_AUDIT_LOG_COUNT = 3 const APP_ID = "app_1" -describe("/api/global/auditlogs", () => { +describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { const config = new TestConfiguration() beforeAll(async () => { + if (method === "sql") { + auditLogs.useSQLSearch() + } await config.beforeAll() }) From 99536c2d0b3c6a4fcf39c0bd78ff35c1e987849b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 May 2024 14:24:49 +0100 Subject: [PATCH 16/33] PR comments. --- packages/backend-core/src/sql/utils.ts | 2 +- scripts/devDocker.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index a8b295d879..2d9b289417 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -8,7 +8,7 @@ const ROW_ID_REGEX = /^\[.*]$/g const ENCODED_SPACE = encodeURIComponent(" ") export function isExternalTableID(tableId: string) { - return tableId.includes(DocumentType.DATASOURCE) + return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR) } export function isInternalTableID(tableId: string) { diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh index e5981b7187..8c9d823fc3 100755 --- a/scripts/devDocker.sh +++ b/scripts/devDocker.sh @@ -7,6 +7,6 @@ if [ ! -d "./packages/pro/src" ]; then fi yarn build --scope @budibase/server --scope @budibase/worker -DOCKER_BUILDKIT=1 docker compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0 +docker compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0 From 90d646facb9b7bf5cf29eff6354863d40ca3b039 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 May 2024 14:37:41 +0100 Subject: [PATCH 17/33] Updating pagination method. --- packages/pro | 2 +- packages/server/src/sdk/app/rows/search/sqs.ts | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/pro b/packages/pro index 78b08caab8..58338686c6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 78b08caab8d83e03c5a4dde07e2e823dbb5d184f +Subproject commit 58338686c65024eb4140bb53965b618d9d647ec0 diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 44d971cc1a..f8dbc859a1 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -177,7 +177,7 @@ export async function search( const limit = params.limit if (paginate && params.limit) { request.paginate = { - limit: params.limit, + limit: params.limit + 1, page: bookmark, } } @@ -195,6 +195,12 @@ export async function search( } ) + // check for pagination final row + let nextRow: Row | undefined + if (paginate && params.limit && processed.length > params.limit) { + nextRow = processed.pop() + } + // get the rows let finalRows = await outputProcessing(table, processed, { preserveLinks: true, @@ -217,10 +223,7 @@ export async function search( limit: 1, page: bookmark * prevLimit + 1, } - // check if there is another row - const nextRow = await runSqlQuery(request, allTables) - // check if there is a row found - const hasNextPage = Array.isArray(nextRow) && nextRow.length >= 1 + const hasNextPage = !!nextRow response.hasNextPage = hasNextPage if (hasNextPage) { response.bookmark = bookmark + 1 From 0c28d05d4050a10a9e9bbcbd49e6f94dc7aa0132 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 May 2024 14:57:38 +0100 Subject: [PATCH 18/33] Some work to correctly handle JSON columns from SQS as well. --- packages/backend-core/src/sql/sql.ts | 14 +++++++------- packages/pro | 2 +- packages/server/src/sdk/app/rows/search/sqs.ts | 17 ++++++++--------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index ba441cc67e..cf35b1d968 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -768,11 +768,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return results.length ? results : [{ [operation.toLowerCase()]: true }] } - convertJsonStringColumns( + convertJsonStringColumns>( table: Table, - results: Record[], + results: T[], aliases?: Record - ): Record[] { + ): T[] { const tableName = getTableName(table) for (const [name, field] of Object.entries(table.schema)) { if (!this._isJsonColumn(field)) { @@ -781,11 +781,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { const aliasedTableName = (tableName && aliases?.[tableName]) || tableName const fullName = `${aliasedTableName}.${name}` for (let row of results) { - if (typeof row[fullName] === "string") { - row[fullName] = JSON.parse(row[fullName]) + if (typeof row[fullName as keyof T] === "string") { + row[fullName as keyof T] = JSON.parse(row[fullName]) } - if (typeof row[name] === "string") { - row[name] = JSON.parse(row[name]) + if (typeof row[name as keyof T] === "string") { + row[name as keyof T] = JSON.parse(row[name]) } } } diff --git a/packages/pro b/packages/pro index 58338686c6..2b023157c5 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 58338686c65024eb4140bb53965b618d9d647ec0 +Subproject commit 2b023157c53adf97bb4f8d4df61cf07c4ad13b07 diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index f8dbc859a1..08d78656af 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -29,6 +29,8 @@ import AliasTables from "../sqlAlias" import { outputProcessing } from "../../../../utilities/rowProcessor" import pick from "lodash/pick" +const builder = new sql.Sql(SqlClient.SQL_LITE) + function buildInternalFieldList( table: Table, tables: Table[], @@ -99,7 +101,6 @@ function buildTableMap(tables: Table[]) { } async function runSqlQuery(json: QueryJson, tables: Table[]) { - const builder = new sql.Sql(SqlClient.SQL_LITE) const alias = new AliasTables(tables.map(table => table.name)) return await alias.queryWithAliasing(json, async json => { const query = builder._query(json, { @@ -184,15 +185,13 @@ export async function search( try { const rows = await runSqlQuery(request, allTables) - // process from the format of tableId.column to expected format - const processed = await sqlOutputProcessing( - rows, - table!, - allTablesMap, - relationships, - { + // process from the format of tableId.column to expected format also + // make sure JSON columns corrected + const processed = builder.convertJsonStringColumns( + table, + await sqlOutputProcessing(rows, table!, allTablesMap, relationships, { sqs: true, - } + }) ) // check for pagination final row From 66686fd970ed8cf0f8f74eebc803dc423ab856d3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 May 2024 15:00:13 +0100 Subject: [PATCH 19/33] Making sure all JSON column types are correctly handled. --- packages/backend-core/src/sql/sql.ts | 6 +++--- packages/types/src/documents/app/row.ts | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index cf35b1d968..dd08730f7c 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -20,6 +20,7 @@ import { INTERNAL_TABLE_SOURCE_ID, SqlClient, QueryOptions, + JsonTypes, } from "@budibase/types" import environment from "../environment" import { helpers } from "@budibase/shared-core" @@ -796,9 +797,8 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { field: FieldSchema ): field is JsonFieldMetadata | BBReferenceFieldMetadata { return ( - field.type === FieldType.JSON || - (field.type === FieldType.BB_REFERENCE && - !helpers.schema.isDeprecatedSingleUserColumn(field)) + JsonTypes.includes(field.type) && + !helpers.schema.isDeprecatedSingleUserColumn(field) ) } diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index 27d7df09fd..8be4646e64 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -118,6 +118,15 @@ export enum FieldType { BB_REFERENCE_SINGLE = "bb_reference_single", } +export const JsonTypes = [ + FieldType.ATTACHMENT_SINGLE, + FieldType.BB_REFERENCE_SINGLE, + FieldType.ATTACHMENTS, + FieldType.BB_REFERENCE, + FieldType.JSON, + FieldType.ARRAY, +] + export interface RowAttachment { size: number name: string From 2d601d64a364073316ae00b5180f9178fca1b2bd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 May 2024 16:32:43 +0100 Subject: [PATCH 20/33] Test fix and pro reference update. --- packages/pro | 2 +- packages/types/src/documents/app/row.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pro b/packages/pro index 2b023157c5..897c6d7f01 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 2b023157c53adf97bb4f8d4df61cf07c4ad13b07 +Subproject commit 897c6d7f01a473566455f2ce62054fbfe0c86b87 diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index 8be4646e64..620e5373d8 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -120,8 +120,8 @@ export enum FieldType { export const JsonTypes = [ FieldType.ATTACHMENT_SINGLE, - FieldType.BB_REFERENCE_SINGLE, FieldType.ATTACHMENTS, + // only BB_REFERENCE is JSON, it's an array, BB_REFERENCE_SINGLE is a string type FieldType.BB_REFERENCE, FieldType.JSON, FieldType.ARRAY, From cf89c6fbdefd42603fabc6c7de2bca878bcaaa38 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 24 May 2024 14:19:53 +0100 Subject: [PATCH 21/33] Updating filter settings - getting fields correctly. --- .../design/settings/controls/SearchFieldSelect.svelte | 7 ++----- .../frontend-core/src/components/FilterBuilder.svelte | 9 +++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/SearchFieldSelect.svelte b/packages/builder/src/components/design/settings/controls/SearchFieldSelect.svelte index 7c19e5992e..be2a1c5b75 100644 --- a/packages/builder/src/components/design/settings/controls/SearchFieldSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/SearchFieldSelect.svelte @@ -1,7 +1,7 @@