From 464f973f122154c2c3169abbd4484417b53c355a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 12:01:47 +0100 Subject: [PATCH] Adding a separation for MariaDB and MySQL, mariaDB is the core of the problem, this solves for it by separating them and allowing us to use the special json_arrayagg for mariaDB, but use a correlated sub-query for MySQL. --- packages/backend-core/src/sql/sql.ts | 20 +++++++++++++------ packages/backend-core/src/sql/sqlTable.ts | 13 +++++++++++- .../src/api/routes/tests/search.spec.ts | 14 ++++++------- packages/server/src/integrations/mysql.ts | 10 ++++++++++ packages/types/src/sdk/search.ts | 1 + 5 files changed, 44 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index db4ddf180b..4bdec363a4 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -150,6 +150,7 @@ class InternalBuilder { return `"${str}"` case SqlClient.MS_SQL: return `[${str}]` + case SqlClient.MARIADB: case SqlClient.MY_SQL: return `\`${str}\`` } @@ -559,7 +560,10 @@ class InternalBuilder { )}${wrap}, FALSE)` ) }) - } else if (this.client === SqlClient.MY_SQL) { + } else if ( + this.client === SqlClient.MY_SQL || + this.client === SqlClient.MARIADB + ) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" iterate(mode, (q, key, value) => { return q[rawFnc]( @@ -1007,7 +1011,7 @@ class InternalBuilder { `json_agg(json_build_object(${fieldList}))` ) break - case SqlClient.MY_SQL: + case SqlClient.MARIADB: // can't use the standard wrap due to correlated sub-query limitations in MariaDB wrapperQuery = subQuery.select( knex.raw( @@ -1015,6 +1019,7 @@ class InternalBuilder { ) ) break + case SqlClient.MY_SQL: case SqlClient.ORACLE: wrapperQuery = standardWrap( `json_arrayagg(json_object(${fieldList}))` @@ -1181,7 +1186,8 @@ class InternalBuilder { if ( this.client === SqlClient.POSTGRES || this.client === SqlClient.SQL_LITE || - this.client === SqlClient.MY_SQL + this.client === SqlClient.MY_SQL || + this.client === SqlClient.MARIADB ) { const primary = this.table.primary if (!primary) { @@ -1328,12 +1334,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] { const sqlClient = this.getSqlClient() const config: Knex.Config = { - client: sqlClient, + client: this.getBaseSqlClient(), } if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) { config.useNullAsDefault = true } - const client = knex(config) let query: Knex.QueryBuilder const builder = new InternalBuilder(sqlClient, client, json) @@ -1442,7 +1447,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { let id if (sqlClient === SqlClient.MS_SQL) { id = results?.[0].id - } else if (sqlClient === SqlClient.MY_SQL) { + } else if ( + sqlClient === SqlClient.MY_SQL || + sqlClient === SqlClient.MARIADB + ) { id = results?.insertId } row = processFn( diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 35d7978449..f5b02cc4e4 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -210,16 +210,27 @@ function buildDeleteTable(knex: SchemaBuilder, table: Table): SchemaBuilder { class SqlTableQueryBuilder { private readonly sqlClient: SqlClient + private extendedSqlClient: SqlClient | undefined // pass through client to get flavour of SQL constructor(client: SqlClient) { this.sqlClient = client } - getSqlClient(): SqlClient { + getBaseSqlClient(): SqlClient { return this.sqlClient } + getSqlClient(): SqlClient { + return this.extendedSqlClient || this.sqlClient + } + + // if working in a database like MySQL with many variants (MariaDB) + // we can set another client which overrides the base one + setExtendedSqlClient(client: SqlClient) { + this.extendedSqlClient = client + } + /** * @param json the input JSON structure from which an SQL query will be built. * @return the operation that was found in the JSON. diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 706151ba2c..c770c4e460 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - // ["in-memory", undefined], - // ["lucene", undefined], - // ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + ["in-memory", undefined], + ["lucene", undefined], + ["sqs", undefined], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index f5b575adb8..8b1ada4184 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -241,6 +241,16 @@ class MySQLIntegration extends Sql implements DatasourcePlus { async connect() { this.client = await mysql.createConnection(this.config) + const res = await this.internalQuery( + { + sql: "SELECT VERSION();", + }, + { connect: false } + ) + const version = res?.[0]?.["VERSION()"] + if (version?.toLowerCase().includes("mariadb")) { + this.setExtendedSqlClient(SqlClient.MARIADB) + } } async disconnect() { diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 7d61aebdfb..7c691beb71 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -195,6 +195,7 @@ export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", MY_SQL = "mysql2", + MARIADB = "mariadb", ORACLE = "oracledb", SQL_LITE = "sqlite3", }