From 831c1743625f6c416808109518c1611033c10fe0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Mar 2024 15:25:37 +0000 Subject: [PATCH] Give SQL integrations their own database when fetching a new datasource. --- .../core/utilities/structures/generator.ts | 1 + packages/server/scripts/test.sh | 2 + .../routes/tests/queries/generic-sql.spec.ts | 66 ++++++------------- .../server/src/integration-test/mysql.spec.ts | 11 +++- .../src/integrations/tests/utils/index.ts | 39 +++++++---- .../src/integrations/tests/utils/mariadb.ts | 27 +++++--- .../src/integrations/tests/utils/mongodb.ts | 2 +- .../src/integrations/tests/utils/mssql.ts | 30 ++++++++- .../src/integrations/tests/utils/mysql.ts | 28 +++++++- .../src/integrations/tests/utils/postgres.ts | 30 ++++++++- 10 files changed, 162 insertions(+), 74 deletions(-) diff --git a/packages/backend-core/tests/core/utilities/structures/generator.ts b/packages/backend-core/tests/core/utilities/structures/generator.ts index 64eb5ecc97..2a7eba6bbe 100644 --- a/packages/backend-core/tests/core/utilities/structures/generator.ts +++ b/packages/backend-core/tests/core/utilities/structures/generator.ts @@ -1,3 +1,4 @@ import Chance from "./Chance" export const generator = new Chance() + diff --git a/packages/server/scripts/test.sh b/packages/server/scripts/test.sh index 3ecf8bb794..c9f063c409 100644 --- a/packages/server/scripts/test.sh +++ b/packages/server/scripts/test.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +export DEBUG=testcontainers* + if [[ -n $CI ]] then export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS" diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index d393430060..ff77d3dc52 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -3,6 +3,7 @@ import * as setup from "../utilities" import { DatabaseName, getDatasource, + rawQuery, } from "../../../../integrations/tests/utils" import pg from "pg" import mysql from "mysql2/promise" @@ -46,6 +47,7 @@ describe.each( ].map(name => [name, getDatasource(name)]) )("queries (%s)", (dbName, dsProvider) => { const config = setup.getConfig() + let rawDatasource: Datasource let datasource: Datasource async function createQuery(query: Partial): Promise { @@ -62,56 +64,19 @@ describe.each( return await config.api.query.save({ ...defaultQuery, ...query }) } - async function rawQuery(sql: string): Promise { - // We re-fetch the datasource here because the one returned by - // config.api.datasource.create has the password field blanked out, and we - // need the password to connect to the database. - const ds = await dsProvider - switch (ds.source) { - case SourceName.POSTGRES: { - const client = new pg.Client(ds.config!) - await client.connect() - try { - const { rows } = await client.query(sql) - return rows - } finally { - await client.end() - } - } - case SourceName.MYSQL: { - const con = await mysql.createConnection(ds.config!) - try { - const [rows] = await con.query(sql) - return rows - } finally { - con.end() - } - } - case SourceName.SQL_SERVER: { - const pool = new mssql.ConnectionPool(ds.config! as mssql.config) - const client = await pool.connect() - try { - const { recordset } = await client.query(sql) - return recordset - } finally { - await pool.close() - } - } - } - } - beforeAll(async () => { await config.init() - datasource = await config.api.datasource.create(await dsProvider) + rawDatasource = await dsProvider + datasource = await config.api.datasource.create(rawDatasource) }) beforeEach(async () => { - await rawQuery(createTableSQL[datasource.source]) - await rawQuery(insertSQL) + await rawQuery(rawDatasource, createTableSQL[datasource.source]) + await rawQuery(rawDatasource, insertSQL) }) afterEach(async () => { - await rawQuery(dropTableSQL) + await rawQuery(rawDatasource, dropTableSQL) }) afterAll(async () => { @@ -145,7 +110,10 @@ describe.each( }, ]) - const rows = await rawQuery("SELECT * FROM test_table WHERE name = 'baz'") + const rows = await rawQuery( + rawDatasource, + "SELECT * FROM test_table WHERE name = 'baz'" + ) expect(rows).toHaveLength(1) }) @@ -173,6 +141,7 @@ describe.each( expect(result.data).toEqual([{ created: true }]) const rows = await rawQuery( + rawDatasource, `SELECT * FROM test_table WHERE birthday = '${date.toISOString()}'` ) expect(rows).toHaveLength(1) @@ -204,6 +173,7 @@ describe.each( expect(result.data).toEqual([{ created: true }]) const rows = await rawQuery( + rawDatasource, `SELECT * FROM test_table WHERE name = '${notDateStr}'` ) expect(rows).toHaveLength(1) @@ -340,7 +310,10 @@ describe.each( }, ]) - const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1") + const rows = await rawQuery( + rawDatasource, + "SELECT * FROM test_table WHERE id = 1" + ) expect(rows).toEqual([ { id: 1, name: "foo", birthday: null, number: null }, ]) @@ -408,7 +381,10 @@ describe.each( }, ]) - const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1") + const rows = await rawQuery( + rawDatasource, + "SELECT * FROM test_table WHERE id = 1" + ) expect(rows).toHaveLength(0) }) }) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 65fbe2949d..fb2d3c5285 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -18,6 +18,13 @@ import { generator } from "@budibase/backend-core/tests" // @ts-ignore fetch.mockSearch() +function uniqueTableName(length?: number): string { + return generator + .guid() + .replaceAll("-", "_") + .substring(0, length || 10) +} + const config = setup.getConfig()! jest.mock("../websockets", () => ({ @@ -53,7 +60,7 @@ describe("mysql integrations", () => { beforeEach(async () => { primaryMySqlTable = await config.createTable({ - name: generator.guid().replaceAll("-", "_").substring(0, 10), + name: uniqueTableName(), type: "table", primary: ["id"], schema: { @@ -249,7 +256,7 @@ describe("mysql integrations", () => { const addColumnToTable: TableRequest = { type: "table", sourceType: TableSourceType.EXTERNAL, - name: generator.guid().replaceAll("-", "_").substring(0, 10), + name: uniqueTableName(), sourceId: mysqlDatasource._id!, primary: ["id"], schema: { diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 57aae02865..5760273d51 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -1,11 +1,11 @@ jest.unmock("pg") -import { Datasource } from "@budibase/types" -import { postgres } from "./postgres" -import { mongodb } from "./mongodb" -import { mysql } from "./mysql" -import { mssql } from "./mssql" -import { mariadb } from "./mariadb" +import { Datasource, SourceName } from "@budibase/types" +import * as postgres from "./postgres" +import * as mongodb from "./mongodb" +import * as mysql from "./mysql" +import * as mssql from "./mssql" +import * as mariadb from "./mariadb" export type DatasourceProvider = () => Promise @@ -18,11 +18,11 @@ export enum DatabaseName { } const providers: Record = { - [DatabaseName.POSTGRES]: postgres, - [DatabaseName.MONGODB]: mongodb, - [DatabaseName.MYSQL]: mysql, - [DatabaseName.SQL_SERVER]: mssql, - [DatabaseName.MARIADB]: mariadb, + [DatabaseName.POSTGRES]: postgres.getDatasource, + [DatabaseName.MONGODB]: mongodb.getDatasource, + [DatabaseName.MYSQL]: mysql.getDatasource, + [DatabaseName.SQL_SERVER]: mssql.getDatasource, + [DatabaseName.MARIADB]: mariadb.getDatasource, } export function getDatasourceProviders( @@ -46,3 +46,20 @@ export async function getDatasources( ): Promise { return Promise.all(sourceNames.map(sourceName => providers[sourceName]())) } + +export async function rawQuery(ds: Datasource, sql: string): Promise { + switch (ds.source) { + case SourceName.POSTGRES: { + return postgres.rawQuery(ds, sql) + } + case SourceName.MYSQL: { + return mysql.rawQuery(ds, sql) + } + case SourceName.SQL_SERVER: { + return mssql.rawQuery(ds, sql) + } + default: { + throw new Error(`Unsupported source: ${ds.source}`) + } + } +} diff --git a/packages/server/src/integrations/tests/utils/mariadb.ts b/packages/server/src/integrations/tests/utils/mariadb.ts index a10c36f9ff..c8890af1fb 100644 --- a/packages/server/src/integrations/tests/utils/mariadb.ts +++ b/packages/server/src/integrations/tests/utils/mariadb.ts @@ -1,6 +1,8 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" +import { rawQuery } from "./mysql" +import { generator } from "@budibase/backend-core/tests" class MariaDBWaitStrategy extends AbstractWaitStrategy { async waitUntilReady(container: any, boundPorts: any, startTime?: Date) { @@ -19,7 +21,7 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy { } } -export async function mariadb(): Promise { +export async function getDatasource(): Promise { const container = await new GenericContainer("mariadb:lts") .withName("budibase-test-mariadb") .withReuse() @@ -31,16 +33,23 @@ export async function mariadb(): Promise { const host = container.getHost() const port = container.getMappedPort(3306) - return { + const config = { + host, + port, + user: "root", + password: "password", + database: "mysql", + } + + const datasource = { type: "datasource_plus", source: SourceName.MYSQL, plus: true, - config: { - host, - port, - user: "root", - password: "password", - database: "mysql", - }, + config, } + + const database = generator.guid().replaceAll("-", "") + await rawQuery(datasource, `CREATE DATABASE \`${database}\``) + datasource.config.database = database + return datasource } diff --git a/packages/server/src/integrations/tests/utils/mongodb.ts b/packages/server/src/integrations/tests/utils/mongodb.ts index ff24bbc62e..6ab5b11191 100644 --- a/packages/server/src/integrations/tests/utils/mongodb.ts +++ b/packages/server/src/integrations/tests/utils/mongodb.ts @@ -1,7 +1,7 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" -export async function mongodb(): Promise { +export async function getDatasource(): Promise { const container = await new GenericContainer("mongo:7.0-jammy") .withName("budibase-test-mongodb") .withReuse() diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index 0f4e290526..c0875b84db 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -1,7 +1,9 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" +import mssql from "mssql" +import { generator } from "@budibase/backend-core/tests" -export async function mssql(): Promise { +export async function getDatasource(): Promise { const container = await new GenericContainer( "mcr.microsoft.com/mssql/server:2022-latest" ) @@ -27,7 +29,7 @@ export async function mssql(): Promise { const host = container.getHost() const port = container.getMappedPort(1433) - return { + const datasource: Datasource = { type: "datasource_plus", source: SourceName.SQL_SERVER, plus: true, @@ -41,4 +43,28 @@ export async function mssql(): Promise { }, }, } + + const database = generator.guid().replaceAll("-", "") + await rawQuery(datasource, `CREATE DATABASE "${database}"`) + datasource.config!.database = database + + return datasource +} + +export async function rawQuery(ds: Datasource, sql: string) { + if (!ds.config) { + throw new Error("Datasource config is missing") + } + if (ds.source !== SourceName.SQL_SERVER) { + throw new Error("Datasource source is not SQL Server") + } + + const pool = new mssql.ConnectionPool(ds.config! as mssql.config) + const client = await pool.connect() + try { + const { recordset } = await client.query(sql) + return recordset + } finally { + await pool.close() + } } diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts index 665d6f0ecf..9fa8b0bd86 100644 --- a/packages/server/src/integrations/tests/utils/mysql.ts +++ b/packages/server/src/integrations/tests/utils/mysql.ts @@ -1,6 +1,8 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" +import mysql from "mysql2/promise" +import { generator } from "@budibase/backend-core/tests" class MySQLWaitStrategy extends AbstractWaitStrategy { async waitUntilReady(container: any, boundPorts: any, startTime?: Date) { @@ -22,7 +24,7 @@ class MySQLWaitStrategy extends AbstractWaitStrategy { } } -export async function mysql(): Promise { +export async function getDatasource(): Promise { const container = await new GenericContainer("mysql:8.3") .withName("budibase-test-mysql") .withReuse() @@ -33,7 +35,7 @@ export async function mysql(): Promise { const host = container.getHost() const port = container.getMappedPort(3306) - return { + const datasource: Datasource = { type: "datasource_plus", source: SourceName.MYSQL, plus: true, @@ -45,4 +47,26 @@ export async function mysql(): Promise { database: "mysql", }, } + + const database = generator.guid().replaceAll("-", "") + await rawQuery(datasource, `CREATE DATABASE \`${database}\``) + datasource.config!.database = database + return datasource +} + +export async function rawQuery(ds: Datasource, sql: string) { + if (!ds.config) { + throw new Error("Datasource config is missing") + } + if (ds.source !== SourceName.MYSQL) { + throw new Error("Datasource source is not MySQL") + } + + const connection = await mysql.createConnection(ds.config) + try { + const [rows] = await connection.query(sql) + return rows + } finally { + connection.end() + } } diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index 896c7ea3e0..b10dfe44cf 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -1,7 +1,9 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" +import pg from "pg" +import { generator } from "@budibase/backend-core/tests" -export async function postgres(): Promise { +export async function getDatasource(): Promise { const container = await new GenericContainer("postgres:16.1-bullseye") .withName("budibase-test-postgres") .withReuse() @@ -16,7 +18,7 @@ export async function postgres(): Promise { const host = container.getHost() const port = container.getMappedPort(5432) - return { + const datasource: Datasource = { type: "datasource_plus", source: SourceName.POSTGRES, plus: true, @@ -32,4 +34,28 @@ export async function postgres(): Promise { ca: false, }, } + + const database = generator.guid().replaceAll("-", "") + await rawQuery(datasource, `CREATE DATABASE "${database}"`) + datasource.config!.database = database + + return datasource +} + +export async function rawQuery(ds: Datasource, sql: string) { + if (!ds.config) { + throw new Error("Datasource config is missing") + } + if (ds.source !== SourceName.POSTGRES) { + throw new Error("Datasource source is not Postgres") + } + + const client = new pg.Client(ds.config) + await client.connect() + try { + const { rows } = await client.query(sql) + return rows + } finally { + await client.end() + } }