Give SQL integrations their own database when fetching a new datasource.

packages/backend-core/tests/core/utilities/structures/generator.ts

@@ -1,3 +1,4 @@
 import Chance from "./Chance"
 
 export const generator = new Chance()
+

packages/server/scripts/test.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+export DEBUG=testcontainers*
+
 if [[ -n $CI ]]
 then
   export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS"

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -3,6 +3,7 @@ import * as setup from "../utilities"
 import {
   DatabaseName,
   getDatasource,
+  rawQuery,
 } from "../../../../integrations/tests/utils"
 import pg from "pg"
 import mysql from "mysql2/promise"
@@ -46,6 +47,7 @@ describe.each(
   ].map(name => [name, getDatasource(name)])
 )("queries (%s)", (dbName, dsProvider) => {
   const config = setup.getConfig()
+  let rawDatasource: Datasource
   let datasource: Datasource
 
   async function createQuery(query: Partial<Query>): Promise<Query> {
@@ -62,56 +64,19 @@ describe.each(
     return await config.api.query.save({ ...defaultQuery, ...query })
   }
 
-  async function rawQuery(sql: string): Promise<any> {
-    // We re-fetch the datasource here because the one returned by
-    // config.api.datasource.create has the password field blanked out, and we
-    // need the password to connect to the database.
-    const ds = await dsProvider
-    switch (ds.source) {
-      case SourceName.POSTGRES: {
-        const client = new pg.Client(ds.config!)
-        await client.connect()
-        try {
-          const { rows } = await client.query(sql)
-          return rows
-        } finally {
-          await client.end()
-        }
-      }
-      case SourceName.MYSQL: {
-        const con = await mysql.createConnection(ds.config!)
-        try {
-          const [rows] = await con.query(sql)
-          return rows
-        } finally {
-          con.end()
-        }
-      }
-      case SourceName.SQL_SERVER: {
-        const pool = new mssql.ConnectionPool(ds.config! as mssql.config)
-        const client = await pool.connect()
-        try {
-          const { recordset } = await client.query(sql)
-          return recordset
-        } finally {
-          await pool.close()
-        }
-      }
-    }
-  }
-
   beforeAll(async () => {
     await config.init()
-    datasource = await config.api.datasource.create(await dsProvider)
+    rawDatasource = await dsProvider
+    datasource = await config.api.datasource.create(rawDatasource)
   })
 
   beforeEach(async () => {
-    await rawQuery(createTableSQL[datasource.source])
+    await rawQuery(rawDatasource, createTableSQL[datasource.source])
-    await rawQuery(insertSQL)
+    await rawQuery(rawDatasource, insertSQL)
   })
 
   afterEach(async () => {
-    await rawQuery(dropTableSQL)
+    await rawQuery(rawDatasource, dropTableSQL)
   })
 
   afterAll(async () => {
@@ -145,7 +110,10 @@ describe.each(
         },
       ])
 
-      const rows = await rawQuery("SELECT * FROM test_table WHERE name = 'baz'")
+      const rows = await rawQuery(
+        rawDatasource,
+        "SELECT * FROM test_table WHERE name = 'baz'"
+      )
       expect(rows).toHaveLength(1)
     })
 
@@ -173,6 +141,7 @@ describe.each(
         expect(result.data).toEqual([{ created: true }])
 
         const rows = await rawQuery(
+          rawDatasource,
           `SELECT * FROM test_table WHERE birthday = '${date.toISOString()}'`
         )
         expect(rows).toHaveLength(1)
@@ -204,6 +173,7 @@ describe.each(
         expect(result.data).toEqual([{ created: true }])
 
         const rows = await rawQuery(
+          rawDatasource,
           `SELECT * FROM test_table WHERE name = '${notDateStr}'`
         )
         expect(rows).toHaveLength(1)
@@ -340,7 +310,10 @@ describe.each(
         },
       ])
 
-      const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
+      const rows = await rawQuery(
+        rawDatasource,
+        "SELECT * FROM test_table WHERE id = 1"
+      )
       expect(rows).toEqual([
         { id: 1, name: "foo", birthday: null, number: null },
       ])
@@ -408,7 +381,10 @@ describe.each(
         },
       ])
 
-      const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
+      const rows = await rawQuery(
+        rawDatasource,
+        "SELECT * FROM test_table WHERE id = 1"
+      )
       expect(rows).toHaveLength(0)
     })
   })

packages/server/src/integration-test/mysql.spec.ts

@@ -18,6 +18,13 @@ import { generator } from "@budibase/backend-core/tests"
 // @ts-ignore
 fetch.mockSearch()
 
+function uniqueTableName(length?: number): string {
+  return generator
+    .guid()
+    .replaceAll("-", "_")
+    .substring(0, length || 10)
+}
+
 const config = setup.getConfig()!
 
 jest.mock("../websockets", () => ({
@@ -53,7 +60,7 @@ describe("mysql integrations", () => {
 
   beforeEach(async () => {
     primaryMySqlTable = await config.createTable({
-      name: generator.guid().replaceAll("-", "_").substring(0, 10),
+      name: uniqueTableName(),
       type: "table",
       primary: ["id"],
       schema: {
@@ -249,7 +256,7 @@ describe("mysql integrations", () => {
       const addColumnToTable: TableRequest = {
         type: "table",
         sourceType: TableSourceType.EXTERNAL,
-        name: generator.guid().replaceAll("-", "_").substring(0, 10),
+        name: uniqueTableName(),
         sourceId: mysqlDatasource._id!,
         primary: ["id"],
         schema: {

packages/server/src/integrations/tests/utils/index.ts

@@ -1,11 +1,11 @@
 jest.unmock("pg")
 
-import { Datasource } from "@budibase/types"
+import { Datasource, SourceName } from "@budibase/types"
-import { postgres } from "./postgres"
+import * as postgres from "./postgres"
-import { mongodb } from "./mongodb"
+import * as mongodb from "./mongodb"
-import { mysql } from "./mysql"
+import * as mysql from "./mysql"
-import { mssql } from "./mssql"
+import * as mssql from "./mssql"
-import { mariadb } from "./mariadb"
+import * as mariadb from "./mariadb"
 
 export type DatasourceProvider = () => Promise<Datasource>
 
@@ -18,11 +18,11 @@ export enum DatabaseName {
 }
 
 const providers: Record<DatabaseName, DatasourceProvider> = {
-  [DatabaseName.POSTGRES]: postgres,
+  [DatabaseName.POSTGRES]: postgres.getDatasource,
-  [DatabaseName.MONGODB]: mongodb,
+  [DatabaseName.MONGODB]: mongodb.getDatasource,
-  [DatabaseName.MYSQL]: mysql,
+  [DatabaseName.MYSQL]: mysql.getDatasource,
-  [DatabaseName.SQL_SERVER]: mssql,
+  [DatabaseName.SQL_SERVER]: mssql.getDatasource,
-  [DatabaseName.MARIADB]: mariadb,
+  [DatabaseName.MARIADB]: mariadb.getDatasource,
 }
 
 export function getDatasourceProviders(
@@ -46,3 +46,20 @@ export async function getDatasources(
 ): Promise<Datasource[]> {
   return Promise.all(sourceNames.map(sourceName => providers[sourceName]()))
 }
+
+export async function rawQuery(ds: Datasource, sql: string): Promise<any> {
+  switch (ds.source) {
+    case SourceName.POSTGRES: {
+      return postgres.rawQuery(ds, sql)
+    }
+    case SourceName.MYSQL: {
+      return mysql.rawQuery(ds, sql)
+    }
+    case SourceName.SQL_SERVER: {
+      return mssql.rawQuery(ds, sql)
+    }
+    default: {
+      throw new Error(`Unsupported source: ${ds.source}`)
+    }
+  }
+}

packages/server/src/integrations/tests/utils/mariadb.ts

@@ -1,6 +1,8 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
 import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
+import { rawQuery } from "./mysql"
+import { generator } from "@budibase/backend-core/tests"
 
 class MariaDBWaitStrategy extends AbstractWaitStrategy {
   async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
@@ -19,7 +21,7 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy {
   }
 }
 
-export async function mariadb(): Promise<Datasource> {
+export async function getDatasource(): Promise<Datasource> {
   const container = await new GenericContainer("mariadb:lts")
     .withName("budibase-test-mariadb")
     .withReuse()
@@ -31,16 +33,23 @@ export async function mariadb(): Promise<Datasource> {
   const host = container.getHost()
   const port = container.getMappedPort(3306)
 
-  return {
+  const config = {
+    host,
+    port,
+    user: "root",
+    password: "password",
+    database: "mysql",
+  }
+
+  const datasource = {
     type: "datasource_plus",
     source: SourceName.MYSQL,
     plus: true,
-    config: {
+    config,
-      host,
-      port,
-      user: "root",
-      password: "password",
-      database: "mysql",
-    },
   }
+
+  const database = generator.guid().replaceAll("-", "")
+  await rawQuery(datasource, `CREATE DATABASE \`${database}\``)
+  datasource.config.database = database
+  return datasource
 }

packages/server/src/integrations/tests/utils/mongodb.ts

@@ -1,7 +1,7 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
 
-export async function mongodb(): Promise<Datasource> {
+export async function getDatasource(): Promise<Datasource> {
   const container = await new GenericContainer("mongo:7.0-jammy")
     .withName("budibase-test-mongodb")
     .withReuse()

packages/server/src/integrations/tests/utils/mssql.ts

@@ -1,7 +1,9 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
+import mssql from "mssql"
+import { generator } from "@budibase/backend-core/tests"
 
-export async function mssql(): Promise<Datasource> {
+export async function getDatasource(): Promise<Datasource> {
   const container = await new GenericContainer(
     "mcr.microsoft.com/mssql/server:2022-latest"
   )
@@ -27,7 +29,7 @@ export async function mssql(): Promise<Datasource> {
   const host = container.getHost()
   const port = container.getMappedPort(1433)
 
-  return {
+  const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.SQL_SERVER,
     plus: true,
@@ -41,4 +43,28 @@ export async function mssql(): Promise<Datasource> {
       },
     },
   }
+
+  const database = generator.guid().replaceAll("-", "")
+  await rawQuery(datasource, `CREATE DATABASE "${database}"`)
+  datasource.config!.database = database
+
+  return datasource
+}
+
+export async function rawQuery(ds: Datasource, sql: string) {
+  if (!ds.config) {
+    throw new Error("Datasource config is missing")
+  }
+  if (ds.source !== SourceName.SQL_SERVER) {
+    throw new Error("Datasource source is not SQL Server")
+  }
+
+  const pool = new mssql.ConnectionPool(ds.config! as mssql.config)
+  const client = await pool.connect()
+  try {
+    const { recordset } = await client.query(sql)
+    return recordset
+  } finally {
+    await pool.close()
+  }
 }

packages/server/src/integrations/tests/utils/mysql.ts

@@ -1,6 +1,8 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
 import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
+import mysql from "mysql2/promise"
+import { generator } from "@budibase/backend-core/tests"
 
 class MySQLWaitStrategy extends AbstractWaitStrategy {
   async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
@@ -22,7 +24,7 @@ class MySQLWaitStrategy extends AbstractWaitStrategy {
   }
 }
 
-export async function mysql(): Promise<Datasource> {
+export async function getDatasource(): Promise<Datasource> {
   const container = await new GenericContainer("mysql:8.3")
     .withName("budibase-test-mysql")
     .withReuse()
@@ -33,7 +35,7 @@ export async function mysql(): Promise<Datasource> {
   const host = container.getHost()
   const port = container.getMappedPort(3306)
 
-  return {
+  const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.MYSQL,
     plus: true,
@@ -45,4 +47,26 @@ export async function mysql(): Promise<Datasource> {
       database: "mysql",
     },
   }
+
+  const database = generator.guid().replaceAll("-", "")
+  await rawQuery(datasource, `CREATE DATABASE \`${database}\``)
+  datasource.config!.database = database
+  return datasource
+}
+
+export async function rawQuery(ds: Datasource, sql: string) {
+  if (!ds.config) {
+    throw new Error("Datasource config is missing")
+  }
+  if (ds.source !== SourceName.MYSQL) {
+    throw new Error("Datasource source is not MySQL")
+  }
+
+  const connection = await mysql.createConnection(ds.config)
+  try {
+    const [rows] = await connection.query(sql)
+    return rows
+  } finally {
+    connection.end()
+  }
 }

packages/server/src/integrations/tests/utils/postgres.ts

@@ -1,7 +1,9 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
+import pg from "pg"
+import { generator } from "@budibase/backend-core/tests"
 
-export async function postgres(): Promise<Datasource> {
+export async function getDatasource(): Promise<Datasource> {
   const container = await new GenericContainer("postgres:16.1-bullseye")
     .withName("budibase-test-postgres")
     .withReuse()
@@ -16,7 +18,7 @@ export async function postgres(): Promise<Datasource> {
   const host = container.getHost()
   const port = container.getMappedPort(5432)
 
-  return {
+  const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.POSTGRES,
     plus: true,
@@ -32,4 +34,28 @@ export async function postgres(): Promise<Datasource> {
       ca: false,
     },
   }
+
+  const database = generator.guid().replaceAll("-", "")
+  await rawQuery(datasource, `CREATE DATABASE "${database}"`)
+  datasource.config!.database = database
+
+  return datasource
+}
+
+export async function rawQuery(ds: Datasource, sql: string) {
+  if (!ds.config) {
+    throw new Error("Datasource config is missing")
+  }
+  if (ds.source !== SourceName.POSTGRES) {
+    throw new Error("Datasource source is not Postgres")
+  }
+
+  const client = new pg.Client(ds.config)
+  await client.connect()
+  try {
+    const { rows } = await client.query(sql)
+    return rows
+  } finally {
+    await client.end()
+  }
 }