Rework how we connect to containers.

.github/workflows/budibase_ci.yml

@@ -136,7 +136,8 @@ jobs:
           fi
 
   test-server:
-    runs-on: budi-tubby-tornado-quad-core-150gb
+    runs-on:
+      group: hosted-runners
     env:
       DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
     steps:

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -1,6 +1,8 @@
 import { DatabaseImpl } from "../../../src/db"
 import { execSync } from "child_process"
 
+const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g")
+
 interface ContainerInfo {
   Command: string
   CreatedAt: string
@@ -19,7 +21,10 @@ interface ContainerInfo {
 }
 
 function getTestcontainers(): ContainerInfo[] {
-  return execSync("docker ps --format json")
+  // We use --format json to make sure the output is nice and machine-readable,
+  // and we use --no-trunc so that the command returns full container IDs so we
+  // can filter on them correctly.
+  return execSync("docker ps --format json --no-trunc")
     .toString()
     .split("\n")
     .filter(x => x.length > 0)
@@ -27,32 +32,51 @@ function getTestcontainers(): ContainerInfo[] {
     .filter(x => x.Labels.includes("org.testcontainers=true"))
 }
 
-function getContainerByImage(image: string) {
-  return getTestcontainers().find(x => x.Image.startsWith(image))
+export function getContainerByImage(image: string) {
+  const containers = getTestcontainers().filter(x => x.Image.startsWith(image))
+  if (containers.length > 1) {
+    throw new Error(`Multiple containers found with image: ${image}`)
+  }
+  return containers[0]
 }
 
-function getExposedPort(container: ContainerInfo, port: number) {
-  const match = container.Ports.match(new RegExp(`0.0.0.0:(\\d+)->${port}/tcp`))
-  if (!match) {
-    return undefined
+export function getContainerById(id: string) {
+  return getTestcontainers().find(x => x.ID === id)
+}
+
+export interface Port {
+  host: number
+  container: number
+}
+
+export function getExposedV4Ports(container: ContainerInfo): Port[] {
+  let ports: Port[] = []
+  for (const match of container.Ports.matchAll(IPV4_PORT_REGEX)) {
+    ports.push({ host: parseInt(match[1]), container: parseInt(match[2]) })
   }
-  return parseInt(match[1])
+  return ports
+}
+
+export function getExposedV4Port(container: ContainerInfo, port: number) {
+  return getExposedV4Ports(container).find(x => x.container === port)?.host
 }
 
 export function setupEnv(...envs: any[]) {
+  // We start couchdb in globalSetup.ts, in the root of the monorepo, so it
+  // should be relatively safe to look for it by its image name.
   const couch = getContainerByImage("budibase/couchdb")
   if (!couch) {
     throw new Error("CouchDB container not found")
   }
 
-  const couchPort = getExposedPort(couch, 5984)
+  const couchPort = getExposedV4Port(couch, 5984)
   if (!couchPort) {
     throw new Error("CouchDB port not found")
   }
 
   const configs = [
     { key: "COUCH_DB_PORT", value: `${couchPort}` },
-    { key: "COUCH_DB_URL", value: `http://localhost:${couchPort}` },
+    { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {
@@ -60,7 +84,4 @@ export function setupEnv(...envs: any[]) {
       env._set(config.key, config.value)
     }
   }
-
-  // @ts-expect-error
-  DatabaseImpl.nano = undefined
 }

packages/server/src/integrations/tests/utils/index.ts

@@ -6,6 +6,8 @@ import * as mongodb from "./mongodb"
 import * as mysql from "./mysql"
 import * as mssql from "./mssql"
 import * as mariadb from "./mariadb"
+import { GenericContainer } from "testcontainers"
+import { testContainerUtils } from "@budibase/backend-core/tests"
 
 export type DatasourceProvider = () => Promise<Datasource>
 
@@ -63,3 +65,26 @@ export async function rawQuery(ds: Datasource, sql: string): Promise<any> {
     }
   }
 }
+
+export async function startContainer(container: GenericContainer) {
+  if (process.env.REUSE_CONTAINERS) {
+    container = container.withReuse()
+  }
+
+  const startedContainer = await container.start()
+
+  const info = testContainerUtils.getContainerById(startedContainer.getId())
+  if (!info) {
+    throw new Error("Container not found")
+  }
+
+  // Some Docker runtimes, when you expose a port, will bind it to both
+  // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6
+  // addresses are not shared, and testcontainers will sometimes give you back
+  // the ipv6 port. There's no way to know that this has happened, and if you
+  // try to then connect to `localhost:port` you may attempt to bind to the v4
+  // address which could be unbound or even an entirely different container. For
+  // that reason, we don't use testcontainers' `getExposedPort` function,
+  // preferring instead our own method that guaranteed v4 ports.
+  return testContainerUtils.getExposedV4Ports(info)
+}

packages/server/src/integrations/tests/utils/mariadb.ts

@@ -2,7 +2,10 @@ import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
 import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
 import { rawQuery } from "./mysql"
-import { generator } from "@budibase/backend-core/tests"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+
+let ports: Promise<testContainerUtils.Port[]>
 
 class MariaDBWaitStrategy extends AbstractWaitStrategy {
   async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
@@ -22,22 +25,22 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy {
 }
 
 export async function getDatasource(): Promise<Datasource> {
-  let container = new GenericContainer("mariadb:lts")
-    .withExposedPorts(3306)
-    .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" })
-    .withWaitStrategy(new MariaDBWaitStrategy())
-
-  if (process.env.REUSE_CONTAINERS) {
-    container = container.withReuse()
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("mariadb:lts")
+        .withExposedPorts(3306)
+        .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" })
+        .withWaitStrategy(new MariaDBWaitStrategy())
+    )
   }
 
-  const startedContainer = await container.start()
-
-  const host = startedContainer.getHost()
-  const port = startedContainer.getMappedPort(3306)
+  const port = (await ports).find(x => x.container === 3306)?.host
+  if (!port) {
+    throw new Error("MariaDB port not found")
+  }
 
   const config = {
-    host,
+    host: "127.0.0.1",
     port,
     user: "root",
     password: "password",

packages/server/src/integrations/tests/utils/mongodb.ts

@@ -1,34 +1,38 @@
+import { testContainerUtils } from "@budibase/backend-core/tests"
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
+import { startContainer } from "."
+
+let ports: Promise<testContainerUtils.Port[]>
 
 export async function getDatasource(): Promise<Datasource> {
-  let container = new GenericContainer("mongo:7.0-jammy")
-    .withExposedPorts(27017)
-    .withEnvironment({
-      MONGO_INITDB_ROOT_USERNAME: "mongo",
-      MONGO_INITDB_ROOT_PASSWORD: "password",
-    })
-    .withWaitStrategy(
-      Wait.forSuccessfulCommand(
-        `mongosh --eval "db.version()"`
-      ).withStartupTimeout(10000)
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("mongo:7.0-jammy")
+        .withExposedPorts(27017)
+        .withEnvironment({
+          MONGO_INITDB_ROOT_USERNAME: "mongo",
+          MONGO_INITDB_ROOT_PASSWORD: "password",
+        })
+        .withWaitStrategy(
+          Wait.forSuccessfulCommand(
+            `mongosh --eval "db.version()"`
+          ).withStartupTimeout(10000)
+        )
     )
-
-  if (process.env.REUSE_CONTAINERS) {
-    container = container.withReuse()
   }
 
-  const startedContainer = await container.start()
-
-  const host = startedContainer.getHost()
-  const port = startedContainer.getMappedPort(27017)
+  const port = (await ports).find(x => x.container === 27017)
+  if (!port) {
+    throw new Error("MongoDB port not found")
+  }
 
   return {
     type: "datasource",
     source: SourceName.MONGODB,
     plus: false,
     config: {
-      connectionString: `mongodb://mongo:password@${host}:${port}`,
+      connectionString: `mongodb://mongo:password@127.0.0.1:${port.host}`,
       db: "mongo",
     },
   }

packages/server/src/integrations/tests/utils/mssql.ts

@@ -1,43 +1,41 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
 import mssql from "mssql"
-import { generator } from "@budibase/backend-core/tests"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+
+let ports: Promise<testContainerUtils.Port[]>
 
 export async function getDatasource(): Promise<Datasource> {
-  let container = new GenericContainer(
-    "mcr.microsoft.com/mssql/server:2022-latest"
-  )
-    .withExposedPorts(1433)
-    .withEnvironment({
-      ACCEPT_EULA: "Y",
-      MSSQL_SA_PASSWORD: "Password_123",
-      // This is important, as Microsoft allow us to use the "Developer" edition
-      // of SQL Server for development and testing purposes. We can't use other
-      // versions without a valid license, and we cannot use the Developer
-      // version in production.
-      MSSQL_PID: "Developer",
-    })
-    .withWaitStrategy(
-      Wait.forSuccessfulCommand(
-        "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'"
-      )
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("mcr.microsoft.com/mssql/server:2022-latest")
+        .withExposedPorts(1433)
+        .withEnvironment({
+          ACCEPT_EULA: "Y",
+          MSSQL_SA_PASSWORD: "Password_123",
+          // This is important, as Microsoft allow us to use the "Developer" edition
+          // of SQL Server for development and testing purposes. We can't use other
+          // versions without a valid license, and we cannot use the Developer
+          // version in production.
+          MSSQL_PID: "Developer",
+        })
+        .withWaitStrategy(
+          Wait.forSuccessfulCommand(
+            "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'"
+          )
+        )
     )
-
-  if (process.env.REUSE_CONTAINERS) {
-    container = container.withReuse()
   }
 
-  const startedContainer = await container.start()
-
-  const host = startedContainer.getHost()
-  const port = startedContainer.getMappedPort(1433)
+  const port = (await ports).find(x => x.container === 1433)?.host
 
   const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.SQL_SERVER,
     plus: true,
     config: {
-      server: host,
+      server: "127.0.0.1",
       port,
       user: "sa",
       password: "Password_123",

packages/server/src/integrations/tests/utils/mysql.ts

@@ -2,7 +2,10 @@ import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
 import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
 import mysql from "mysql2/promise"
-import { generator } from "@budibase/backend-core/tests"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+
+let ports: Promise<testContainerUtils.Port[]>
 
 class MySQLWaitStrategy extends AbstractWaitStrategy {
   async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
@@ -25,26 +28,23 @@ class MySQLWaitStrategy extends AbstractWaitStrategy {
 }
 
 export async function getDatasource(): Promise<Datasource> {
-  let container = new GenericContainer("mysql:8.3")
-    .withExposedPorts(3306)
-    .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
-    .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000))
-
-  if (process.env.REUSE_CONTAINERS) {
-    container = container.withReuse()
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("mysql:8.3")
+        .withExposedPorts(3306)
+        .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
+        .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000))
+    )
   }
 
-  const startedContainer = await container.start()
-
-  const host = startedContainer.getHost()
-  const port = startedContainer.getMappedPort(3306)
+  const port = (await ports).find(x => x.container === 3306)?.host
 
   const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.MYSQL,
     plus: true,
     config: {
-      host,
+      host: "127.0.0.1",
       port,
       user: "root",
       password: "password",

packages/server/src/integrations/tests/utils/postgres.ts

@@ -1,33 +1,33 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
 import pg from "pg"
-import { generator } from "@budibase/backend-core/tests"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+
+let ports: Promise<testContainerUtils.Port[]>
 
 export async function getDatasource(): Promise<Datasource> {
-  let container = new GenericContainer("postgres:16.1-bullseye")
-    .withExposedPorts(5432)
-    .withEnvironment({ POSTGRES_PASSWORD: "password" })
-    .withWaitStrategy(
-      Wait.forSuccessfulCommand(
-        "pg_isready -h localhost -p 5432"
-      ).withStartupTimeout(10000)
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("postgres:16.1-bullseye")
+        .withExposedPorts(5432)
+        .withEnvironment({ POSTGRES_PASSWORD: "password" })
+        .withWaitStrategy(
+          Wait.forSuccessfulCommand(
+            "pg_isready -h localhost -p 5432"
+          ).withStartupTimeout(10000)
+        )
     )
-
-  if (process.env.REUSE_CONTAINERS) {
-    container = container.withReuse()
   }
 
-  const startedContainer = await container.start()
-
-  const host = startedContainer.getHost()
-  const port = startedContainer.getMappedPort(5432)
+  const port = (await ports).find(x => x.container === 5432)?.host
 
   const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.POSTGRES,
     plus: true,
     config: {
-      host,
+      host: "127.0.0.1",
       port,
       database: "postgres",
       user: "postgres",