From 90cfdd661de448ebb484a426dc2822bf9c1feb2f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Mar 2024 17:36:26 +0000 Subject: [PATCH] Rework how we connect to containers. --- .github/workflows/budibase_ci.yml | 3 +- .../core/utilities/testContainerUtils.ts | 47 ++++++++++++----- .../src/integrations/tests/utils/index.ts | 25 ++++++++++ .../src/integrations/tests/utils/mariadb.ts | 29 ++++++----- .../src/integrations/tests/utils/mongodb.ts | 40 ++++++++------- .../src/integrations/tests/utils/mssql.ts | 50 +++++++++---------- .../src/integrations/tests/utils/mysql.ts | 26 +++++----- .../src/integrations/tests/utils/postgres.ts | 32 ++++++------ 8 files changed, 152 insertions(+), 100 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 6ad5e68cbd..224537d216 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -136,7 +136,8 @@ jobs: fi test-server: - runs-on: budi-tubby-tornado-quad-core-150gb + runs-on: + group: hosted-runners env: DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull steps: diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 5d4f5a3c11..2f33db65d3 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -1,6 +1,8 @@ import { DatabaseImpl } from "../../../src/db" import { execSync } from "child_process" +const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g") + interface ContainerInfo { Command: string CreatedAt: string @@ -19,7 +21,10 @@ interface ContainerInfo { } function getTestcontainers(): ContainerInfo[] { - return execSync("docker ps --format json") + // We use --format json to make sure the output is nice and machine-readable, + // and we use --no-trunc so that the command returns full container IDs so we + // can filter on them correctly. + return execSync("docker ps --format json --no-trunc") .toString() .split("\n") .filter(x => x.length > 0) @@ -27,32 +32,51 @@ function getTestcontainers(): ContainerInfo[] { .filter(x => x.Labels.includes("org.testcontainers=true")) } -function getContainerByImage(image: string) { - return getTestcontainers().find(x => x.Image.startsWith(image)) +export function getContainerByImage(image: string) { + const containers = getTestcontainers().filter(x => x.Image.startsWith(image)) + if (containers.length > 1) { + throw new Error(`Multiple containers found with image: ${image}`) + } + return containers[0] } -function getExposedPort(container: ContainerInfo, port: number) { - const match = container.Ports.match(new RegExp(`0.0.0.0:(\\d+)->${port}/tcp`)) - if (!match) { - return undefined +export function getContainerById(id: string) { + return getTestcontainers().find(x => x.ID === id) +} + +export interface Port { + host: number + container: number +} + +export function getExposedV4Ports(container: ContainerInfo): Port[] { + let ports: Port[] = [] + for (const match of container.Ports.matchAll(IPV4_PORT_REGEX)) { + ports.push({ host: parseInt(match[1]), container: parseInt(match[2]) }) } - return parseInt(match[1]) + return ports +} + +export function getExposedV4Port(container: ContainerInfo, port: number) { + return getExposedV4Ports(container).find(x => x.container === port)?.host } export function setupEnv(...envs: any[]) { + // We start couchdb in globalSetup.ts, in the root of the monorepo, so it + // should be relatively safe to look for it by its image name. const couch = getContainerByImage("budibase/couchdb") if (!couch) { throw new Error("CouchDB container not found") } - const couchPort = getExposedPort(couch, 5984) + const couchPort = getExposedV4Port(couch, 5984) if (!couchPort) { throw new Error("CouchDB port not found") } const configs = [ { key: "COUCH_DB_PORT", value: `${couchPort}` }, - { key: "COUCH_DB_URL", value: `http://localhost:${couchPort}` }, + { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` }, ] for (const config of configs.filter(x => !!x.value)) { @@ -60,7 +84,4 @@ export function setupEnv(...envs: any[]) { env._set(config.key, config.value) } } - - // @ts-expect-error - DatabaseImpl.nano = undefined } diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 5760273d51..bbdb41b38a 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -6,6 +6,8 @@ import * as mongodb from "./mongodb" import * as mysql from "./mysql" import * as mssql from "./mssql" import * as mariadb from "./mariadb" +import { GenericContainer } from "testcontainers" +import { testContainerUtils } from "@budibase/backend-core/tests" export type DatasourceProvider = () => Promise @@ -63,3 +65,26 @@ export async function rawQuery(ds: Datasource, sql: string): Promise { } } } + +export async function startContainer(container: GenericContainer) { + if (process.env.REUSE_CONTAINERS) { + container = container.withReuse() + } + + const startedContainer = await container.start() + + const info = testContainerUtils.getContainerById(startedContainer.getId()) + if (!info) { + throw new Error("Container not found") + } + + // Some Docker runtimes, when you expose a port, will bind it to both + // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6 + // addresses are not shared, and testcontainers will sometimes give you back + // the ipv6 port. There's no way to know that this has happened, and if you + // try to then connect to `localhost:port` you may attempt to bind to the v4 + // address which could be unbound or even an entirely different container. For + // that reason, we don't use testcontainers' `getExposedPort` function, + // preferring instead our own method that guaranteed v4 ports. + return testContainerUtils.getExposedV4Ports(info) +} diff --git a/packages/server/src/integrations/tests/utils/mariadb.ts b/packages/server/src/integrations/tests/utils/mariadb.ts index 2634c9f913..fcd79b8e56 100644 --- a/packages/server/src/integrations/tests/utils/mariadb.ts +++ b/packages/server/src/integrations/tests/utils/mariadb.ts @@ -2,7 +2,10 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" import { rawQuery } from "./mysql" -import { generator } from "@budibase/backend-core/tests" +import { generator, testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." + +let ports: Promise class MariaDBWaitStrategy extends AbstractWaitStrategy { async waitUntilReady(container: any, boundPorts: any, startTime?: Date) { @@ -22,22 +25,22 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy { } export async function getDatasource(): Promise { - let container = new GenericContainer("mariadb:lts") - .withExposedPorts(3306) - .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" }) - .withWaitStrategy(new MariaDBWaitStrategy()) - - if (process.env.REUSE_CONTAINERS) { - container = container.withReuse() + if (!ports) { + ports = startContainer( + new GenericContainer("mariadb:lts") + .withExposedPorts(3306) + .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" }) + .withWaitStrategy(new MariaDBWaitStrategy()) + ) } - const startedContainer = await container.start() - - const host = startedContainer.getHost() - const port = startedContainer.getMappedPort(3306) + const port = (await ports).find(x => x.container === 3306)?.host + if (!port) { + throw new Error("MariaDB port not found") + } const config = { - host, + host: "127.0.0.1", port, user: "root", password: "password", diff --git a/packages/server/src/integrations/tests/utils/mongodb.ts b/packages/server/src/integrations/tests/utils/mongodb.ts index 26fbff966e..c5c0340dc9 100644 --- a/packages/server/src/integrations/tests/utils/mongodb.ts +++ b/packages/server/src/integrations/tests/utils/mongodb.ts @@ -1,34 +1,38 @@ +import { testContainerUtils } from "@budibase/backend-core/tests" import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" +import { startContainer } from "." + +let ports: Promise export async function getDatasource(): Promise { - let container = new GenericContainer("mongo:7.0-jammy") - .withExposedPorts(27017) - .withEnvironment({ - MONGO_INITDB_ROOT_USERNAME: "mongo", - MONGO_INITDB_ROOT_PASSWORD: "password", - }) - .withWaitStrategy( - Wait.forSuccessfulCommand( - `mongosh --eval "db.version()"` - ).withStartupTimeout(10000) + if (!ports) { + ports = startContainer( + new GenericContainer("mongo:7.0-jammy") + .withExposedPorts(27017) + .withEnvironment({ + MONGO_INITDB_ROOT_USERNAME: "mongo", + MONGO_INITDB_ROOT_PASSWORD: "password", + }) + .withWaitStrategy( + Wait.forSuccessfulCommand( + `mongosh --eval "db.version()"` + ).withStartupTimeout(10000) + ) ) - - if (process.env.REUSE_CONTAINERS) { - container = container.withReuse() } - const startedContainer = await container.start() - - const host = startedContainer.getHost() - const port = startedContainer.getMappedPort(27017) + const port = (await ports).find(x => x.container === 27017) + if (!port) { + throw new Error("MongoDB port not found") + } return { type: "datasource", source: SourceName.MONGODB, plus: false, config: { - connectionString: `mongodb://mongo:password@${host}:${port}`, + connectionString: `mongodb://mongo:password@127.0.0.1:${port.host}`, db: "mongo", }, } diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index 290bc78246..647f461272 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -1,43 +1,41 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import mssql from "mssql" -import { generator } from "@budibase/backend-core/tests" +import { generator, testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." + +let ports: Promise export async function getDatasource(): Promise { - let container = new GenericContainer( - "mcr.microsoft.com/mssql/server:2022-latest" - ) - .withExposedPorts(1433) - .withEnvironment({ - ACCEPT_EULA: "Y", - MSSQL_SA_PASSWORD: "Password_123", - // This is important, as Microsoft allow us to use the "Developer" edition - // of SQL Server for development and testing purposes. We can't use other - // versions without a valid license, and we cannot use the Developer - // version in production. - MSSQL_PID: "Developer", - }) - .withWaitStrategy( - Wait.forSuccessfulCommand( - "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'" - ) + if (!ports) { + ports = startContainer( + new GenericContainer("mcr.microsoft.com/mssql/server:2022-latest") + .withExposedPorts(1433) + .withEnvironment({ + ACCEPT_EULA: "Y", + MSSQL_SA_PASSWORD: "Password_123", + // This is important, as Microsoft allow us to use the "Developer" edition + // of SQL Server for development and testing purposes. We can't use other + // versions without a valid license, and we cannot use the Developer + // version in production. + MSSQL_PID: "Developer", + }) + .withWaitStrategy( + Wait.forSuccessfulCommand( + "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'" + ) + ) ) - - if (process.env.REUSE_CONTAINERS) { - container = container.withReuse() } - const startedContainer = await container.start() - - const host = startedContainer.getHost() - const port = startedContainer.getMappedPort(1433) + const port = (await ports).find(x => x.container === 1433)?.host const datasource: Datasource = { type: "datasource_plus", source: SourceName.SQL_SERVER, plus: true, config: { - server: host, + server: "127.0.0.1", port, user: "sa", password: "Password_123", diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts index 0f83128f26..a78833e1de 100644 --- a/packages/server/src/integrations/tests/utils/mysql.ts +++ b/packages/server/src/integrations/tests/utils/mysql.ts @@ -2,7 +2,10 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" import mysql from "mysql2/promise" -import { generator } from "@budibase/backend-core/tests" +import { generator, testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." + +let ports: Promise class MySQLWaitStrategy extends AbstractWaitStrategy { async waitUntilReady(container: any, boundPorts: any, startTime?: Date) { @@ -25,26 +28,23 @@ class MySQLWaitStrategy extends AbstractWaitStrategy { } export async function getDatasource(): Promise { - let container = new GenericContainer("mysql:8.3") - .withExposedPorts(3306) - .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" }) - .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000)) - - if (process.env.REUSE_CONTAINERS) { - container = container.withReuse() + if (!ports) { + ports = startContainer( + new GenericContainer("mysql:8.3") + .withExposedPorts(3306) + .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" }) + .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000)) + ) } - const startedContainer = await container.start() - - const host = startedContainer.getHost() - const port = startedContainer.getMappedPort(3306) + const port = (await ports).find(x => x.container === 3306)?.host const datasource: Datasource = { type: "datasource_plus", source: SourceName.MYSQL, plus: true, config: { - host, + host: "127.0.0.1", port, user: "root", password: "password", diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index 237bc19a17..4191b107e9 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -1,33 +1,33 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import pg from "pg" -import { generator } from "@budibase/backend-core/tests" +import { generator, testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." + +let ports: Promise export async function getDatasource(): Promise { - let container = new GenericContainer("postgres:16.1-bullseye") - .withExposedPorts(5432) - .withEnvironment({ POSTGRES_PASSWORD: "password" }) - .withWaitStrategy( - Wait.forSuccessfulCommand( - "pg_isready -h localhost -p 5432" - ).withStartupTimeout(10000) + if (!ports) { + ports = startContainer( + new GenericContainer("postgres:16.1-bullseye") + .withExposedPorts(5432) + .withEnvironment({ POSTGRES_PASSWORD: "password" }) + .withWaitStrategy( + Wait.forSuccessfulCommand( + "pg_isready -h localhost -p 5432" + ).withStartupTimeout(10000) + ) ) - - if (process.env.REUSE_CONTAINERS) { - container = container.withReuse() } - const startedContainer = await container.start() - - const host = startedContainer.getHost() - const port = startedContainer.getMappedPort(5432) + const port = (await ports).find(x => x.container === 5432)?.host const datasource: Datasource = { type: "datasource_plus", source: SourceName.POSTGRES, plus: true, config: { - host, + host: "127.0.0.1", port, database: "postgres", user: "postgres",