WIP: Remove Redis mocks.

globalSetup.ts

@@ -82,7 +82,20 @@ export default async function setup() {
         Wait.forHttp("/minio/health/ready", 9000).withStartupTimeout(10000)
       )
 
-    await Promise.all([couchdb.start(), minio.start()])
+    const redis = new GenericContainer("redis")
+      .withExposedPorts(6379)
+      .withReuse()
+      .withEnvironment({
+        REDIS_PASSWORD: "budibase",
+      })
+      .withLabels({ "com.budibase": "true" })
+      .withWaitStrategy(
+        Wait.forLogMessage("Ready to accept connections", 1).withStartupTimeout(
+          20000
+        )
+      )
+
+    await Promise.all([couchdb.start(), minio.start(), redis.start()])
   } finally {
     lockfile.unlockSync(lockPath)
   }

packages/backend-core/src/environment.ts

@@ -157,7 +157,6 @@ const environment = {
     process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
   TEMP_BUCKET_NAME: process.env.TEMP_BUCKET_NAME || DefaultBucketName.TEMP,
   USE_COUCH: process.env.USE_COUCH || true,
-  MOCK_REDIS: process.env.MOCK_REDIS,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
   SERVICE: process.env.SERVICE || "budibase",
   LOG_LEVEL: process.env.LOG_LEVEL || "info",

packages/backend-core/src/redis/redis.ts

@@ -1,15 +1,6 @@
 import env from "../environment"
 import Redis, { Cluster } from "ioredis"
 // mock-redis doesn't have any typing
-let MockRedis: any | undefined
-if (env.MOCK_REDIS) {
-  try {
-    // ioredis mock is all in memory
-    MockRedis = require("ioredis-mock")
-  } catch (err) {
-    console.log("Mock redis unavailable")
-  }
-}
 import {
   addDbPrefix,
   removeDbPrefix,
@@ -28,14 +19,9 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 
 // for testing just generate the client once
 let CLOSED = false
-const CLIENTS: Record<number, Redis> = {}
+const CLIENTS: Record<number, Redis | Cluster> = {}
 let CONNECTED = false
 
-// mock redis always connected
-if (env.MOCK_REDIS) {
-  CONNECTED = true
-}
-
 function pickClient(selectDb: number) {
   return CLIENTS[selectDb]
 }
@@ -60,7 +46,6 @@ function connectionError(timeout: NodeJS.Timeout, err: Error | string) {
  * will return the ioredis client which will be ready to use.
  */
 function init(selectDb = DEFAULT_SELECT_DB) {
-  const RedisCore = env.MOCK_REDIS && MockRedis ? MockRedis : Redis
   let timeout: NodeJS.Timeout
   CLOSED = false
   let client = pickClient(selectDb)
@@ -68,10 +53,6 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   if (client && CONNECTED) {
     return
   }
-  // testing uses a single in memory client
-  if (env.MOCK_REDIS) {
-    CLIENTS[selectDb] = new RedisCore(getRedisOptions())
-  }
   // start the timer - only allowed 5 seconds to connect
   timeout = setTimeout(() => {
     if (!CONNECTED) {
@@ -87,9 +68,9 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   const opts = getRedisOptions()
 
   if (CLUSTERED) {
-    client = new RedisCore.Cluster([{ host, port }], opts)
+    client = new Cluster([{ host, port }], opts)
   } else {
-    client = new RedisCore(opts)
+    client = new Redis(opts)
   }
   // attach handlers
   client.on("end", (err: Error) => {
@@ -111,6 +92,12 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   CLIENTS[selectDb] = client
 }
 
+export function close() {
+  for (let key in CLIENTS) {
+    CLIENTS[key].disconnect()
+  }
+}
+
 function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) {
   return new Promise(resolve => {
     if (pickClient(selectDb) == null) {

packages/backend-core/src/redis/tests/redis.spec.ts

@@ -1,28 +1,8 @@
-import { GenericContainer, StartedTestContainer } from "testcontainers"
 import { generator, structures } from "../../../tests"
 import RedisWrapper from "../redis"
-import { env } from "../.."
-
-jest.setTimeout(30000)
 
 describe("redis", () => {
   let redis: RedisWrapper
-  let container: StartedTestContainer
-
-  beforeAll(async () => {
-    const container = await new GenericContainer("redis")
-      .withExposedPorts(6379)
-      .start()
-
-    env._set(
-      "REDIS_URL",
-      `${container.getHost()}:${container.getMappedPort(6379)}`
-    )
-    env._set("MOCK_REDIS", 0)
-    env._set("REDIS_PASSWORD", 0)
-  })
-
-  afterAll(() => container?.stop())
 
   beforeEach(async () => {
     redis = new RedisWrapper(structures.db.id())

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -44,6 +44,9 @@ export function getContainerByImage(image: string) {
     }
     throw new Error(errorMessage)
   }
+  if (containers.length === 0) {
+    return undefined
+  }
   return containers[0]
 }
 
@@ -87,17 +90,31 @@ export function setupEnv(...envs: any[]) {
   }
 
   const minio = getContainerByImage("minio/minio")
+  if (!minio) {
+    throw new Error("Minio container not found")
+  }
 
   const minioPort = getExposedV4Port(minio, 9000)
   if (!minioPort) {
     throw new Error("Minio port not found")
   }
 
+  const redis = getContainerByImage("redis")
+  if (!redis) {
+    throw new Error("Redis container not found")
+  }
+
+  const redisPort = getExposedV4Port(redis, 6379)
+  if (!redisPort) {
+    throw new Error("Redis port not found")
+  }
+
   const configs = [
     { key: "COUCH_DB_PORT", value: `${couchPort}` },
     { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
     { key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
     { key: "MINIO_URL", value: `http://127.0.0.1:${minioPort}` },
+    { key: "REDIS_URL", value: `redis://127.0.0.1:${redisPort}` },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/backend-core/tests/jestEnv.ts

@@ -1,7 +1,6 @@
 process.env.SELF_HOSTED = "1"
 process.env.MULTI_TENANCY = "1"
 process.env.NODE_ENV = "jest"
-process.env.MOCK_REDIS = "1"
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.REDIS_PASSWORD = "budibase"
 process.env.COUCH_DB_PASSWORD = "budibase"

packages/backend-core/tests/jestSetup.ts

@@ -2,6 +2,7 @@ import "./core/logging"
 import env from "../src/environment"
 import { cleanup } from "../src/timers"
 import { mocks, testContainerUtils } from "./core/utilities"
+import { close } from "../src/redis/redis"
 
 // must explicitly enable fetch mock
 mocks.fetch.enable()
@@ -26,4 +27,5 @@ testContainerUtils.setupEnv(env)
 
 afterAll(() => {
   cleanup()
+  close()
 })

packages/server/src/app.ts

@@ -14,14 +14,17 @@ import Koa from "koa"
 import { Server } from "http"
 import { startup } from "./startup"
 
-let app: Koa, server: Server
+let app: Koa, server: Server, started: Promise<void>
 
 async function start() {
   const koa = createKoaApp()
-  app = koa.app
-  server = koa.server
-  // startup includes automation runner - if enabled
-  await startup({ app, server })
+  started = new Promise(resolve => {
+    app = koa.app
+    server = koa.server
+    // startup includes automation runner - if enabled
+    startup({ app, server }).then(resolve)
+  })
+  await started
 }
 
 start().catch(err => {
@@ -29,6 +32,7 @@ start().catch(err => {
   throw err
 })
 
-export function getServer(): Server {
+export async function getServer(): Promise<Server> {
+  await started
   return server
 }

packages/server/src/tests/jestEnv.ts

@@ -6,7 +6,6 @@ process.env.MULTI_TENANCY = "1"
 // @ts-ignore
 process.env.BUDIBASE_DIR = tmpdir("budibase-unittests")
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
-process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"
 process.env.BUDIBASE_VERSION = "0.0.0+jest"

packages/server/src/tests/utilities/api/base.ts

@@ -110,7 +110,7 @@ export abstract class TestAPI {
       ? this.config.publicHeaders.bind(this.config)
       : this.config.defaultHeaders.bind(this.config)
 
-    const app = getServer()
+    const app = await getServer()
     let req = request(app)[method](url)
     req = req.set(
       headersFn({

packages/worker/src/tests/jestEnv.ts

@@ -9,7 +9,6 @@ process.env.MINIO_SECRET_KEY = "budibase"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.INTERNAL_API_KEY = "tet"
 process.env.DISABLE_ACCOUNT_PORTAL = "0"
-process.env.MOCK_REDIS = "1"
 process.env.BUDIBASE_VERSION = "0.0.0+jest"
 process.env.COUCH_DB_PASSWORD = "budibase"
 process.env.COUCH_DB_USER = "budibase"