Merge pull request #12390 from Budibase/fix/cyclic-issues

Fix for cyclics causing test failures in master

packages/backend-core/src/cache/generic.ts

@@ -1,6 +1,6 @@
-const BaseCache = require("./base")
+import BaseCache from "./base"
 
-const GENERIC = new BaseCache.default()
+const GENERIC = new BaseCache()
 
 export enum CacheKey {
   CHECKLIST = "checklist",
@@ -19,6 +19,7 @@ export enum TTL {
 }
 
 function performExport(funcName: string) {
+  // @ts-ignore
   return (...args: any) => GENERIC[funcName](...args)
 }
 

packages/backend-core/src/cache/index.ts

@@ -2,4 +2,6 @@ export * as generic from "./generic"
 export * as user from "./user"
 export * as app from "./appMetadata"
 export * as writethrough from "./writethrough"
+export * as invite from "./invite"
+export * as passwordReset from "./passwordReset"
 export * from "./generic"

packages/backend-core/src/cache/invite.ts

@@ -1,7 +1,10 @@
-import { utils, tenancy, redis } from "../"
+import * as utils from "../utils"
+import { Duration, DurationType } from "../utils"
 import env from "../environment"
+import { getTenantId } from "../context"
+import * as redis from "../redis/init"
 
-const TTL_SECONDS = 60 * 60 * 24 * 7
+const TTL_SECONDS = Duration.fromDays(7).toSeconds()
 
 interface Invite {
   email: string
@@ -12,25 +15,13 @@ interface InviteWithCode extends Invite {
   code: string
 }
 
-let client: redis.Client
-
-export async function init() {
-  if (!client) {
-    client = new redis.Client(redis.utils.Databases.INVITATIONS)
-  }
-  return client
-}
-
-export async function shutdown() {
-  if (client) await client.finish()
-}
-
 /**
  * Given an invite code and invite body, allow the update an existing/valid invite in redis
- * @param inviteCode The invite code for an invite in redis
+ * @param code The invite code for an invite in redis
  * @param value The body of the updated user invitation
  */
 export async function updateCode(code: string, value: Invite) {
+  const client = await redis.getInviteClient()
   await client.store(code, value, TTL_SECONDS)
 }
 
@@ -42,16 +33,18 @@ export async function updateCode(code: string, value: Invite) {
  */
 export async function createCode(email: string, info: any): Promise<string> {
   const code = utils.newid()
+  const client = await redis.getInviteClient()
   await client.store(code, { email, info }, TTL_SECONDS)
   return code
 }
 
 /**
  * Checks that the provided invite code is valid - will return the email address of user that was invited.
- * @param inviteCode the invite code that was provided as part of the link.
+ * @param code the invite code that was provided as part of the link.
  * @return If the code is valid then an email address will be returned.
  */
 export async function getCode(code: string): Promise<Invite> {
+  const client = await redis.getInviteClient()
   const value = (await client.get(code)) as Invite | undefined
   if (!value) {
     throw "Invitation is not valid or has expired, please request a new one."
@@ -60,13 +53,15 @@ export async function getCode(code: string): Promise<Invite> {
 }
 
 export async function deleteCode(code: string) {
+  const client = await redis.getInviteClient()
   await client.delete(code)
 }
 
-/** 
+/**
-  Get all currently available user invitations for the current tenant.
+ Get all currently available user invitations for the current tenant.
-**/
+ **/
 export async function getInviteCodes(): Promise<InviteWithCode[]> {
+  const client = await redis.getInviteClient()
   const invites: { key: string; value: Invite }[] = await client.scan()
 
   const results: InviteWithCode[] = invites.map(invite => {
@@ -78,7 +73,7 @@ export async function getInviteCodes(): Promise<InviteWithCode[]> {
   if (!env.MULTI_TENANCY) {
     return results
   }
-  const tenantId = tenancy.getTenantId()
+  const tenantId = getTenantId()
   return results.filter(invite => tenantId === invite.info.tenantId)
 }
 

packages/backend-core/src/cache/passwordReset.ts

@@ -1,25 +1,14 @@
-import { redis, utils } from "../"
+import * as redis from "../redis/init"
+import * as utils from "../utils"
+import { Duration, DurationType } from "../utils"
 
-const TTL_SECONDS = 60 * 60
+const TTL_SECONDS = Duration.fromHours(1).toSeconds()
 
 interface PasswordReset {
   userId: string
   info: any
 }
 
-let client: redis.Client
-
-export async function init() {
-  if (!client) {
-    client = new redis.Client(redis.utils.Databases.PW_RESETS)
-  }
-  return client
-}
-
-export async function shutdown() {
-  if (client) await client.finish()
-}
-
 /**
  * Given a user ID this will store a code (that is returned) for an hour in redis.
  * The user can then return this code for resetting their password (through their reset link).
@@ -29,6 +18,7 @@ export async function shutdown() {
  */
 export async function createCode(userId: string, info: any): Promise<string> {
   const code = utils.newid()
+  const client = await redis.getPasswordResetClient()
   await client.store(code, { userId, info }, TTL_SECONDS)
   return code
 }
@@ -39,6 +29,7 @@ export async function createCode(userId: string, info: any): Promise<string> {
  * @return returns the user ID if it is found
  */
 export async function getCode(code: string): Promise<PasswordReset> {
+  const client = await redis.getPasswordResetClient()
   const value = (await client.get(code)) as PasswordReset | undefined
   if (!value) {
     throw "Provided information is not valid, cannot reset password - please try again."

packages/backend-core/src/redis/index.ts

@@ -4,5 +4,3 @@ export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
 export * as locks from "./redlockImpl"
-export * as invite from "./invite"
-export * as passwordReset from "./passwordReset"

packages/backend-core/src/redis/init.ts

@@ -7,15 +7,19 @@ let userClient: Client,
   cacheClient: Client,
   writethroughClient: Client,
   lockClient: Client,
-  socketClient: Client
+  socketClient: Client,
+  inviteClient: Client,
+  passwordResetClient: Client
 
-async function init() {
+export async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
   sessionClient = await new Client(utils.Databases.SESSIONS).init()
   appClient = await new Client(utils.Databases.APP_METADATA).init()
   cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
   lockClient = await new Client(utils.Databases.LOCKS).init()
   writethroughClient = await new Client(utils.Databases.WRITE_THROUGH).init()
+  inviteClient = await new Client(utils.Databases.INVITATIONS).init()
+  passwordResetClient = await new Client(utils.Databases.PW_RESETS).init()
   socketClient = await new Client(
     utils.Databases.SOCKET_IO,
     utils.SelectableDatabase.SOCKET_IO
@@ -29,6 +33,8 @@ export async function shutdown() {
   if (cacheClient) await cacheClient.finish()
   if (writethroughClient) await writethroughClient.finish()
   if (lockClient) await lockClient.finish()
+  if (inviteClient) await inviteClient.finish()
+  if (passwordResetClient) await passwordResetClient.finish()
   if (socketClient) await socketClient.finish()
 }
 
@@ -84,3 +90,17 @@ export async function getSocketClient() {
   }
   return socketClient
 }
+
+export async function getInviteClient() {
+  if (!inviteClient) {
+    await init()
+  }
+  return inviteClient
+}
+
+export async function getPasswordResetClient() {
+  if (!passwordResetClient) {
+    await init()
+  }
+  return passwordResetClient
+}

packages/backend-core/src/users/lookup.ts

@@ -6,7 +6,7 @@ import {
 } from "@budibase/types"
 import * as dbUtils from "../db"
 import { ViewName } from "../constants"
-import { getExistingInvites } from "../redis/invite"
+import { getExistingInvites } from "../cache/invite"
 
 /**
  * Apply a system-wide search on emails:

packages/backend-core/src/utils/Duration.ts

@@ -28,6 +28,9 @@ export class Duration {
       toMs: () => {
         return Duration.convert(from, DurationType.MILLISECONDS, duration)
       },
+      toSeconds: () => {
+        return Duration.convert(from, DurationType.SECONDS, duration)
+      },
     }
   }
 

packages/server/src/middleware/tests/currentapp.spec.js

@@ -9,11 +9,11 @@ function mockWorker() {
       return {
         _id: "us_uuid1",
         roles: {
-          "app_test": "BASIC",
+          app_test: "BASIC",
         },
         roleId: "BASIC",
       }
-    }
+    },
   }))
 }
 
@@ -109,7 +109,7 @@ class TestConfiguration {
       path: "",
       cookies: {
         set: jest.fn(),
-      }
+      },
     }
   }
 

packages/worker/src/api/controllers/global/users.ts

@@ -1,4 +1,3 @@
-import { redis } from "@budibase/backend-core"
 import * as userSdk from "../../../sdk/users"
 import env from "../../../environment"
 import {
@@ -308,7 +307,7 @@ export const checkInvite = async (ctx: any) => {
   const { code } = ctx.params
   let invite
   try {
-    invite = await redis.invite.getCode(code)
+    invite = await cache.invite.getCode(code)
   } catch (e) {
     console.warn("Error getting invite from code", e)
     ctx.throw(400, "There was a problem with the invite")
@@ -322,7 +321,7 @@ export const checkInvite = async (ctx: any) => {
 export const getUserInvites = async (ctx: any) => {
   try {
     // Restricted to the currently authenticated tenant
-    ctx.body = await redis.invite.getInviteCodes()
+    ctx.body = await cache.invite.getInviteCodes()
   } catch (e) {
     ctx.throw(400, "There was a problem fetching invites")
   }
@@ -336,7 +335,7 @@ export const updateInvite = async (ctx: any) => {
 
   let invite
   try {
-    invite = await redis.invite.getCode(code)
+    invite = await cache.invite.getCode(code)
   } catch (e) {
     ctx.throw(400, "There was a problem with the invite")
     return
@@ -364,7 +363,7 @@ export const updateInvite = async (ctx: any) => {
     }
   }
 
-  await redis.invite.updateCode(code, updated)
+  await cache.invite.updateCode(code, updated)
   ctx.body = { ...invite }
 }
 
@@ -374,8 +373,8 @@ export const inviteAccept = async (
   const { inviteCode, password, firstName, lastName } = ctx.request.body
   try {
     // info is an extension of the user object that was stored by global
-    const { email, info }: any = await redis.invite.getCode(inviteCode)
+    const { email, info }: any = await cache.invite.getCode(inviteCode)
-    await redis.invite.deleteCode(inviteCode)
+    await cache.invite.deleteCode(inviteCode)
     const user = await tenancy.doInTenant(info.tenantId, async () => {
       let request: any = {
         firstName,

packages/worker/src/index.ts

@@ -72,9 +72,8 @@ server.on("close", async () => {
   shuttingDown = true
   console.log("Server Closed")
   timers.cleanup()
-  await redis.invite.shutdown()
+  events.shutdown()
-  await redis.passwordReset.shutdown()
+  await redis.clients.shutdown()
-  await events.shutdown()
   await queue.shutdown()
   if (!env.isTest()) {
     process.exit(errCode)
@@ -89,8 +88,7 @@ const shutdown = () => {
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
-  await redis.invite.init()
+  await redis.clients.init()
-  await redis.passwordReset.init()
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues
   await events.processors.init(proSdk.auditLogs.write)

packages/worker/src/sdk/auth/auth.ts

@@ -6,7 +6,7 @@ import {
   sessions,
   tenancy,
   utils as coreUtils,
-  redis,
+  cache,
 } from "@budibase/backend-core"
 import { PlatformLogoutOpts, User } from "@budibase/types"
 import jwt from "jsonwebtoken"
@@ -73,7 +73,7 @@ export const reset = async (email: string) => {
  * Perform the user password update if the provided reset code is valid.
  */
 export const resetUpdate = async (resetCode: string, password: string) => {
-  const { userId } = await redis.passwordReset.getCode(resetCode)
+  const { userId } = await cache.passwordReset.getCode(resetCode)
 
   let user = await userSdk.db.getUser(userId)
   user.password = password

packages/worker/src/utilities/email.ts

@@ -3,9 +3,8 @@ import { EmailTemplatePurpose, TemplateType } from "../constants"
 import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
 import { getSettingsTemplateContext } from "./templates"
 import { processString } from "@budibase/string-templates"
-import { redis } from "@budibase/backend-core"
 import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
-import { configs } from "@budibase/backend-core"
+import { configs, cache } from "@budibase/backend-core"
 import ical from "ical-generator"
 const nodemailer = require("nodemailer")
 
@@ -61,9 +60,9 @@ async function getLinkCode(
 ) {
   switch (purpose) {
     case EmailTemplatePurpose.PASSWORD_RECOVERY:
-      return redis.passwordReset.createCode(user._id!, info)
+      return cache.passwordReset.createCode(user._id!, info)
     case EmailTemplatePurpose.INVITATION:
-      return redis.invite.createCode(email, info)
+      return cache.invite.createCode(email, info)
     default:
       return null
   }