Merge branch 'develop' of github.com:Budibase/budibase into feature/BUDI-7052

.tool-versions

@@ -1,2 +1,3 @@
 nodejs 14.21.3
-python 3.10.0
+python 3.10.0
+yarn 1.22.19

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.29-alpha.2",
+  "version": "2.8.29-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/jest.config.ts

@@ -1,8 +1,6 @@
 import { Config } from "@jest/types"
-const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
-  ...preset,
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./tests/jestSetup.ts"],

packages/backend-core/package.json

@@ -24,7 +24,6 @@
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
-    "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
@@ -59,12 +58,13 @@
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.5.0",
-    "@swc/core": "^1.3.25",
-    "@swc/jest": "^0.2.24",
+    "@jest/test-sequencer": "29.6.2",
+    "@shopify/jest-koa-mocks": "5.1.1",
+    "@swc/core": "1.3.71",
+    "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
-    "@types/jest": "29.5.0",
+    "@types/jest": "29.5.3",
     "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
     "@types/node": "14.18.20",
@@ -76,15 +76,14 @@
     "@types/uuid": "8.3.4",
     "chance": "1.1.8",
     "ioredis-mock": "8.7.0",
-    "jest": "29.5.0",
-    "jest-environment-node": "29.5.0",
-    "jest-serial-runner": "^1.2.1",
+    "jest": "29.6.2",
+    "jest-environment-node": "29.6.2",
+    "jest-serial-runner": "1.2.1",
     "koa": "2.13.4",
     "nodemon": "2.0.16",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-jest": "29.0.5",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"

packages/backend-core/scripts/test.sh

@@ -8,6 +8,6 @@ then
   jest --coverage --runInBand --forceExit
 else
   # --maxWorkers performs better in development
-  echo "jest --coverage --forceExit"
-  jest --coverage --forceExit
+  echo "jest --coverage --detectOpenHandles"
+  jest --coverage --detectOpenHandles
 fi

packages/backend-core/src/security/permissions.ts

@@ -1,7 +1,7 @@
-import { flatten } from "lodash"
-import { cloneDeep } from "lodash/fp"
 import { PermissionType, PermissionLevel } from "@budibase/types"
 export { PermissionType, PermissionLevel } from "@budibase/types"
+import flatten from "lodash/flatten"
+import cloneDeep from "lodash/fp/cloneDeep"
 
 export type RoleHierarchy = {
   permissionId: string

packages/backend-core/src/security/roles.ts

@@ -3,7 +3,7 @@ import { prefixRoleID, getRoleParams, DocumentType, SEPARATOR } from "../db"
 import { getAppDB } from "../context"
 import { doWithDB } from "../db"
 import { Screen, Role as RoleDoc } from "@budibase/types"
-const { cloneDeep } = require("lodash/fp")
+import cloneDeep from "lodash/fp/cloneDeep"
 
 export const BUILTIN_ROLE_IDS = {
   ADMIN: "ADMIN",

packages/backend-core/src/security/tests/permissions.spec.ts

@@ -1,4 +1,4 @@
-import { cloneDeep } from "lodash"
+import cloneDeep from "lodash/cloneDeep"
 import * as permissions from "../permissions"
 import { BUILTIN_ROLE_IDS } from "../roles"
 

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -1,5 +1,5 @@
 import { Feature, License, Quotas } from "@budibase/types"
-import _ from "lodash"
+import cloneDeep from "lodash/cloneDeep"
 
 let CLOUD_FREE_LICENSE: License
 let UNLIMITED_LICENSE: License
@@ -58,7 +58,7 @@ export const useCloudFree = () => {
 // FEATURES
 
 const useFeature = (feature: Feature) => {
-  const license = _.cloneDeep(UNLIMITED_LICENSE)
+  const license = cloneDeep(UNLIMITED_LICENSE)
   const opts: UseLicenseOpts = {
     features: [feature],
   }
@@ -101,7 +101,7 @@ export const useAppBuilders = () => {
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {
-  const license = _.cloneDeep(UNLIMITED_LICENSE)
+  const license = cloneDeep(UNLIMITED_LICENSE)
   license.quotas.constant.automationLogRetentionDays.value = value
   return useLicense(license)
 }

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -11,7 +11,7 @@ import {
   CreateAccount,
   CreatePassswordAccount,
 } from "@budibase/types"
-import _ from "lodash"
+import sample from "lodash/sample"
 
 export const account = (partial: Partial<Account> = {}): Account => {
   return {
@@ -46,13 +46,11 @@ export const cloudAccount = (): CloudAccount => {
 }
 
 function providerType(): AccountSSOProviderType {
-  return _.sample(
-    Object.values(AccountSSOProviderType)
-  ) as AccountSSOProviderType
+  return sample(Object.values(AccountSSOProviderType)) as AccountSSOProviderType
 }
 
 function provider(): AccountSSOProvider {
-  return _.sample(Object.values(AccountSSOProvider)) as AccountSSOProvider
+  return sample(Object.values(AccountSSOProvider)) as AccountSSOProvider
 }
 
 export function ssoAccount(account: Account = cloudAccount()): SSOAccount {

packages/backend-core/tests/core/utilities/structures/scim.ts

@@ -1,7 +1,6 @@
 import { ScimCreateGroupRequest, ScimCreateUserRequest } from "@budibase/types"
 import { uuid } from "./common"
 import { generator } from "./generator"
-import _ from "lodash"
 
 interface CreateUserRequestFields {
   externalId: string
@@ -20,10 +19,10 @@ export function createUserRequest(userData?: Partial<CreateUserRequestFields>) {
     username: generator.name(),
   }
 
-  const { externalId, email, firstName, lastName, username } = _.assign(
-    defaultValues,
-    userData
-  )
+  const { externalId, email, firstName, lastName, username } = {
+    ...defaultValues,
+    ...userData,
+  }
 
   let user: ScimCreateUserRequest = {
     schemas: [

packages/backend-core/tests/core/utilities/structures/sso.ts

@@ -15,7 +15,7 @@ import { generator } from "./generator"
 import { email, uuid } from "./common"
 import * as shared from "./shared"
 import { user } from "./shared"
-import _ from "lodash"
+import sample from "lodash/sample"
 
 export function OAuth(): OAuth2 {
   return {
@@ -47,7 +47,7 @@ export function authDetails(userDoc?: User): SSOAuthDetails {
 }
 
 export function providerType(): SSOProviderType {
-  return _.sample(Object.values(SSOProviderType)) as SSOProviderType
+  return sample(Object.values(SSOProviderType)) as SSOProviderType
 }
 
 export function ssoProfile(user?: User): SSOProfile {

packages/builder/package.json

@@ -101,14 +101,14 @@
     "@rollup/plugin-replace": "^2.4.2",
     "@roxi/routify": "2.18.5",
     "@sveltejs/vite-plugin-svelte": "1.0.1",
-    "@testing-library/jest-dom": "^5.11.10",
+    "@testing-library/jest-dom": "5.17.0",
     "@testing-library/svelte": "^3.2.2",
-    "babel-jest": "^26.6.3",
+    "babel-jest": "29.6.2",
     "cypress": "^9.3.1",
     "cypress-multi-reporters": "^1.6.0",
     "cypress-terminal-report": "^1.4.1",
     "identity-obj-proxy": "^3.0.0",
-    "jest": "^26.6.3",
+    "jest": "29.6.2",
     "jsdom": "^21.1.1",
     "mochawesome": "^7.1.3",
     "mochawesome-merge": "^4.2.1",

packages/cli/package.json

@@ -53,9 +53,9 @@
     "yaml": "^2.1.1"
   },
   "devDependencies": {
-    "@swc/core": "^1.3.25",
-    "@swc/jest": "^0.2.24",
-    "@types/jest": "^29.4.0",
+    "@swc/core": "1.3.71",
+    "@swc/jest": "0.2.27",
+    "@types/jest": "29.5.3",
     "@types/node-fetch": "2.6.1",
     "@types/pouchdb": "^6.4.0",
     "copyfiles": "^2.4.1",

packages/pro

@@ -1 +1 @@
-Subproject commit ca41b937a80e38dc1ff4746b555cbb0e673a75ad
+Subproject commit c31758c33fdb5533131b9174b5a54be18f8f03ee

packages/server/jest.config.ts

@@ -2,10 +2,8 @@ import { Config } from "@jest/types"
 
 import * as fs from "fs"
 import { join } from "path"
-const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
-  ...preset,
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./src/tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./src/tests/jestSetup.ts"],

packages/server/package.json

@@ -131,15 +131,15 @@
     "@babel/core": "7.17.4",
     "@babel/preset-env": "7.16.11",
     "@budibase/standard-components": "^0.9.139",
-    "@jest/test-sequencer": "29.5.0",
-    "@swc/core": "^1.3.25",
-    "@swc/jest": "^0.2.24",
-    "@trendyol/jest-testcontainers": "^2.1.1",
+    "@jest/test-sequencer": "29.6.2",
+    "@swc/core": "1.3.71",
+    "@swc/jest": "0.2.27",
+    "@trendyol/jest-testcontainers": "2.1.1",
     "@types/apidoc": "0.50.0",
     "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
-    "@types/jest": "29.5.0",
+    "@types/jest": "29.5.3",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
     "@types/lodash": "4.14.180",
@@ -155,15 +155,15 @@
     "@types/tar": "6.1.5",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
-    "babel-jest": "29.5.0",
+    "babel-jest": "29.6.2",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "eslint": "6.8.0",
     "is-wsl": "2.2.0",
-    "jest": "29.5.0",
+    "jest": "29.6.2",
     "jest-openapi": "0.14.2",
-    "jest-runner": "29.5.0",
-    "jest-serial-runner": "^1.2.1",
+    "jest-runner": "29.6.2",
+    "jest-serial-runner": "1.2.1",
     "nodemon": "2.0.15",
     "openapi-types": "9.3.1",
     "openapi-typescript": "5.2.0",
@@ -172,7 +172,6 @@
     "supertest": "6.2.2",
     "swagger-jsdoc": "6.1.0",
     "timekeeper": "2.2.0",
-    "ts-jest": "29.0.5",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3",

packages/server/src/api/controllers/row/staticFormula.ts

@@ -9,7 +9,7 @@ import { context } from "@budibase/backend-core"
 import { Table, Row } from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
 import sdk from "../../../sdk"
-import { isEqual } from "lodash"
+import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
 
 /**

packages/server/src/api/controllers/table/bulkFormula.ts

@@ -2,7 +2,8 @@ import { FieldTypes, FormulaTypes } from "../../../constants"
 import { clearColumns } from "./utils"
 import { doesContainStrings } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
-import { isEqual, uniq } from "lodash"
+import isEqual from "lodash/isEqual"
+import uniq from "lodash/uniq"
 import { updateAllFormulasInTable } from "../row/staticFormula"
 import { context } from "@budibase/backend-core"
 import { FieldSchema, Table } from "@budibase/types"

packages/server/src/api/controllers/table/internal.ts

@@ -11,7 +11,7 @@ import {
 import { runStaticFormulaChecks } from "./bulkFormula"
 import { Table } from "@budibase/types"
 import { quotas } from "@budibase/pro"
-import { isEqual } from "lodash"
+import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
 import sdk from "../../../sdk"
 

packages/server/src/api/controllers/table/utils.ts

@@ -1,6 +1,6 @@
 import { parse, isSchema, isRows } from "../../../utilities/schema"
 import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
-import { isEqual } from "lodash"
+import isEqual from "lodash/isEqual"
 import {
   AutoFieldSubTypes,
   FieldTypes,

packages/server/src/api/controllers/view/views.ts

@@ -17,7 +17,8 @@ import {
 } from "@budibase/types"
 import { builderSocket } from "../../../websockets"
 
-const { cloneDeep, isEqual } = require("lodash")
+const cloneDeep = require("lodash/cloneDeep")
+import isEqual from "lodash/isEqual"
 
 export async function fetch(ctx: Ctx) {
   ctx.body = await getViews()

packages/server/src/automations/tests/automation.spec.ts

@@ -4,6 +4,7 @@ jest.mock("../../utilities/redis", () => ({
   checkTestFlag: () => {
     return false
   },
+  shutdown: jest.fn(),
 }))
 
 jest.spyOn(global.console, "error")

packages/server/src/db/dynamoClient.ts

@@ -1,4 +1,4 @@
-import { merge } from "lodash"
+import merge from "lodash/merge"
 import env from "../environment"
 
 export const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"

packages/server/src/db/linkedRows/index.ts

@@ -8,10 +8,10 @@ import {
   getLinkedTableIDs,
   getLinkedTable,
 } from "./linkUtils"
-import { flatten } from "lodash"
+import flatten from "lodash/flatten"
 import { FieldTypes } from "../../constants"
 import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
-import { partition } from "lodash"
+import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"

packages/server/src/integrations/index.ts

@@ -17,7 +17,7 @@ import oracle from "./oracle"
 import { SourceName, Integration, PluginType } from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
 import env from "../environment"
-import { cloneDeep } from "lodash"
+import cloneDeep from "lodash/cloneDeep"
 import sdk from "../sdk"
 
 const DEFINITIONS: Record<SourceName, Integration | undefined> = {
@@ -54,7 +54,6 @@ const INTEGRATIONS: Record<SourceName, any> = {
   [SourceName.FIRESTORE]: firebase.integration,
   [SourceName.GOOGLE_SHEETS]: googlesheets.integration,
   [SourceName.REDIS]: redis.integration,
-  [SourceName.FIRESTORE]: firebase.integration,
   [SourceName.SNOWFLAKE]: snowflake.integration,
   [SourceName.ORACLE]: undefined,
 }

packages/server/src/integrations/rest.ts

@@ -11,7 +11,7 @@ import {
   RestBasicAuthConfig,
   RestBearerAuthConfig,
 } from "@budibase/types"
-import { get } from "lodash"
+import get from "lodash/get"
 import * as https from "https"
 import qs from "querystring"
 import fetch from "node-fetch"

packages/server/src/sdk/app/datasources/datasources.ts

@@ -15,7 +15,7 @@ import {
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
 import { getDefinitions, getDefinition } from "../../../integrations"
-import _ from "lodash"
+import merge from "lodash/merge"
 import {
   BudibaseInternalDB,
   getDatasourceParams,
@@ -227,7 +227,7 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
   }
 
   if (old.config?.auth) {
-    update.config = _.merge(old.config, update.config)
+    update.config = merge(old.config, update.config)
   }
 
   // update back to actual passwords for everything else

packages/server/src/sdk/app/tables/index.ts

@@ -14,7 +14,6 @@ import {
 import datasources from "../datasources"
 import { populateExternalTableSchemas, isEditableColumn } from "./validation"
 import sdk from "../../../sdk"
-import _ from "lodash"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {

packages/server/src/sdk/app/views/index.ts

@@ -3,7 +3,7 @@ import { TableSchema, UIFieldMetadata, View, ViewV2 } from "@budibase/types"
 
 import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
-import _ from "lodash"
+import merge from "lodash/merge"
 
 export async function get(viewId: string): Promise<ViewV2 | undefined> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
@@ -103,7 +103,7 @@ function enrichViewV2Schema(
       delete tableFieldSchema.order
     }
 
-    result[columnName] = _.merge(tableFieldSchema, columnUIMetadata)
+    result[columnName] = merge(tableFieldSchema, columnUIMetadata)
   }
   return result
 }

packages/server/src/sdk/users/utils.ts

@@ -6,7 +6,7 @@ import {
   getUserMetadataParams,
   InternalTables,
 } from "../../db/utils"
-import { isEqual } from "lodash"
+import isEqual from "lodash/isEqual"
 import { ContextUser, UserMetadata, User, Database } from "@budibase/types"
 
 export function combineMetadataAndUser(

packages/server/src/utilities/global.ts

@@ -10,7 +10,7 @@ import {
 import env from "../environment"
 import { groups } from "@budibase/pro"
 import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
-import { cloneDeep } from "lodash"
+import cloneDeep from "lodash/cloneDeep"
 
 export function updateAppRole(
   user: ContextUser,

packages/string-templates/package.json

@@ -36,8 +36,8 @@
     "@rollup/plugin-commonjs": "^17.1.0",
     "@rollup/plugin-json": "^4.1.0",
     "doctrine": "^3.0.0",
-    "jest": "^26.6.3",
-    "jest-environment-node": "^26.6.2",
+    "jest": "29.6.2",
+    "jest-environment-node": "29.6.2",
     "marked": "^4.0.10",
     "rollup": "^2.36.2",
     "rollup-plugin-inject-process-env": "^1.3.1",

packages/worker/jest.config.ts

@@ -1,9 +1,7 @@
 import { Config } from "@jest/types"
 import * as fs from "fs"
-const preset = require("ts-jest/jest-preset")
 
 const config: Config.InitialOptions = {
-  ...preset,
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./src/tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./src/tests/jestSetup.ts"],

packages/worker/package.json

@@ -74,10 +74,10 @@
     "server-destroy": "1.0.1"
   },
   "devDependencies": {
-    "@swc/core": "^1.3.25",
-    "@swc/jest": "^0.2.24",
-    "@trendyol/jest-testcontainers": "^2.1.1",
-    "@types/jest": "28.1.1",
+    "@swc/core": "1.3.71",
+    "@swc/jest": "0.2.27",
+    "@trendyol/jest-testcontainers": "2.1.1",
+    "@types/jest": "29.5.3",
     "@types/jsonwebtoken": "8.5.1",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
@@ -91,14 +91,13 @@
     "@typescript-eslint/parser": "5.45.0",
     "copyfiles": "2.4.1",
     "eslint": "6.8.0",
-    "jest": "28.1.1",
+    "jest": "29.6.2",
     "lodash": "4.17.21",
     "nodemon": "2.0.15",
     "pouchdb-adapter-memory": "7.2.2",
     "rimraf": "3.0.2",
     "supertest": "6.2.2",
     "timekeeper": "2.2.0",
-    "ts-jest": "28.0.4",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3",

packages/worker/src/api/routes/global/tests/scim.spec.ts

@@ -314,7 +314,7 @@ describe("scim", () => {
 
           const user = await config.getUser(email)
           expect(user).toBeDefined()
-          expect(user.email).toEqual(email)
+          expect(user!.email).toEqual(email)
         })
 
         it("if multiple emails are provided, the first primary one is used as email", async () => {
@@ -345,7 +345,7 @@ describe("scim", () => {
 
           const user = await config.getUser(email)
           expect(user).toBeDefined()
-          expect(user.email).toEqual(email)
+          expect(user!.email).toEqual(email)
         })
 
         it("if no email is provided and the user name is not an email, an exception is thrown", async () => {

qa-core/package.json

@@ -24,18 +24,18 @@
   },
   "devDependencies": {
     "@budibase/types": "^2.3.17",
-    "@trendyol/jest-testcontainers": "^2.1.1",
-    "@types/jest": "29.0.0",
+    "@trendyol/jest-testcontainers": "2.1.1",
+    "@types/jest": "29.5.3",
     "@types/node-fetch": "2.6.2",
     "chance": "1.1.8",
     "dotenv": "16.0.1",
-    "jest": "29.0.0",
+    "jest": "29.6.2",
     "prettier": "2.7.1",
     "start-server-and-test": "1.14.0",
-    "@swc/core": "^1.3.25",
-    "@swc/jest": "^0.2.24",
+    "@swc/core": "1.3.71",
+    "@swc/jest": "0.2.27",
     "timekeeper": "2.2.0",
-    "ts-jest": "29.0.0",
+    "ts-jest": "29.1.1",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"