From 4c873b9921e2645c6f43614df81996e0c3c32da9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 22:38:16 +0100 Subject: [PATCH 01/22] Attempting to fix some potential app migration issues around versions. --- .../src/appMigrations/appMigrationMetadata.ts | 14 ++++++++------ packages/server/src/appMigrations/index.ts | 5 ++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index d87ddff3ef..613f46cf3d 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -1,4 +1,4 @@ -import { Duration, cache, context, db, env } from "@budibase/backend-core" +import { Duration, cache, db, env } from "@budibase/backend-core" import { Database, DocumentType, Document } from "@budibase/types" export interface AppMigrationDoc extends Document { @@ -42,7 +42,10 @@ export async function getAppMigrationVersion(appId: string): Promise { version = "" } - await cache.store(cacheKey, version, EXPIRY_SECONDS) + // only cache if we have a valid version + if (version) { + await cache.store(cacheKey, version, EXPIRY_SECONDS) + } return version } @@ -54,8 +57,7 @@ export async function updateAppMigrationMetadata({ appId: string version: string }): Promise { - const db = context.getAppDB() - + const appDb = db.getDB(appId) let appMigrationDoc: AppMigrationDoc try { @@ -70,7 +72,7 @@ export async function updateAppMigrationMetadata({ version: "", history: {}, } - await db.put(appMigrationDoc) + await appDb.put(appMigrationDoc) appMigrationDoc = await getFromDB(appId) } @@ -82,7 +84,7 @@ export async function updateAppMigrationMetadata({ [version]: { runAt: new Date().toISOString() }, }, } - await db.put(updatedMigrationDoc) + await appDb.put(updatedMigrationDoc) const cacheKey = getCacheKey(appId) diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index 89c71ae26f..a24bf9c0a3 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -16,7 +16,10 @@ export type AppMigration = { export function getLatestEnabledMigrationId(migrations?: AppMigration[]) { let latestMigrationId: string | undefined - for (let migration of migrations || MIGRATIONS) { + if (!migrations) { + migrations = MIGRATIONS + } + for (let migration of migrations) { // if a migration is disabled, all migrations after it are disabled if (migration.disabled) { break From 8c1735a1bd7068e127ef80ac9675c5775f79d8a5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 22:58:28 +0100 Subject: [PATCH 02/22] Adding concurrency, and changing how context is set. --- .../src/appMigrations/migrationsProcessor.ts | 69 +++++++++---------- packages/server/src/appMigrations/queue.ts | 13 ++-- packages/server/src/startup/index.ts | 3 +- 3 files changed, 40 insertions(+), 45 deletions(-) diff --git a/packages/server/src/appMigrations/migrationsProcessor.ts b/packages/server/src/appMigrations/migrationsProcessor.ts index 0337fc09f4..1441388564 100644 --- a/packages/server/src/appMigrations/migrationsProcessor.ts +++ b/packages/server/src/appMigrations/migrationsProcessor.ts @@ -13,8 +13,8 @@ export async function processMigrations( ) { console.log(`Processing app migration for "${appId}"`) try { - // have to wrap in context, this gets the tenant from the app ID - await context.doInAppContext(appId, async () => { + // first step - setup full context - tenancy, app and guards + await context.doInAppMigrationContext(appId, async () => { console.log(`Acquiring app migration lock for "${appId}"`) await locks.doWithLock( { @@ -23,48 +23,45 @@ export async function processMigrations( resource: appId, }, async () => { - await context.doInAppMigrationContext(appId, async () => { - console.log(`Lock acquired starting app migration for "${appId}"`) - let currentVersion = await getAppMigrationVersion(appId) + console.log(`Lock acquired starting app migration for "${appId}"`) + let currentVersion = await getAppMigrationVersion(appId) - const pendingMigrations = migrations - .filter(m => m.id > currentVersion) - .sort((a, b) => a.id.localeCompare(b.id)) + const pendingMigrations = migrations + .filter(m => m.id > currentVersion) + .sort((a, b) => a.id.localeCompare(b.id)) - const migrationIds = migrations.map(m => m.id).sort() - console.log( - `App migrations to run for "${appId}" - ${migrationIds.join(",")}` - ) + const migrationIds = migrations.map(m => m.id).sort() + console.log( + `App migrations to run for "${appId}" - ${migrationIds.join(",")}` + ) - let index = 0 - for (const { id, func } of pendingMigrations) { - const expectedMigration = - migrationIds[migrationIds.indexOf(currentVersion) + 1] + let index = 0 + for (const { id, func } of pendingMigrations) { + const expectedMigration = + migrationIds[migrationIds.indexOf(currentVersion) + 1] - if (expectedMigration !== id) { - throw new Error( - `Migration ${id} could not run, update for "${id}" is running but ${expectedMigration} is expected` - ) - } - - const counter = `(${++index}/${pendingMigrations.length})` - console.info(`Running migration ${id}... ${counter}`, { - migrationId: id, - appId, - }) - await func() - await updateAppMigrationMetadata({ - appId, - version: id, - }) - currentVersion = id + if (expectedMigration !== id) { + throw new Error( + `Migration ${id} could not run, update for "${id}" is running but ${expectedMigration} is expected` + ) } - }) + + const counter = `(${++index}/${pendingMigrations.length})` + console.info(`Running migration ${id}... ${counter}`, { + migrationId: id, + appId, + }) + await func() + await updateAppMigrationMetadata({ + appId, + version: id, + }) + currentVersion = id + } } ) - - console.log(`App migration for "${appId}" processed`) }) + console.log(`App migration for "${appId}" processed`) } catch (err) { logging.logAlert("Failed to run app migration", err) throw err diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts index 5c932bcb7f..e2bc4406f1 100644 --- a/packages/server/src/appMigrations/queue.ts +++ b/packages/server/src/appMigrations/queue.ts @@ -2,9 +2,10 @@ import { queue, logging } from "@budibase/backend-core" import { Job } from "bull" import { MIGRATIONS } from "./migrations" import { processMigrations } from "./migrationsProcessor" -import { apiEnabled } from "../features" -const MAX_ATTEMPTS = 1 +const MAX_ATTEMPTS = 3 +// max number of migrations to run at same time, per node +const MIGRATION_CONCURRENCY = 5 export type AppMigrationJob = { appId: string @@ -13,10 +14,6 @@ export type AppMigrationJob = { let appMigrationQueue: queue.Queue | undefined export function init() { - // only run app migrations in main API services - if (!apiEnabled()) { - return - } appMigrationQueue = queue.createQueue( queue.JobQueue.APP_MIGRATION, { @@ -34,10 +31,10 @@ export function init() { } ) - return appMigrationQueue.process(processMessage) + return appMigrationQueue.process(MIGRATION_CONCURRENCY, processMessage) } -async function processMessage(job: Job) { +async function processMessage(job: Job) { const { appId } = job.data await processMigrations(appId, MIGRATIONS) diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts index 750acdb0aa..c14ec0ca1b 100644 --- a/packages/server/src/startup/index.ts +++ b/packages/server/src/startup/index.ts @@ -115,8 +115,9 @@ export async function startup( // configure events to use the pro audit log write // can't integrate directly into backend-core due to cyclic issues queuePromises.push(events.processors.init(pro.sdk.auditLogs.write)) - queuePromises.push(appMigrations.init()) + // app migrations and automations on other service if (automationsEnabled()) { + queuePromises.push(appMigrations.init()) queuePromises.push(automations.init()) } queuePromises.push(initPro()) From 553c2186b1b30e1694bd66222e8519b2057dbb28 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 23:01:39 +0100 Subject: [PATCH 03/22] Only try to lookup migrations if there are migrations to work with. --- packages/server/src/appMigrations/index.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index 5aa28e14db..744f5a328f 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -38,8 +38,14 @@ export async function checkMissingMigrations( next: Next, appId: string ) { - const currentVersion = await getAppMigrationVersion(appId) const latestMigration = getLatestEnabledMigrationId() + + // no migrations set - edge case, don't try to do anything + if (!latestMigration) { + return + } + + const currentVersion = await getAppMigrationVersion(appId) const queue = getAppMigrationQueue() if ( From d0736cbe9e46ad70248aa6b39df57f12eee0db5a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 23:11:53 +0100 Subject: [PATCH 04/22] Missing next(). --- packages/server/src/appMigrations/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index 744f5a328f..de15666215 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -42,7 +42,7 @@ export async function checkMissingMigrations( // no migrations set - edge case, don't try to do anything if (!latestMigration) { - return + return next() } const currentVersion = await getAppMigrationVersion(appId) From 75c3b842ade3c28d08687b816e68bb72b3160ccb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 23:34:08 +0100 Subject: [PATCH 05/22] Fixing issue with in memory queue. --- packages/backend-core/src/queue/inMemoryQueue.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 333accc985..62b971f9f5 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -63,12 +63,12 @@ class InMemoryQueue implements Partial { * Same callback API as Bull, each callback passed to this will consume messages as they are * available. Please note this is a queue service, not a notification service, so each * consumer will receive different messages. - * @param func The callback function which will return a "Job", the same * as the Bull API, within this job the property "data" contains the JSON message. Please * note this is incredibly limited compared to Bull as in reality the Job would contain * a lot more information about the queue and current status of Bull cluster. */ - async process(func: any) { + async process(concurrencyOrFunc: number | any, func?: any) { + func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc this._emitter.on("message", async () => { if (this._messages.length <= 0) { return From 66789888ace6c636aa63e1db930d7f72ed83bfe1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Jun 2024 11:23:52 +0100 Subject: [PATCH 06/22] Caching is making testing difficult - want to confirm app migrations are doing what they are expected to do --- packages/server/src/appMigrations/appMigrationMetadata.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index 613f46cf3d..54edd826c3 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -26,9 +26,9 @@ export async function getAppMigrationVersion(appId: string): Promise { let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey) // We don't want to cache in dev, in order to be able to tweak it - if (metadata && !env.isDev()) { - return metadata.version - } + // if (metadata && !env.isDev()) { + // return metadata.version + // } let version try { From 334334bbcdd5057fff4edc11bccbebc74dc2cd39 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Jun 2024 12:13:18 +0100 Subject: [PATCH 07/22] Adding a check for QA. --- packages/backend-core/src/environment.ts | 6 ++++++ packages/server/src/appMigrations/appMigrationMetadata.ts | 8 ++++---- packages/server/src/startup/index.ts | 6 +++++- packages/worker/src/environment.ts | 1 + packages/worker/src/index.ts | 6 +++++- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 1e7da2f9a2..c84a162ab6 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -93,15 +93,21 @@ function isApps() { return environment.SERVICE_TYPE === ServiceType.APPS } +function isQA() { + return environment.BUDIBASE_ENVIRONMENT === "QA" +} + const environment = { isTest, isJest, isDev, isWorker, isApps, + isQA, isProd: () => { return !isDev() }, + BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, JS_BCRYPT: process.env.JS_BCRYPT, JWT_SECRET: process.env.JWT_SECRET, JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK, diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index 54edd826c3..a9a75bc104 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -25,10 +25,10 @@ export async function getAppMigrationVersion(appId: string): Promise { let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey) - // We don't want to cache in dev, in order to be able to tweak it - // if (metadata && !env.isDev()) { - // return metadata.version - // } + // We don't want to cache in dev or QA, in order to be able to tweak it + if (metadata && !env.isDev() && !env.isQA()) { + return metadata.version + } let version try { diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts index c14ec0ca1b..7f244f6573 100644 --- a/packages/server/src/startup/index.ts +++ b/packages/server/src/startup/index.ts @@ -72,7 +72,11 @@ export async function startup( printFeatures() STARTUP_RAN = true if (app && server && !env.CLUSTER_MODE) { - console.log(`Budibase running on ${JSON.stringify(server.address())}`) + let startupLog = `Budibase running on ${JSON.stringify(server.address())}` + if (env.BUDIBASE_ENVIRONMENT) { + startupLog = `${startupLog} - environment: "${env.BUDIBASE_ENVIRONMENT}"` + } + console.log(startupLog) const address = server.address() as AddressInfo env._set("PORT", address.port) } diff --git a/packages/worker/src/environment.ts b/packages/worker/src/environment.ts index 70fb911ee1..d0f5e1fb67 100644 --- a/packages/worker/src/environment.ts +++ b/packages/worker/src/environment.ts @@ -46,6 +46,7 @@ const environment = { SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, + BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, // smtp SMTP_USER: process.env.SMTP_USER, SMTP_PASSWORD: process.env.SMTP_PASSWORD, diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 4e770c6ecb..85e5d6ad2e 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -88,7 +88,11 @@ const shutdown = () => { } export default server.listen(parseInt(env.PORT || "4002"), async () => { - console.log(`Worker running on ${JSON.stringify(server.address())}`) + let startupLog = `Worker running on ${JSON.stringify(server.address())}` + if (env.BUDIBASE_ENVIRONMENT) { + startupLog = `${startupLog} - environment: "${env.BUDIBASE_ENVIRONMENT}"` + } + console.log(startupLog) await initPro() await redis.clients.init() cache.docWritethrough.init() From 115737c46cfd5e0c2507d9d895ddb3ae44bf33a2 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Tue, 11 Jun 2024 12:16:15 +0100 Subject: [PATCH 08/22] Create Tenant endpoint + tenant_info doc (#13902) * Create Tenant endpoint + tenant_info doc * Don't catch on tenant_info put * PR comments * unit test --- packages/backend-core/src/tenancy/db.ts | 10 ++++ packages/types/src/documents/global/index.ts | 1 + .../types/src/documents/global/tenantInfo.ts | 13 +++++ .../src/api/controllers/global/tenant.ts | 10 ++++ packages/worker/src/api/index.ts | 8 ++++ .../worker/src/api/routes/global/tenant.ts | 33 +++++++++++++ .../api/routes/global/tests/tenant.spec.ts | 47 +++++++++++++++++++ packages/worker/src/api/routes/index.ts | 2 + packages/worker/src/tests/api/tenants.ts | 9 ++++ yarn.lock | 24 +++++----- 10 files changed, 145 insertions(+), 12 deletions(-) create mode 100644 packages/types/src/documents/global/tenantInfo.ts create mode 100644 packages/worker/src/api/controllers/global/tenant.ts create mode 100644 packages/worker/src/api/routes/global/tenant.ts create mode 100644 packages/worker/src/api/routes/global/tests/tenant.spec.ts diff --git a/packages/backend-core/src/tenancy/db.ts b/packages/backend-core/src/tenancy/db.ts index 10477a8579..f2e4705fa8 100644 --- a/packages/backend-core/src/tenancy/db.ts +++ b/packages/backend-core/src/tenancy/db.ts @@ -1,6 +1,16 @@ import { getDB } from "../db/db" import { getGlobalDBName } from "../context" +import { TenantInfo } from "@budibase/types" export function getTenantDB(tenantId: string) { return getDB(getGlobalDBName(tenantId)) } + +export async function saveTenantInfo(tenantInfo: TenantInfo) { + const db = getTenantDB(tenantInfo.tenantId) + // save the tenant info to db + return await db.put({ + _id: "tenant_info", + ...tenantInfo, + }) +} diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts index b728439dd6..6784f2638c 100644 --- a/packages/types/src/documents/global/index.ts +++ b/packages/types/src/documents/global/index.ts @@ -7,3 +7,4 @@ export * from "./schedule" export * from "./templates" export * from "./environmentVariables" export * from "./auditLogs" +export * from "./tenantInfo" diff --git a/packages/types/src/documents/global/tenantInfo.ts b/packages/types/src/documents/global/tenantInfo.ts new file mode 100644 index 0000000000..2fa8f4ad96 --- /dev/null +++ b/packages/types/src/documents/global/tenantInfo.ts @@ -0,0 +1,13 @@ +import { Document } from "../document" + +export interface TenantInfo extends Document { + owner: { + email: string + password?: string + ssoId?: string + givenName?: string + familyName?: string + budibaseUserId?: string + } + tenantId: string +} diff --git a/packages/worker/src/api/controllers/global/tenant.ts b/packages/worker/src/api/controllers/global/tenant.ts new file mode 100644 index 0000000000..1e86fb8246 --- /dev/null +++ b/packages/worker/src/api/controllers/global/tenant.ts @@ -0,0 +1,10 @@ +import { tenancy } from "@budibase/backend-core" +import { TenantInfo, Ctx } from "@budibase/types" + +export const save = async (ctx: Ctx) => { + const response = await tenancy.saveTenantInfo(ctx.request.body) + ctx.body = { + _id: response.id, + _rev: response.rev, + } +} diff --git a/packages/worker/src/api/index.ts b/packages/worker/src/api/index.ts index 82495df4ee..feec8b4de1 100644 --- a/packages/worker/src/api/index.ts +++ b/packages/worker/src/api/index.ts @@ -76,6 +76,10 @@ const PUBLIC_ENDPOINTS = [ route: "/api/global/users/invite", method: "GET", }, + { + route: "/api/global/tenant", + method: "POST", + }, ] const NO_TENANCY_ENDPOINTS = [ @@ -121,6 +125,10 @@ const NO_TENANCY_ENDPOINTS = [ route: "/api/global/users/invite/:code", method: "GET", }, + { + route: "/api/global/tenant", + method: "POST", + }, ] // most public endpoints are gets, but some are posts diff --git a/packages/worker/src/api/routes/global/tenant.ts b/packages/worker/src/api/routes/global/tenant.ts new file mode 100644 index 0000000000..7179532cde --- /dev/null +++ b/packages/worker/src/api/routes/global/tenant.ts @@ -0,0 +1,33 @@ +import Router from "@koa/router" +import Joi from "joi" +import { auth } from "@budibase/backend-core" +import * as controller from "../../controllers/global/tenant" +import cloudRestricted from "../../../middleware/cloudRestricted" + +const router: Router = new Router() +const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("") + +function buildTenantInfoValidation() { + return auth.joiValidator.body( + Joi.object({ + owner: Joi.object({ + email: Joi.string().required(), + password: OPTIONAL_STRING, + ssoId: OPTIONAL_STRING, + givenName: OPTIONAL_STRING, + familyName: OPTIONAL_STRING, + budibaseUserId: OPTIONAL_STRING, + }).required(), + tenantId: Joi.string().required(), + }).required() + ) +} + +router.post( + "/api/global/tenant", + cloudRestricted, + buildTenantInfoValidation(), + controller.save +) + +export default router diff --git a/packages/worker/src/api/routes/global/tests/tenant.spec.ts b/packages/worker/src/api/routes/global/tests/tenant.spec.ts new file mode 100644 index 0000000000..36036eceee --- /dev/null +++ b/packages/worker/src/api/routes/global/tests/tenant.spec.ts @@ -0,0 +1,47 @@ +import { TenantInfo } from "@budibase/types" +import { TestConfiguration } from "../../../../tests" +import { tenancy as _tenancy } from "@budibase/backend-core" + +const tenancy = jest.mocked(_tenancy) + +describe("/api/global/tenant", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + beforeEach(() => { + jest.clearAllMocks() + }) + + describe("POST /api/global/tenant", () => { + it("should save the tenantInfo", async () => { + tenancy.saveTenantInfo = jest.fn().mockImplementation(async () => ({ + id: "DOC_ID", + ok: true, + rev: "DOC_REV", + })) + const tenantInfo: TenantInfo = { + owner: { + email: "test@example.com", + password: "PASSWORD", + ssoId: "SSO_ID", + givenName: "Jane", + familyName: "Doe", + budibaseUserId: "USER_ID", + }, + tenantId: "tenant123", + } + const response = await config.api.tenants.saveTenantInfo(tenantInfo) + + expect(_tenancy.saveTenantInfo).toHaveBeenCalledTimes(1) + expect(_tenancy.saveTenantInfo).toHaveBeenCalledWith(tenantInfo) + expect(response.text).toEqual('{"_id":"DOC_ID","_rev":"DOC_REV"}') + }) + }) +}) diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index e6cacf110f..2eb4b5cd5d 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -1,6 +1,7 @@ import Router from "@koa/router" import { api as pro } from "@budibase/pro" import userRoutes from "./global/users" +import tenantRoutes from "./global/tenant" import configRoutes from "./global/configs" import workspaceRoutes from "./global/workspaces" import templateRoutes from "./global/templates" @@ -40,6 +41,7 @@ export const routes: Router[] = [ accountRoutes, restoreRoutes, eventRoutes, + tenantRoutes, pro.scim, ] diff --git a/packages/worker/src/tests/api/tenants.ts b/packages/worker/src/tests/api/tenants.ts index 16f970915a..c404b8ad58 100644 --- a/packages/worker/src/tests/api/tenants.ts +++ b/packages/worker/src/tests/api/tenants.ts @@ -1,3 +1,4 @@ +import { TenantInfo } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI, TestAPIOpts } from "./base" @@ -14,4 +15,12 @@ export class TenantAPI extends TestAPI { .set(opts?.headers) .expect(opts?.status ? opts.status : 204) } + + saveTenantInfo = (tenantInfo: TenantInfo) => { + return this.request + .post("/api/global/tenant") + .set(this.config.internalAPIHeaders()) + .send(tenantInfo) + .expect(200) + } } diff --git a/yarn.lock b/yarn.lock index 5297fe0cad..e0e1229fbf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3483,10 +3483,10 @@ dependencies: lodash "^4.17.21" -"@koa/cors@^3.1.0": - version "3.4.3" - resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-3.4.3.tgz#d669ee6e8d6e4f0ec4a7a7b0a17e7a3ed3752ebb" - integrity sha512-WPXQUaAeAMVaLTEFpoq3T2O1C+FstkjJnDQqy95Ck1UdILajsRhu6mhJ8H2f4NFPRBoCNN+qywTJfq/gGki5mw== +"@koa/cors@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-5.0.0.tgz#0029b5f057fa0d0ae0e37dd2c89ece315a0daffd" + integrity sha512-x/iUDjcS90W69PryLDIMgFyV21YLTnG9zOpPXS7Bkt2b8AsY3zZsIpOLBkYr9fBcF3HbkKaER5hOBZLfpLgYNw== dependencies: vary "^1.1.2" @@ -5797,10 +5797,10 @@ "@types/koa-compose" "*" "@types/node" "*" -"@types/koa__cors@^3.1.1": - version "3.3.1" - resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-3.3.1.tgz#0ec7543c4c620fd23451bfdd3e21b9a6aadedccd" - integrity sha512-aFGYhTFW7651KhmZZ05VG0QZJre7QxBxDj2LF1lf6GA/wSXEfKVAJxiQQWzRV4ZoMzQIO8vJBXKsUcRuvYK9qw== +"@types/koa__cors@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-5.0.0.tgz#74567a045b599266e2cd3940cef96cedecc2ef1f" + integrity sha512-LCk/n25Obq5qlernGOK/2LUwa/2YJb2lxHUkkvYFDOpLXlVI6tKcdfCHRBQnOY4LwH6el5WOLs6PD/a8Uzau6g== dependencies: "@types/koa" "*" @@ -16299,10 +16299,10 @@ node-source-walk@^5.0.0: dependencies: "@babel/parser" "^7.0.0" -nodemailer@6.7.2: - version "6.7.2" - resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.2.tgz#44b2ad5f7ed71b7067f7a21c4fedabaec62b85e0" - integrity sha512-Dz7zVwlef4k5R71fdmxwR8Q39fiboGbu3xgswkzGwczUfjp873rVxt1O46+Fh0j1ORnAC6L9+heI8uUpO6DT7Q== +nodemailer@6.9.13: + version "6.9.13" + resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.13.tgz#5b292bf1e92645f4852ca872c56a6ba6c4a3d3d6" + integrity sha512-7o38Yogx6krdoBf3jCAqnIN4oSQFx+fMa0I7dK1D+me9kBxx12D+/33wSb+fhOCtIxvYJ+4x4IMEhmhCKfAiOA== nodemailer@6.9.9: version "6.9.9" From ae74c2498c23a1152b9b76d25717bc74c31ea75f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 11 Jun 2024 14:38:56 +0100 Subject: [PATCH 09/22] small change to ensure that name of datasources is available in dropdown to prevent dupes --- .../controls/DataSourceSelect/DataSourceCategory.svelte | 4 ++-- .../controls/DataSourceSelect/DataSourceSelect.svelte | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte index 71de1716f2..6d0e5ef9e7 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte @@ -28,8 +28,8 @@ tabindex="0" on:click={() => onSelect(data)} > - - {data.label} + + {data.datasource?.name ? `${data.datasource.name} -` : ""} {data.label} ds._id === m.sourceId || m.datasourceId), })) $: viewsV1 = $viewsStore.list.map(view => ({ ...view, From 1d1ca694c83f8da2ed709f0de3c4d70cad710564 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Jun 2024 14:39:44 +0100 Subject: [PATCH 10/22] Lock starting containers. --- .gitignore | 2 + .../src/integrations/tests/utils/index.ts | 53 ++++++++++++++----- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index b68ddd975f..32d1416f4a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ bb-airgapped.tar.gz packages/server/build/oldClientVersions/**/* packages/builder/src/components/deploy/clientVersions.json +packages/server/src/integrations/tests/utils/*.lock + # Logs logs *.log diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index a54d0ac1a7..64e466e6a0 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -6,6 +6,9 @@ import * as mssql from "./mssql" import * as mariadb from "./mariadb" import { GenericContainer } from "testcontainers" import { testContainerUtils } from "@budibase/backend-core/tests" +import lockfile from "proper-lockfile" +import path from "path" +import fs from "fs" export type DatasourceProvider = () => Promise @@ -67,20 +70,44 @@ export async function rawQuery(ds: Datasource, sql: string): Promise { export async function startContainer(container: GenericContainer) { container = container.withReuse().withLabels({ "com.budibase": "true" }) - const startedContainer = await container.start() + // If two tests try to spin up the same container at the same time, there's a + // possibility that two containers of the same type will be started. To avoid + // this, we use a filesystem lock to ensure that only one container of a given + // type is started at a time. + const imageName = (container as any).imageName.string as string + const lockPath = path.resolve( + __dirname, + `${imageName.replaceAll("/", "-")}.lock` + ) - const info = testContainerUtils.getContainerById(startedContainer.getId()) - if (!info) { - throw new Error("Container not found") + // The `proper-lockfile` library needs the file we're locking on to exist + // before it can lock it. + if (!fs.existsSync(lockPath)) { + fs.writeFileSync(lockPath, "") } - // Some Docker runtimes, when you expose a port, will bind it to both - // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6 - // addresses are not shared, and testcontainers will sometimes give you back - // the ipv6 port. There's no way to know that this has happened, and if you - // try to then connect to `localhost:port` you may attempt to bind to the v4 - // address which could be unbound or even an entirely different container. For - // that reason, we don't use testcontainers' `getExposedPort` function, - // preferring instead our own method that guaranteed v4 ports. - return testContainerUtils.getExposedV4Ports(info) + await lockfile.lock(lockPath, { + retries: 10, + }) + + try { + const startedContainer = await container.start() + + const info = testContainerUtils.getContainerById(startedContainer.getId()) + if (!info) { + throw new Error("Container not found") + } + + // Some Docker runtimes, when you expose a port, will bind it to both + // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6 + // addresses are not shared, and testcontainers will sometimes give you back + // the ipv6 port. There's no way to know that this has happened, and if you + // try to then connect to `localhost:port` you may attempt to bind to the v4 + // address which could be unbound or even an entirely different container. For + // that reason, we don't use testcontainers' `getExposedPort` function, + // preferring instead our own method that guaranteed v4 ports. + return testContainerUtils.getExposedV4Ports(info) + } finally { + await lockfile.unlock(lockPath) + } } From af0802df303b22a277bd3ed070399e0684ad1bf6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Jun 2024 14:43:45 +0100 Subject: [PATCH 11/22] Only hold the lock during container start. --- .../src/integrations/tests/utils/index.ts | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 64e466e6a0..537e523fc6 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -4,7 +4,7 @@ import * as mongodb from "./mongodb" import * as mysql from "./mysql" import * as mssql from "./mssql" import * as mariadb from "./mariadb" -import { GenericContainer } from "testcontainers" +import { GenericContainer, StartedTestContainer } from "testcontainers" import { testContainerUtils } from "@budibase/backend-core/tests" import lockfile from "proper-lockfile" import path from "path" @@ -90,24 +90,25 @@ export async function startContainer(container: GenericContainer) { retries: 10, }) + let startedContainer: StartedTestContainer try { - const startedContainer = await container.start() - - const info = testContainerUtils.getContainerById(startedContainer.getId()) - if (!info) { - throw new Error("Container not found") - } - - // Some Docker runtimes, when you expose a port, will bind it to both - // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6 - // addresses are not shared, and testcontainers will sometimes give you back - // the ipv6 port. There's no way to know that this has happened, and if you - // try to then connect to `localhost:port` you may attempt to bind to the v4 - // address which could be unbound or even an entirely different container. For - // that reason, we don't use testcontainers' `getExposedPort` function, - // preferring instead our own method that guaranteed v4 ports. - return testContainerUtils.getExposedV4Ports(info) + startedContainer = await container.start() } finally { await lockfile.unlock(lockPath) } + + const info = testContainerUtils.getContainerById(startedContainer.getId()) + if (!info) { + throw new Error("Container not found") + } + + // Some Docker runtimes, when you expose a port, will bind it to both + // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6 + // addresses are not shared, and testcontainers will sometimes give you back + // the ipv6 port. There's no way to know that this has happened, and if you + // try to then connect to `localhost:port` you may attempt to bind to the v4 + // address which could be unbound or even an entirely different container. For + // that reason, we don't use testcontainers' `getExposedPort` function, + // preferring instead our own method that guaranteed v4 ports. + return testContainerUtils.getExposedV4Ports(info) } From f8f05a59d46869dc22f93a7917d068ab948a7046 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 11 Jun 2024 14:38:56 +0100 Subject: [PATCH 12/22] small change to ensure that name of datasources is available in dropdown to prevent dupes --- .../controls/DataSourceSelect/DataSourceCategory.svelte | 2 +- .../settings/controls/DataSourceSelect/DataSourceSelect.svelte | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte index 71de1716f2..8e4ce3b4c3 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte @@ -29,7 +29,7 @@ on:click={() => onSelect(data)} > - {data.label} + {data.datasource?.name ? `${data.datasource.name} -` : ""} {data.label} ds._id === m.sourceId || m.datasourceId), })) $: viewsV1 = $viewsStore.list.map(view => ({ ...view, From 2e67ae115e024f74d9872921d88787370e43274d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Jun 2024 15:33:18 +0100 Subject: [PATCH 13/22] Attempt to use unluck we get back from lock. --- packages/server/src/integrations/tests/utils/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 537e523fc6..eaebb8f2d5 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -86,7 +86,7 @@ export async function startContainer(container: GenericContainer) { fs.writeFileSync(lockPath, "") } - await lockfile.lock(lockPath, { + const unlock = await lockfile.lock(lockPath, { retries: 10, }) @@ -94,7 +94,7 @@ export async function startContainer(container: GenericContainer) { try { startedContainer = await container.start() } finally { - await lockfile.unlock(lockPath) + await unlock() } const info = testContainerUtils.getContainerById(startedContainer.getId()) From eac6106b06aced3114ebc42287dbe459ce452357 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Jun 2024 15:38:25 +0100 Subject: [PATCH 14/22] Try the sync versions? --- packages/server/src/integrations/tests/utils/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index eaebb8f2d5..7487ee079c 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -86,7 +86,7 @@ export async function startContainer(container: GenericContainer) { fs.writeFileSync(lockPath, "") } - const unlock = await lockfile.lock(lockPath, { + const unlock = lockfile.lockSync(lockPath, { retries: 10, }) @@ -94,7 +94,7 @@ export async function startContainer(container: GenericContainer) { try { startedContainer = await container.start() } finally { - await unlock() + unlock() } const info = testContainerUtils.getContainerById(startedContainer.getId()) From af60ff4da77d61ad732dbbc4983b0e8cad0dacf0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Jun 2024 15:43:12 +0100 Subject: [PATCH 15/22] Can't use retries with the sync API. --- packages/server/src/integrations/tests/utils/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 7487ee079c..a462cdce05 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -86,9 +86,7 @@ export async function startContainer(container: GenericContainer) { fs.writeFileSync(lockPath, "") } - const unlock = lockfile.lockSync(lockPath, { - retries: 10, - }) + const unlock = lockfile.lockSync(lockPath) let startedContainer: StartedTestContainer try { From 2b2079b4f1233ca8415849f8a9c1ebf18e2cf0c1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Jun 2024 15:45:07 +0100 Subject: [PATCH 16/22] Moving app migrations back to API service. --- .../src/appMigrations/appMigrationMetadata.ts | 4 ++-- packages/server/src/startup/index.ts | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index a9a75bc104..176cd0ff84 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -25,8 +25,8 @@ export async function getAppMigrationVersion(appId: string): Promise { let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey) - // We don't want to cache in dev or QA, in order to be able to tweak it - if (metadata && !env.isDev() && !env.isQA()) { + // We don't want to cache in dev in order to be able to tweak it + if (metadata && !env.isDev()) { return metadata.version } diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts index 7f244f6573..0e08075698 100644 --- a/packages/server/src/startup/index.ts +++ b/packages/server/src/startup/index.ts @@ -20,7 +20,7 @@ import * as pro from "@budibase/pro" import * as api from "../api" import sdk from "../sdk" import { initialise as initialiseWebsockets } from "../websockets" -import { automationsEnabled, printFeatures } from "../features" +import { apiEnabled, automationsEnabled, printFeatures } from "../features" import * as jsRunner from "../jsRunner" import Koa from "koa" import { Server } from "http" @@ -70,13 +70,12 @@ export async function startup( return } printFeatures() + if (env.BUDIBASE_ENVIRONMENT) { + console.log(`service running environment: "${env.BUDIBASE_ENVIRONMENT}"`) + } STARTUP_RAN = true if (app && server && !env.CLUSTER_MODE) { - let startupLog = `Budibase running on ${JSON.stringify(server.address())}` - if (env.BUDIBASE_ENVIRONMENT) { - startupLog = `${startupLog} - environment: "${env.BUDIBASE_ENVIRONMENT}"` - } - console.log(startupLog) + console.log(`Budibase running on ${JSON.stringify(server.address())}`) const address = server.address() as AddressInfo env._set("PORT", address.port) } @@ -121,9 +120,11 @@ export async function startup( queuePromises.push(events.processors.init(pro.sdk.auditLogs.write)) // app migrations and automations on other service if (automationsEnabled()) { - queuePromises.push(appMigrations.init()) queuePromises.push(automations.init()) } + if (apiEnabled()) { + queuePromises.push(appMigrations.init()) + } queuePromises.push(initPro()) if (app) { // bring routes online as final step once everything ready From 6a54b58303002bc42cc62df7511631172554b37b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Jun 2024 16:00:04 +0100 Subject: [PATCH 17/22] ? --- packages/server/specs/openapi.json | 12 +++++++++--- packages/server/specs/openapi.yaml | 6 ++++++ .../server/src/integrations/tests/utils/index.ts | 13 +++++-------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 7d07b424f0..b21554505b 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -860,8 +860,10 @@ "json", "internal", "barcodeqr", + "signature_single", "bigint", - "bb_reference" + "bb_reference", + "bb_reference_single" ], "description": "Defines the type of the column, most explain themselves, a link column is a relationship." }, @@ -1067,8 +1069,10 @@ "json", "internal", "barcodeqr", + "signature_single", "bigint", - "bb_reference" + "bb_reference", + "bb_reference_single" ], "description": "Defines the type of the column, most explain themselves, a link column is a relationship." }, @@ -1285,8 +1289,10 @@ "json", "internal", "barcodeqr", + "signature_single", "bigint", - "bb_reference" + "bb_reference", + "bb_reference_single" ], "description": "Defines the type of the column, most explain themselves, a link column is a relationship." }, diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 3a798c424b..6a2ae89c61 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -782,8 +782,10 @@ components: - json - internal - barcodeqr + - signature_single - bigint - bb_reference + - bb_reference_single description: Defines the type of the column, most explain themselves, a link column is a relationship. constraints: @@ -948,8 +950,10 @@ components: - json - internal - barcodeqr + - signature_single - bigint - bb_reference + - bb_reference_single description: Defines the type of the column, most explain themselves, a link column is a relationship. constraints: @@ -1121,8 +1125,10 @@ components: - json - internal - barcodeqr + - signature_single - bigint - bb_reference + - bb_reference_single description: Defines the type of the column, most explain themselves, a link column is a relationship. constraints: diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index a462cdce05..d9a2960848 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -80,19 +80,16 @@ export async function startContainer(container: GenericContainer) { `${imageName.replaceAll("/", "-")}.lock` ) - // The `proper-lockfile` library needs the file we're locking on to exist - // before it can lock it. - if (!fs.existsSync(lockPath)) { - fs.writeFileSync(lockPath, "") - } - - const unlock = lockfile.lockSync(lockPath) + const unlock = await lockfile.lock(lockPath, { + retries: 10, + realpath: false, + }) let startedContainer: StartedTestContainer try { startedContainer = await container.start() } finally { - unlock() + await unlock() } const info = testContainerUtils.getContainerById(startedContainer.getId()) From c2415012c26edf83306cc301e72291ffd67df287 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Jun 2024 16:02:38 +0100 Subject: [PATCH 18/22] Keep QA check for no caching of app migration info. --- packages/server/src/appMigrations/appMigrationMetadata.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index 176cd0ff84..971dde1e7a 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -25,8 +25,8 @@ export async function getAppMigrationVersion(appId: string): Promise { let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey) - // We don't want to cache in dev in order to be able to tweak it - if (metadata && !env.isDev()) { + // We don't want to cache in dev or QA in order to be able to tweak it + if (metadata && !env.isDev() && !env.isQA()) { return metadata.version } From 9fcefb18b4adfc9da33d7a79d19ced2761e1f996 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 11 Jun 2024 16:09:13 +0100 Subject: [PATCH 19/22] Update packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte Co-authored-by: Andrew Kingston --- .../controls/DataSourceSelect/DataSourceCategory.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte index 8e4ce3b4c3..fa6f477ed9 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte @@ -29,7 +29,7 @@ on:click={() => onSelect(data)} > - {data.datasource?.name ? `${data.datasource.name} -` : ""} {data.label} + {data.datasource?.name ? `${data.datasource.name} - ` : ""}{data.label} Date: Tue, 11 Jun 2024 16:56:35 +0100 Subject: [PATCH 20/22] Fix lint on master --- .../controls/DataSourceSelect/DataSourceSelect.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index a3ea677df9..d2b5174139 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -55,7 +55,9 @@ label: m.name, tableId: m._id, type: "table", - datasource: $datasources.list.find(ds => ds._id === m.sourceId || m.datasourceId), + datasource: $datasources.list.find( + ds => ds._id === m.sourceId || m.datasourceId + ), })) $: viewsV1 = $viewsStore.list.map(view => ({ ...view, From 85c59c0350f87526566e4a071411e8c3d955b29c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Jun 2024 17:41:48 +0100 Subject: [PATCH 21/22] Changing tactic to relying on stable container names to prevent duplication. --- .../src/integrations/tests/utils/index.ts | 49 ++++++++++++------- .../src/integrations/tests/utils/mssql.ts | 3 ++ .../src/integrations/tests/utils/mysql.ts | 3 ++ .../src/integrations/tests/utils/postgres.ts | 3 ++ 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index d9a2960848..afbf4310a8 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -9,6 +9,7 @@ import { testContainerUtils } from "@budibase/backend-core/tests" import lockfile from "proper-lockfile" import path from "path" import fs from "fs" +import _ from "lodash" export type DatasourceProvider = () => Promise @@ -68,28 +69,38 @@ export async function rawQuery(ds: Datasource, sql: string): Promise { } export async function startContainer(container: GenericContainer) { - container = container.withReuse().withLabels({ "com.budibase": "true" }) - - // If two tests try to spin up the same container at the same time, there's a - // possibility that two containers of the same type will be started. To avoid - // this, we use a filesystem lock to ensure that only one container of a given - // type is started at a time. const imageName = (container as any).imageName.string as string - const lockPath = path.resolve( - __dirname, - `${imageName.replaceAll("/", "-")}.lock` - ) + const key = imageName.replaceAll("/", "-").replaceAll(":", "-") - const unlock = await lockfile.lock(lockPath, { - retries: 10, - realpath: false, - }) + container = container + .withReuse() + .withLabels({ "com.budibase": "true" }) + .withName(key) - let startedContainer: StartedTestContainer - try { - startedContainer = await container.start() - } finally { - await unlock() + let startedContainer: StartedTestContainer | undefined = undefined + let lastError = undefined + for (let i = 0; i < 10; i++) { + try { + // container.start() is not an idempotent operation, calling `start` + // modifies the internal state of a GenericContainer instance such that + // the hash it uses to determine reuse changes. We need to clone the + // container before calling start to ensure that we're using the same + // reuse hash every time. + const containerCopy = _.cloneDeep(container) + startedContainer = await containerCopy.start() + lastError = undefined + break + } catch (e: any) { + lastError = e + await new Promise(resolve => setTimeout(resolve, 1000)) + } + } + + if (!startedContainer) { + if (lastError) { + throw lastError + } + throw new Error(`failed to start container: ${imageName}`) } const info = testContainerUtils.getContainerById(startedContainer.getId()) diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index 647f461272..57c5fe8049 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -29,6 +29,9 @@ export async function getDatasource(): Promise { } const port = (await ports).find(x => x.container === 1433)?.host + if (!port) { + throw new Error("SQL Server port not found") + } const datasource: Datasource = { type: "datasource_plus", diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts index a78833e1de..560d6bb2d4 100644 --- a/packages/server/src/integrations/tests/utils/mysql.ts +++ b/packages/server/src/integrations/tests/utils/mysql.ts @@ -38,6 +38,9 @@ export async function getDatasource(): Promise { } const port = (await ports).find(x => x.container === 3306)?.host + if (!port) { + throw new Error("MySQL port not found") + } const datasource: Datasource = { type: "datasource_plus", diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index 4191b107e9..8c0cd886e8 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -21,6 +21,9 @@ export async function getDatasource(): Promise { } const port = (await ports).find(x => x.container === 5432)?.host + if (!port) { + throw new Error("Postgres port not found") + } const datasource: Datasource = { type: "datasource_plus", From 96efb17678e9503f14dfdb98e7e17f060c838b29 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Jun 2024 17:52:02 +0100 Subject: [PATCH 22/22] Fix lint. --- packages/server/src/integrations/tests/utils/index.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index afbf4310a8..64617461bb 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -6,10 +6,7 @@ import * as mssql from "./mssql" import * as mariadb from "./mariadb" import { GenericContainer, StartedTestContainer } from "testcontainers" import { testContainerUtils } from "@budibase/backend-core/tests" -import lockfile from "proper-lockfile" -import path from "path" -import fs from "fs" -import _ from "lodash" +import cloneDeep from "lodash/cloneDeep" export type DatasourceProvider = () => Promise @@ -86,7 +83,7 @@ export async function startContainer(container: GenericContainer) { // the hash it uses to determine reuse changes. We need to clone the // container before calling start to ensure that we're using the same // reuse hash every time. - const containerCopy = _.cloneDeep(container) + const containerCopy = cloneDeep(container) startedContainer = await containerCopy.start() lastError = undefined break