From aadf8ff3b3ebf6005fbd8b01941e77a952f396fe Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 13:55:26 +0100 Subject: [PATCH 1/8] Updating app migration test to not create the app context, relying on the app migration processor to do this for us instead (including tenancy context). --- .../tests/20240604153647_initial_sqs.spec.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index 64420d239f..49aef4c310 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -16,7 +16,17 @@ import { generateLinkID, generateRowID, } from "../../../db/utils" +import { processMigrations } from "../../migrationsProcessor" import migration from "../20240604153647_initial_sqs" +import { AppMigration } from "src/appMigrations" + +const MIGRATIONS: AppMigration[] = [ + { + id: "20240604153647_initial_sqs", + func: migration, + disabled: false, + }, +] const config = setup.getConfig() let tableId: string @@ -91,9 +101,7 @@ describe("SQS migration", () => { expect(error.status).toBe(404) }) await sqsEnabled(async () => { - await context.doInAppContext(config.appId!, async () => { - await migration() - }) + await processMigrations(config.appId!, MIGRATIONS) const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) expect(designDoc.sql.tables).toBeDefined() const mainTableDef = designDoc.sql.tables[tableId] From f1ea6a52aef7d79d0dbdc45990ecc2dda42c8ffb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 14:10:41 +0100 Subject: [PATCH 2/8] Linting. --- .../migrations/tests/20240604153647_initial_sqs.spec.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index 49aef4c310..86e50a5812 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -1,10 +1,6 @@ import * as setup from "../../../api/routes/tests/utilities" import { basicTable } from "../../../tests/utilities/structures" -import { - db as dbCore, - SQLITE_DESIGN_DOC_ID, - context, -} from "@budibase/backend-core" +import { db as dbCore, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" import { LinkDocument, DocumentType, From f6ed2c9afcbf8879261440c0998e21d6210eecc8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 10 Jun 2024 15:48:15 +0100 Subject: [PATCH 3/8] Enable DataDog test instrumentation. --- .github/workflows/budibase_ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 7f1e08601a..eb11627758 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -179,6 +179,13 @@ jobs: - run: yarn --frozen-lockfile - name: Test server + env: + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: "${{ secrets.DATADOG_API_KEY }}" + DD_SITE: "datadoghq.eu" + NODE_OPTIONS: "-r dd-trace/ci/init" + DD_ENV: "ci" + DD_SERVICE: "budibase/packages/server" run: | if ${{ env.USE_NX_AFFECTED }}; then yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} From 13ec629109cc9bdb52543777e099195275726eed Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 10 Jun 2024 15:50:04 +0100 Subject: [PATCH 4/8] Trigger server tests. --- packages/server/src/api/routes/tests/row.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 3ded61207e..428cfb3128 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1683,3 +1683,5 @@ describe.each([ }) }) }) + +// todo: remove me From c328dd5cd43f41872981d619960811bb67db46c2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 18:32:06 +0100 Subject: [PATCH 5/8] Making sure any error that occurs in the app migration system gets logged. --- .../src/appMigrations/migrationsProcessor.ts | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/server/src/appMigrations/migrationsProcessor.ts b/packages/server/src/appMigrations/migrationsProcessor.ts index badd50f53c..d4f474cd30 100644 --- a/packages/server/src/appMigrations/migrationsProcessor.ts +++ b/packages/server/src/appMigrations/migrationsProcessor.ts @@ -12,16 +12,16 @@ export async function processMigrations( migrations: AppMigration[] ) { console.log(`Processing app migration for "${appId}"`) - // have to wrap in context, this gets the tenant from the app ID - await context.doInAppContext(appId, async () => { - await locks.doWithLock( - { - name: LockName.APP_MIGRATION, - type: LockType.AUTO_EXTEND, - resource: appId, - }, - async () => { - try { + try { + // have to wrap in context, this gets the tenant from the app ID + await context.doInAppContext(appId, async () => { + await locks.doWithLock( + { + name: LockName.APP_MIGRATION, + type: LockType.AUTO_EXTEND, + resource: appId, + }, + async () => { await context.doInAppMigrationContext(appId, async () => { let currentVersion = await getAppMigrationVersion(appId) @@ -55,13 +55,13 @@ export async function processMigrations( currentVersion = id } }) - } catch (err) { - logging.logAlert("Failed to run app migration", err) - throw err } - } - ) + ) - console.log(`App migration for "${appId}" processed`) - }) + console.log(`App migration for "${appId}" processed`) + }) + } catch (err) { + logging.logAlert("Failed to run app migration", err) + throw err + } } From 3a95aa6aeb1b9b5e28401981206737ee49aeab4b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 18:56:24 +0100 Subject: [PATCH 6/8] Adding the version to the status to help understand what version the service is using. --- packages/types/src/api/web/system/index.ts | 1 + packages/types/src/api/web/system/status.ts | 11 ++++++++++ .../src/api/controllers/system/status.ts | 22 +++++++++++++------ 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 packages/types/src/api/web/system/status.ts diff --git a/packages/types/src/api/web/system/index.ts b/packages/types/src/api/web/system/index.ts index d19c9296c9..8820bf5cd1 100644 --- a/packages/types/src/api/web/system/index.ts +++ b/packages/types/src/api/web/system/index.ts @@ -1 +1,2 @@ export * from "./environment" +export * from "./status" diff --git a/packages/types/src/api/web/system/status.ts b/packages/types/src/api/web/system/status.ts new file mode 100644 index 0000000000..3d64cc4d97 --- /dev/null +++ b/packages/types/src/api/web/system/status.ts @@ -0,0 +1,11 @@ +export type SystemStatusResponse = { + passing?: boolean + checks?: { + login: boolean + search: boolean + } + health?: { + passing: boolean + } + version?: string +} diff --git a/packages/worker/src/api/controllers/system/status.ts b/packages/worker/src/api/controllers/system/status.ts index b763a67d4f..662ca05c48 100644 --- a/packages/worker/src/api/controllers/system/status.ts +++ b/packages/worker/src/api/controllers/system/status.ts @@ -1,16 +1,24 @@ -import { accounts } from "@budibase/backend-core" +import { accounts, env as coreEnv } from "@budibase/backend-core" +import { Ctx, SystemStatusResponse } from "@budibase/types" import env from "../../../environment" -import { BBContext } from "@budibase/types" -export const fetch = async (ctx: BBContext) => { +export const fetch = async (ctx: Ctx) => { + let status: SystemStatusResponse | undefined if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { - const status = await accounts.getStatus() - ctx.body = status - } else { - ctx.body = { + status = await accounts.getStatus() + } + + if (!status) { + status = { health: { passing: true, }, } } + + if (coreEnv.VERSION) { + status.version = coreEnv.VERSION + } + + ctx.body = status } From d4d3537f41e198069d2bba56579643ec2f297160 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 19:02:30 +0100 Subject: [PATCH 7/8] Updating test. --- packages/worker/src/api/routes/system/tests/status.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/worker/src/api/routes/system/tests/status.spec.ts b/packages/worker/src/api/routes/system/tests/status.spec.ts index 71e01a0e72..cfb9e2d68f 100644 --- a/packages/worker/src/api/routes/system/tests/status.spec.ts +++ b/packages/worker/src/api/routes/system/tests/status.spec.ts @@ -27,6 +27,7 @@ describe("/api/system/status", () => { health: { passing: true, }, + version: expect.any(String), }) expect(accounts.getStatus).toHaveBeenCalledTimes(0) config.cloudHosted() From 834de1f64a0a264ecbb27c449e383ba0dd519294 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 19:19:43 +0100 Subject: [PATCH 8/8] Log every step of the app migration process for easier visibility. --- packages/server/src/appMigrations/migrationsProcessor.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/server/src/appMigrations/migrationsProcessor.ts b/packages/server/src/appMigrations/migrationsProcessor.ts index d4f474cd30..0337fc09f4 100644 --- a/packages/server/src/appMigrations/migrationsProcessor.ts +++ b/packages/server/src/appMigrations/migrationsProcessor.ts @@ -15,6 +15,7 @@ export async function processMigrations( try { // have to wrap in context, this gets the tenant from the app ID await context.doInAppContext(appId, async () => { + console.log(`Acquiring app migration lock for "${appId}"`) await locks.doWithLock( { name: LockName.APP_MIGRATION, @@ -23,6 +24,7 @@ export async function processMigrations( }, async () => { await context.doInAppMigrationContext(appId, async () => { + console.log(`Lock acquired starting app migration for "${appId}"`) let currentVersion = await getAppMigrationVersion(appId) const pendingMigrations = migrations @@ -30,6 +32,9 @@ export async function processMigrations( .sort((a, b) => a.id.localeCompare(b.id)) const migrationIds = migrations.map(m => m.id).sort() + console.log( + `App migrations to run for "${appId}" - ${migrationIds.join(",")}` + ) let index = 0 for (const { id, func } of pendingMigrations) {