diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index ed79a14340..4db63ad695 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -8,6 +8,7 @@ import { DatabaseOpts, DatabasePutOpts, DatabaseQueryOpts, + DBError, Document, isDocument, RowResponse, @@ -41,7 +42,7 @@ function buildNano(couchInfo: { url: string; cookie: string }) { type DBCall = () => Promise -class CouchDBError extends Error { +class CouchDBError extends Error implements DBError { status: number statusCode: number reason: string diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index e73058239b..830acc55bf 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -358,11 +358,14 @@ async function performAppCreate(ctx: UserCtx) { await createApp(appId) } - // Initialise the app migration version as the latest one - await appMigrations.updateAppMigrationMetadata({ - appId, - version: appMigrations.getLatestMigrationId(), - }) + const latestMigrationId = appMigrations.getLatestEnabledMigrationId() + if (latestMigrationId) { + // Initialise the app migration version as the latest one + await appMigrations.updateAppMigrationMetadata({ + appId, + version: latestMigrationId, + }) + } await cache.app.invalidateAppMetadata(appId, newApplication) return newApplication diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index c8f786578d..89255e89b1 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -3,7 +3,7 @@ import { migrate as migrationImpl, MIGRATIONS } from "../../migrations" import { Ctx } from "@budibase/types" import { getAppMigrationVersion, - getLatestMigrationId, + getLatestEnabledMigrationId, } from "../../appMigrations" export async function migrate(ctx: Ctx) { @@ -27,7 +27,9 @@ export async function getMigrationStatus(ctx: Ctx) { const latestAppliedMigration = await getAppMigrationVersion(appId) - const migrated = latestAppliedMigration === getLatestMigrationId() + const latestMigrationId = getLatestEnabledMigrationId() + const migrated = + !latestMigrationId || latestAppliedMigration >= latestMigrationId ctx.body = { migrated } ctx.status = 200 diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 63240c8ecb..8db7816a9c 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -31,7 +31,7 @@ import { } from "@budibase/types" import { getAppMigrationVersion, - getLatestMigrationId, + getLatestEnabledMigrationId, } from "../../../appMigrations" import send from "koa-send" @@ -133,7 +133,7 @@ const requiresMigration = async (ctx: Ctx) => { ctx.throw("AppId could not be found") } - const latestMigration = getLatestMigrationId() + const latestMigration = getLatestEnabledMigrationId() if (!latestMigration) { return false } diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index 202e78d964..d87ddff3ef 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -33,7 +33,7 @@ export async function getAppMigrationVersion(appId: string): Promise { let version try { metadata = await getFromDB(appId) - version = metadata.version + version = metadata.version || "" } catch (err: any) { if (err.status !== 404) { throw err diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index 0758b9f324..8bd3ae7425 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -10,14 +10,25 @@ export * from "./appMigrationMetadata" export type AppMigration = { id: string func: () => Promise + // disabled so that by default all migrations listed are enabled + disabled?: boolean } -export const getLatestMigrationId = () => - MIGRATIONS.map(m => m.id) - .sort() - .reverse()[0] +export function getLatestEnabledMigrationId(migrations?: AppMigration[]) { + let latestMigrationId: string | undefined + for (let migration of migrations || MIGRATIONS) { + // if a migration is disabled, all migrations after it are disabled + if (migration.disabled) { + break + } + latestMigrationId = migration.id + } + return latestMigrationId +} -const getTimestamp = (versionId: string) => versionId?.split("_")[0] || "" +function getTimestamp(versionId: string) { + return versionId?.split("_")[0] || "" +} export async function checkMissingMigrations( ctx: UserCtx, @@ -25,9 +36,12 @@ export async function checkMissingMigrations( appId: string ) { const currentVersion = await getAppMigrationVersion(appId) - const latestMigration = getLatestMigrationId() + const latestMigration = getLatestEnabledMigrationId() - if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) { + if ( + latestMigration && + getTimestamp(currentVersion) < getTimestamp(latestMigration) + ) { await queue.add( { appId, diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts index d66e2e8895..14eb9d0923 100644 --- a/packages/server/src/appMigrations/migrations.ts +++ b/packages/server/src/appMigrations/migrations.ts @@ -1,7 +1,15 @@ // This file should never be manually modified, use `yarn add-app-migration` in order to add a new one +import env from "../environment" import { AppMigration } from "." +import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs" + +// Migrations will be executed sorted by ID export const MIGRATIONS: AppMigration[] = [ - // Migrations will be executed sorted by id + { + id: "20240604153647_initial_sqs", + func: m20240604153647_initial_sqs, + disabled: !env.SQS_SEARCH_ENABLE, + }, ] diff --git a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts new file mode 100644 index 0000000000..800de1418a --- /dev/null +++ b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts @@ -0,0 +1,52 @@ +import { context } from "@budibase/backend-core" +import { allLinkDocs } from "../../db/utils" +import LinkDocumentImpl from "../../db/linkedRows/LinkDocument" +import sdk from "../../sdk" +import env from "../../environment" + +const migration = async () => { + const linkDocs = await allLinkDocs() + + const docsToUpdate = [] + for (const linkDoc of linkDocs) { + if (linkDoc.tableId) { + // It already had the required data + continue + } + + // it already has the junction table ID - no need to migrate + if (!linkDoc.tableId) { + const newLink = new LinkDocumentImpl( + linkDoc.doc1.tableId, + linkDoc.doc1.fieldName, + linkDoc.doc1.rowId, + linkDoc.doc2.tableId, + linkDoc.doc2.fieldName, + linkDoc.doc2.rowId + ) + newLink._id = linkDoc._id! + newLink._rev = linkDoc._rev + docsToUpdate.push(newLink) + } + } + + const db = context.getAppDB() + if (docsToUpdate.length) { + await db.bulkDocs(docsToUpdate) + } + + // at the end make sure design doc is ready + await sdk.tables.sqs.syncDefinition() + // only do initial search if environment is using SQS already + // initial search makes sure that all the indexes have been created + // and are ready to use, avoiding any initial waits for large tables + if (env.SQS_SEARCH_ENABLE) { + const tables = await sdk.tables.getAllInternalTables() + // do these one by one - running in parallel could cause problems + for (let table of tables) { + await db.sql(`select * from ${table._id} limit 1`) + } + } +} + +export default migration diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts new file mode 100644 index 0000000000..64420d239f --- /dev/null +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -0,0 +1,116 @@ +import * as setup from "../../../api/routes/tests/utilities" +import { basicTable } from "../../../tests/utilities/structures" +import { + db as dbCore, + SQLITE_DESIGN_DOC_ID, + context, +} from "@budibase/backend-core" +import { + LinkDocument, + DocumentType, + SQLiteDefinition, + SQLiteType, +} from "@budibase/types" +import { + generateJunctionTableID, + generateLinkID, + generateRowID, +} from "../../../db/utils" +import migration from "../20240604153647_initial_sqs" + +const config = setup.getConfig() +let tableId: string + +function oldLinkDocInfo() { + const tableId1 = `${DocumentType.TABLE}_a`, + tableId2 = `${DocumentType.TABLE}_b` + return { + tableId1, + tableId2, + rowId1: generateRowID(tableId1, "b"), + rowId2: generateRowID(tableId2, "a"), + col1: "columnB", + col2: "columnA", + } +} + +function oldLinkDocID() { + const { tableId1, tableId2, rowId1, rowId2, col1, col2 } = oldLinkDocInfo() + return generateLinkID(tableId1, tableId2, rowId1, rowId2, col1, col2) +} + +function oldLinkDocument(): Omit { + const { tableId1, tableId2, rowId1, rowId2, col1, col2 } = oldLinkDocInfo() + return { + type: "link", + _id: oldLinkDocID(), + doc1: { + tableId: tableId1, + fieldName: col1, + rowId: rowId1, + }, + doc2: { + tableId: tableId2, + fieldName: col2, + rowId: rowId2, + }, + } +} + +async function sqsDisabled(cb: () => Promise) { + await config.withEnv({ SQS_SEARCH_ENABLE: "" }, cb) +} + +async function sqsEnabled(cb: () => Promise) { + await config.withEnv({ SQS_SEARCH_ENABLE: "1" }, cb) +} + +beforeAll(async () => { + await sqsDisabled(async () => { + await config.init() + const table = await config.api.table.save(basicTable()) + tableId = table._id! + const db = dbCore.getDB(config.appId!) + // old link document + await db.put(oldLinkDocument()) + }) +}) + +describe("SQS migration", () => { + it("test migration runs as expected against an older DB", async () => { + const db = dbCore.getDB(config.appId!) + // confirm nothing exists initially + await sqsDisabled(async () => { + let error: any | undefined + try { + await db.get(SQLITE_DESIGN_DOC_ID) + } catch (err: any) { + error = err + } + expect(error).toBeDefined() + expect(error.status).toBe(404) + }) + await sqsEnabled(async () => { + await context.doInAppContext(config.appId!, async () => { + await migration() + }) + const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) + expect(designDoc.sql.tables).toBeDefined() + const mainTableDef = designDoc.sql.tables[tableId] + expect(mainTableDef).toBeDefined() + expect(mainTableDef.fields.name).toEqual(SQLiteType.TEXT) + expect(mainTableDef.fields.description).toEqual(SQLiteType.TEXT) + + const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() + const linkDoc = await db.get(oldLinkDocID()) + expect(linkDoc.tableId).toEqual( + generateJunctionTableID(tableId1, tableId2) + ) + // should have swapped the documents + expect(linkDoc.doc1.tableId).toEqual(tableId2) + expect(linkDoc.doc1.rowId).toEqual(rowId2) + expect(linkDoc.doc2.tableId).toEqual(tableId1) + expect(linkDoc.doc2.rowId).toEqual(rowId1) + }) + }) +}) diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts index 7af2346934..1da94f503f 100644 --- a/packages/server/src/appMigrations/tests/migrations.spec.ts +++ b/packages/server/src/appMigrations/tests/migrations.spec.ts @@ -1,6 +1,7 @@ import { Header } from "@budibase/backend-core" import * as setup from "../../api/routes/tests/utilities" import * as migrations from "../migrations" +import { AppMigration, getLatestEnabledMigrationId } from "../index" import { getAppMigrationVersion } from "../appMigrationMetadata" jest.mock("../migrations", () => ({ @@ -52,4 +53,29 @@ describe("migrations", () => { }, }) }) + + it("should disable all migrations after one that is disabled", () => { + const MIGRATION_ID1 = "20231211105810_new-test", + MIGRATION_ID2 = "20231211105812_new-test", + MIGRATION_ID3 = "20231211105814_new-test" + // create some migrations to test with + const migrations: AppMigration[] = [ + { + id: MIGRATION_ID1, + func: async () => {}, + }, + { + id: MIGRATION_ID2, + func: async () => {}, + }, + { + id: MIGRATION_ID3, + func: async () => {}, + }, + ] + + expect(getLatestEnabledMigrationId(migrations)).toBe(MIGRATION_ID3) + migrations[1].disabled = true + expect(getLatestEnabledMigrationId(migrations)).toBe(MIGRATION_ID1) + }) }) diff --git a/packages/server/src/db/linkedRows/LinkDocument.ts b/packages/server/src/db/linkedRows/LinkDocument.ts index 8d25bbe93c..a67d21599c 100644 --- a/packages/server/src/db/linkedRows/LinkDocument.ts +++ b/packages/server/src/db/linkedRows/LinkDocument.ts @@ -59,6 +59,9 @@ class LinkDocumentImpl implements LinkDocument { this.doc1 = docA.tableId > docB.tableId ? docA : docB this.doc2 = docA.tableId > docB.tableId ? docB : docA } + _rev?: string | undefined + createdAt?: string | number | undefined + updatedAt?: string | undefined } export default LinkDocumentImpl diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index b8221d208d..3bd1749d77 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -1,5 +1,5 @@ import newid from "./newid" -import { db as dbCore } from "@budibase/backend-core" +import { context, db as dbCore } from "@budibase/backend-core" import { DatabaseQueryOpts, Datasource, @@ -10,6 +10,7 @@ import { RelationshipFieldMetadata, SourceName, VirtualDocumentType, + LinkDocument, } from "@budibase/types" export { DocumentType, VirtualDocumentType } from "@budibase/types" @@ -137,10 +138,24 @@ export function generateLinkID( /** * Gets parameters for retrieving link docs, this is a utility function for the getDocParams function. */ -export function getLinkParams(otherProps: any = {}) { +function getLinkParams(otherProps: Partial = {}) { return getDocParams(DocumentType.LINK, null, otherProps) } +/** + * Gets all the link docs document from the current app db. + */ +export async function allLinkDocs() { + const db = context.getAppDB() + + const response = await db.allDocs( + getLinkParams({ + include_docs: true, + }) + ) + return response.rows.map(row => row.doc!) +} + /** * Generates a new layout ID. * @returns The new layout ID which the layout doc can be stored under. diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 2c91f1cb48..b44d7547a2 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -96,6 +96,7 @@ const environment = { DISABLE_THREADING: process.env.DISABLE_THREADING, DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS, DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING, + DISABLE_APP_MIGRATIONS: process.env.SKIP_APP_MIGRATIONS || false, MULTI_TENANCY: process.env.MULTI_TENANCY, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, SELF_HOSTED: process.env.SELF_HOSTED, diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts index 36e021c7ed..6ad356427b 100644 --- a/packages/server/src/middleware/appMigrations.ts +++ b/packages/server/src/middleware/appMigrations.ts @@ -1,9 +1,16 @@ import { UserCtx } from "@budibase/types" import { checkMissingMigrations } from "../appMigrations" +import env from "../environment" export default async (ctx: UserCtx, next: any) => { const { appId } = ctx + // migrations can be disabled via environment variable if you + // need to completely disable migrations, e.g. for testing + if (env.DISABLE_APP_MIGRATIONS) { + return next() + } + if (!appId) { return next() } diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index eb57d1f3b8..4819b9d8d5 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -14,6 +14,7 @@ import { CONSTANT_INTERNAL_ROW_COLS, generateJunctionTableID, } from "../../../../db/utils" +import { isEqual } from "lodash" const FieldTypeMap: Record = { [FieldType.BOOLEAN]: SQLiteType.NUMERIC, @@ -107,8 +108,22 @@ async function buildBaseDefinition(): Promise { export async function syncDefinition(): Promise { const db = context.getAppDB() + let existing: SQLiteDefinition | undefined + try { + existing = await db.get(SQLITE_DESIGN_DOC_ID) + } catch (err: any) { + if (err.status !== 404) { + throw err + } + } const definition = await buildBaseDefinition() - await db.put(definition) + if (existing) { + definition._rev = existing._rev + } + // only write if something has changed + if (!existing || !isEqual(existing.sql, definition.sql)) { + await db.put(definition) + } } export async function addTable(table: Table) { diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts index 5636fef15b..516669bd59 100644 --- a/packages/types/src/documents/app/sqlite.ts +++ b/packages/types/src/documents/app/sqlite.ts @@ -30,4 +30,7 @@ export interface SQLiteDefinition { } } -export type PreSaveSQLiteDefinition = Omit +export interface PreSaveSQLiteDefinition + extends Omit { + _rev?: string +} diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 7ad740ad05..63c37195b7 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -165,3 +165,13 @@ export interface Database { deleteIndex(...args: any[]): Promise getIndexes(...args: any[]): Promise } + +export interface DBError extends Error { + status: number + statusCode: number + reason: string + name: string + errid: string + error: string + description: string +} diff --git a/scripts/add-app-migration.js b/scripts/add-app-migration.js index a58d3a4fbe..a4e01be635 100644 --- a/scripts/add-app-migration.js +++ b/scripts/add-app-migration.js @@ -21,7 +21,9 @@ const generateTimestamp = () => { } const createMigrationFile = () => { - const migrationFilename = `${generateTimestamp()}_${title}` + const migrationFilename = `${generateTimestamp()}_${title + .replace(/-/g, "_") + .replace(/ /g, "_")}` const migrationsDir = "../packages/server/src/appMigrations" const template = `const migration = async () => {