Run as array

packages/server/src/appMigrations/appMigrationMetadata.ts

@@ -20,25 +20,30 @@ async function getFromDB(appId: string) {
 
 const getCacheKey = (appId: string) => `appmigrations_${env.VERSION}_${appId}`
 
-export async function getAppMigrationMetadata(appId: string): Promise<string> {
+export async function getAppMigrationVersion(appId: string): Promise<string> {
   const cacheKey = getCacheKey(appId)
 
   let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey)
-  if (!metadata || env.isDev()) {
-    try {
-      metadata = await getFromDB(appId)
-    } catch (err: any) {
-      if (err.status !== 404) {
-        throw err
-      }
 
-      metadata = { version: "", history: {} }
-    }
-
-    await cache.store(cacheKey, metadata, EXPIRY_SECONDS)
+  if (metadata && !env.isDev()) {
+    return metadata.version
   }
 
-  return metadata.version
+  let version
+  try {
+    metadata = await getFromDB(appId)
+    version = metadata.version
+  } catch (err: any) {
+    if (err.status !== 404) {
+      throw err
+    }
+
+    version = ""
+  }
+
+  await cache.store(cacheKey, version, EXPIRY_SECONDS)
+
+  return version
 }
 
 export async function updateAppMigrationMetadata({
@@ -49,7 +54,25 @@ export async function updateAppMigrationMetadata({
   version: string
 }): Promise<void> {
   const db = context.getAppDB()
-  const appMigrationDoc = await getFromDB(appId)
+
+  let appMigrationDoc: AppMigrationDoc
+
+  try {
+    appMigrationDoc = await getFromDB(appId)
+  } catch (err: any) {
+    if (err.status !== 404) {
+      throw err
+    }
+
+    appMigrationDoc = {
+      _id: DocumentType.APP_MIGRATION_METADATA,
+      version: "",
+      history: {},
+    }
+    await db.put(appMigrationDoc)
+    appMigrationDoc = await getFromDB(appId)
+  }
+
   const updatedMigrationDoc: AppMigrationDoc = {
     ...appMigrationDoc,
     version,

packages/server/src/appMigrations/index.ts

@@ -1,11 +1,13 @@
-import queue, { PROCESS_MIGRATION_TIMEOUT } from "./queue"
-import { getAppMigrationMetadata } from "./appMigrationMetadata"
+import queue from "./queue"
+import { getAppMigrationVersion } from "./appMigrationMetadata"
 import { MIGRATIONS } from "./migrations"
 
-const latestMigration = Object.keys(MIGRATIONS).sort().reverse()[0]
+const latestMigration = MIGRATIONS.map(m => m.migrationId)
+  .sort()
+  .reverse()[0]
 
 export async function checkMissingMigrations(appId: string) {
-  const currentVersion = await getAppMigrationMetadata(appId)
+  const currentVersion = await getAppMigrationVersion(appId)
 
   if (currentVersion < latestMigration) {
     await queue.add(
@@ -16,7 +18,6 @@ export async function checkMissingMigrations(appId: string) {
         jobId: `${appId}_${latestMigration}`,
         removeOnComplete: true,
         removeOnFail: true,
-        timeout: PROCESS_MIGRATION_TIMEOUT,
       }
     )
   }

packages/server/src/appMigrations/queue.ts

@@ -3,7 +3,7 @@ import { LockName, LockType } from "@budibase/types"
 import { Job } from "bull"
 import { MIGRATIONS } from "./migrations"
 import {
-  getAppMigrationMetadata,
+  getAppMigrationVersion,
   updateAppMigrationMetadata,
 } from "./appMigrationMetadata"
 import environment from "../environment"
@@ -28,24 +28,34 @@ async function processMessage(job: Job) {
     },
     async () => {
       await context.doInAppContext(appId, async () => {
-        const currentVersion = await getAppMigrationMetadata(appId)
+        let currentVersion = await getAppMigrationVersion(appId)
 
         const pendingMigrations = MIGRATIONS.filter(
           m => m.migrationId > currentVersion
-        )
+        ).sort((a, b) => a.migrationId.localeCompare(b.migrationId))
+
+        const migrationIds = MIGRATIONS.map(m => m.migrationId).sort()
 
         let index = 0
-        for (const migration of pendingMigrations) {
+        for (const { migrationId, migrationFunc } of pendingMigrations) {
+          const expectedMigration =
+            migrationIds[migrationIds.indexOf(currentVersion) + 1]
+
+          if (expectedMigration !== migrationId) {
+            throw `Migration ${migrationId} could not run, update for "${migrationId}" is running but ${expectedMigration} is expected`
+          }
+
           const counter = `(${++index}/${pendingMigrations.length})`
-          console.info(`Running migration ${migration}... ${counter}`, {
-            migration,
+          console.info(`Running migration ${migrationId}... ${counter}`, {
+            migrationId,
             appId,
           })
-          await migration.migrationFunc()
+          await migrationFunc()
           await updateAppMigrationMetadata({
             appId,
-            version: migration.migrationId,
+            version: migrationId,
           })
+          currentVersion = migrationId
         }
       })
     }

scripts/add-app-migration.js

@@ -50,13 +50,20 @@ export default migration
     migrationFileContent += `import m${migration} from "./migrations/${migration}"\n`
   }
 
-  migrationFileContent += `\nexport const MIGRATIONS: Record<string, { migration: () => Promise<void> }> = {\n`
+  migrationFileContent += `\nexport const MIGRATIONS: {
+  migrationId: string
+  migrationFunc: () => Promise<void>
+}[] = [
+  // Migrations will be executed sorted by migrationId\n`
 
   for (const migration of migrations) {
-    migrationFileContent += `  [${migration}]: { migration: m${migration} },\n`
+    migrationFileContent += `  {
+    migrationId: "${migration}",
+    migrationFunc: m${migration}
+  },\n`
   }
 
-  migrationFileContent += `}\n`
+  migrationFileContent += `]\n`
 
   fs.writeFileSync(
     path.resolve(__dirname, migrationsFilePath),