Merge pull request #13854 from Budibase/BUDI-7656/add-migration
SQS migrations/app migrations
This commit is contained in:
commit
232f2df643
18 changed files with 302 additions and 24 deletions
|
@ -8,6 +8,7 @@ import {
|
|||
DatabaseOpts,
|
||||
DatabasePutOpts,
|
||||
DatabaseQueryOpts,
|
||||
DBError,
|
||||
Document,
|
||||
isDocument,
|
||||
RowResponse,
|
||||
|
@ -41,7 +42,7 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
|
|||
|
||||
type DBCall<T> = () => Promise<T>
|
||||
|
||||
class CouchDBError extends Error {
|
||||
class CouchDBError extends Error implements DBError {
|
||||
status: number
|
||||
statusCode: number
|
||||
reason: string
|
||||
|
|
|
@ -358,11 +358,14 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
|
|||
await createApp(appId)
|
||||
}
|
||||
|
||||
const latestMigrationId = appMigrations.getLatestEnabledMigrationId()
|
||||
if (latestMigrationId) {
|
||||
// Initialise the app migration version as the latest one
|
||||
await appMigrations.updateAppMigrationMetadata({
|
||||
appId,
|
||||
version: appMigrations.getLatestMigrationId(),
|
||||
version: latestMigrationId,
|
||||
})
|
||||
}
|
||||
|
||||
await cache.app.invalidateAppMetadata(appId, newApplication)
|
||||
return newApplication
|
||||
|
|
|
@ -3,7 +3,7 @@ import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
|
|||
import { Ctx } from "@budibase/types"
|
||||
import {
|
||||
getAppMigrationVersion,
|
||||
getLatestMigrationId,
|
||||
getLatestEnabledMigrationId,
|
||||
} from "../../appMigrations"
|
||||
|
||||
export async function migrate(ctx: Ctx) {
|
||||
|
@ -27,7 +27,9 @@ export async function getMigrationStatus(ctx: Ctx) {
|
|||
|
||||
const latestAppliedMigration = await getAppMigrationVersion(appId)
|
||||
|
||||
const migrated = latestAppliedMigration === getLatestMigrationId()
|
||||
const latestMigrationId = getLatestEnabledMigrationId()
|
||||
const migrated =
|
||||
!latestMigrationId || latestAppliedMigration >= latestMigrationId
|
||||
|
||||
ctx.body = { migrated }
|
||||
ctx.status = 200
|
||||
|
|
|
@ -31,7 +31,7 @@ import {
|
|||
} from "@budibase/types"
|
||||
import {
|
||||
getAppMigrationVersion,
|
||||
getLatestMigrationId,
|
||||
getLatestEnabledMigrationId,
|
||||
} from "../../../appMigrations"
|
||||
|
||||
import send from "koa-send"
|
||||
|
@ -133,7 +133,7 @@ const requiresMigration = async (ctx: Ctx) => {
|
|||
ctx.throw("AppId could not be found")
|
||||
}
|
||||
|
||||
const latestMigration = getLatestMigrationId()
|
||||
const latestMigration = getLatestEnabledMigrationId()
|
||||
if (!latestMigration) {
|
||||
return false
|
||||
}
|
||||
|
|
|
@ -33,7 +33,7 @@ export async function getAppMigrationVersion(appId: string): Promise<string> {
|
|||
let version
|
||||
try {
|
||||
metadata = await getFromDB(appId)
|
||||
version = metadata.version
|
||||
version = metadata.version || ""
|
||||
} catch (err: any) {
|
||||
if (err.status !== 404) {
|
||||
throw err
|
||||
|
|
|
@ -10,14 +10,25 @@ export * from "./appMigrationMetadata"
|
|||
export type AppMigration = {
|
||||
id: string
|
||||
func: () => Promise<void>
|
||||
// disabled so that by default all migrations listed are enabled
|
||||
disabled?: boolean
|
||||
}
|
||||
|
||||
export const getLatestMigrationId = () =>
|
||||
MIGRATIONS.map(m => m.id)
|
||||
.sort()
|
||||
.reverse()[0]
|
||||
export function getLatestEnabledMigrationId(migrations?: AppMigration[]) {
|
||||
let latestMigrationId: string | undefined
|
||||
for (let migration of migrations || MIGRATIONS) {
|
||||
// if a migration is disabled, all migrations after it are disabled
|
||||
if (migration.disabled) {
|
||||
break
|
||||
}
|
||||
latestMigrationId = migration.id
|
||||
}
|
||||
return latestMigrationId
|
||||
}
|
||||
|
||||
const getTimestamp = (versionId: string) => versionId?.split("_")[0] || ""
|
||||
function getTimestamp(versionId: string) {
|
||||
return versionId?.split("_")[0] || ""
|
||||
}
|
||||
|
||||
export async function checkMissingMigrations(
|
||||
ctx: UserCtx,
|
||||
|
@ -25,9 +36,12 @@ export async function checkMissingMigrations(
|
|||
appId: string
|
||||
) {
|
||||
const currentVersion = await getAppMigrationVersion(appId)
|
||||
const latestMigration = getLatestMigrationId()
|
||||
const latestMigration = getLatestEnabledMigrationId()
|
||||
|
||||
if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) {
|
||||
if (
|
||||
latestMigration &&
|
||||
getTimestamp(currentVersion) < getTimestamp(latestMigration)
|
||||
) {
|
||||
await queue.add(
|
||||
{
|
||||
appId,
|
||||
|
|
|
@ -1,7 +1,15 @@
|
|||
// This file should never be manually modified, use `yarn add-app-migration` in order to add a new one
|
||||
|
||||
import env from "../environment"
|
||||
import { AppMigration } from "."
|
||||
|
||||
import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs"
|
||||
|
||||
// Migrations will be executed sorted by ID
|
||||
export const MIGRATIONS: AppMigration[] = [
|
||||
// Migrations will be executed sorted by id
|
||||
{
|
||||
id: "20240604153647_initial_sqs",
|
||||
func: m20240604153647_initial_sqs,
|
||||
disabled: !env.SQS_SEARCH_ENABLE,
|
||||
},
|
||||
]
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
import { context } from "@budibase/backend-core"
|
||||
import { allLinkDocs } from "../../db/utils"
|
||||
import LinkDocumentImpl from "../../db/linkedRows/LinkDocument"
|
||||
import sdk from "../../sdk"
|
||||
import env from "../../environment"
|
||||
|
||||
const migration = async () => {
|
||||
const linkDocs = await allLinkDocs()
|
||||
|
||||
const docsToUpdate = []
|
||||
for (const linkDoc of linkDocs) {
|
||||
if (linkDoc.tableId) {
|
||||
// It already had the required data
|
||||
continue
|
||||
}
|
||||
|
||||
// it already has the junction table ID - no need to migrate
|
||||
if (!linkDoc.tableId) {
|
||||
const newLink = new LinkDocumentImpl(
|
||||
linkDoc.doc1.tableId,
|
||||
linkDoc.doc1.fieldName,
|
||||
linkDoc.doc1.rowId,
|
||||
linkDoc.doc2.tableId,
|
||||
linkDoc.doc2.fieldName,
|
||||
linkDoc.doc2.rowId
|
||||
)
|
||||
newLink._id = linkDoc._id!
|
||||
newLink._rev = linkDoc._rev
|
||||
docsToUpdate.push(newLink)
|
||||
}
|
||||
}
|
||||
|
||||
const db = context.getAppDB()
|
||||
if (docsToUpdate.length) {
|
||||
await db.bulkDocs(docsToUpdate)
|
||||
}
|
||||
|
||||
// at the end make sure design doc is ready
|
||||
await sdk.tables.sqs.syncDefinition()
|
||||
// only do initial search if environment is using SQS already
|
||||
// initial search makes sure that all the indexes have been created
|
||||
// and are ready to use, avoiding any initial waits for large tables
|
||||
if (env.SQS_SEARCH_ENABLE) {
|
||||
const tables = await sdk.tables.getAllInternalTables()
|
||||
// do these one by one - running in parallel could cause problems
|
||||
for (let table of tables) {
|
||||
await db.sql(`select * from ${table._id} limit 1`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default migration
|
|
@ -0,0 +1,116 @@
|
|||
import * as setup from "../../../api/routes/tests/utilities"
|
||||
import { basicTable } from "../../../tests/utilities/structures"
|
||||
import {
|
||||
db as dbCore,
|
||||
SQLITE_DESIGN_DOC_ID,
|
||||
context,
|
||||
} from "@budibase/backend-core"
|
||||
import {
|
||||
LinkDocument,
|
||||
DocumentType,
|
||||
SQLiteDefinition,
|
||||
SQLiteType,
|
||||
} from "@budibase/types"
|
||||
import {
|
||||
generateJunctionTableID,
|
||||
generateLinkID,
|
||||
generateRowID,
|
||||
} from "../../../db/utils"
|
||||
import migration from "../20240604153647_initial_sqs"
|
||||
|
||||
const config = setup.getConfig()
|
||||
let tableId: string
|
||||
|
||||
function oldLinkDocInfo() {
|
||||
const tableId1 = `${DocumentType.TABLE}_a`,
|
||||
tableId2 = `${DocumentType.TABLE}_b`
|
||||
return {
|
||||
tableId1,
|
||||
tableId2,
|
||||
rowId1: generateRowID(tableId1, "b"),
|
||||
rowId2: generateRowID(tableId2, "a"),
|
||||
col1: "columnB",
|
||||
col2: "columnA",
|
||||
}
|
||||
}
|
||||
|
||||
function oldLinkDocID() {
|
||||
const { tableId1, tableId2, rowId1, rowId2, col1, col2 } = oldLinkDocInfo()
|
||||
return generateLinkID(tableId1, tableId2, rowId1, rowId2, col1, col2)
|
||||
}
|
||||
|
||||
function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
|
||||
const { tableId1, tableId2, rowId1, rowId2, col1, col2 } = oldLinkDocInfo()
|
||||
return {
|
||||
type: "link",
|
||||
_id: oldLinkDocID(),
|
||||
doc1: {
|
||||
tableId: tableId1,
|
||||
fieldName: col1,
|
||||
rowId: rowId1,
|
||||
},
|
||||
doc2: {
|
||||
tableId: tableId2,
|
||||
fieldName: col2,
|
||||
rowId: rowId2,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function sqsDisabled(cb: () => Promise<void>) {
|
||||
await config.withEnv({ SQS_SEARCH_ENABLE: "" }, cb)
|
||||
}
|
||||
|
||||
async function sqsEnabled(cb: () => Promise<void>) {
|
||||
await config.withEnv({ SQS_SEARCH_ENABLE: "1" }, cb)
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
await sqsDisabled(async () => {
|
||||
await config.init()
|
||||
const table = await config.api.table.save(basicTable())
|
||||
tableId = table._id!
|
||||
const db = dbCore.getDB(config.appId!)
|
||||
// old link document
|
||||
await db.put(oldLinkDocument())
|
||||
})
|
||||
})
|
||||
|
||||
describe("SQS migration", () => {
|
||||
it("test migration runs as expected against an older DB", async () => {
|
||||
const db = dbCore.getDB(config.appId!)
|
||||
// confirm nothing exists initially
|
||||
await sqsDisabled(async () => {
|
||||
let error: any | undefined
|
||||
try {
|
||||
await db.get(SQLITE_DESIGN_DOC_ID)
|
||||
} catch (err: any) {
|
||||
error = err
|
||||
}
|
||||
expect(error).toBeDefined()
|
||||
expect(error.status).toBe(404)
|
||||
})
|
||||
await sqsEnabled(async () => {
|
||||
await context.doInAppContext(config.appId!, async () => {
|
||||
await migration()
|
||||
})
|
||||
const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
|
||||
expect(designDoc.sql.tables).toBeDefined()
|
||||
const mainTableDef = designDoc.sql.tables[tableId]
|
||||
expect(mainTableDef).toBeDefined()
|
||||
expect(mainTableDef.fields.name).toEqual(SQLiteType.TEXT)
|
||||
expect(mainTableDef.fields.description).toEqual(SQLiteType.TEXT)
|
||||
|
||||
const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
|
||||
const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
|
||||
expect(linkDoc.tableId).toEqual(
|
||||
generateJunctionTableID(tableId1, tableId2)
|
||||
)
|
||||
// should have swapped the documents
|
||||
expect(linkDoc.doc1.tableId).toEqual(tableId2)
|
||||
expect(linkDoc.doc1.rowId).toEqual(rowId2)
|
||||
expect(linkDoc.doc2.tableId).toEqual(tableId1)
|
||||
expect(linkDoc.doc2.rowId).toEqual(rowId1)
|
||||
})
|
||||
})
|
||||
})
|
|
@ -1,6 +1,7 @@
|
|||
import { Header } from "@budibase/backend-core"
|
||||
import * as setup from "../../api/routes/tests/utilities"
|
||||
import * as migrations from "../migrations"
|
||||
import { AppMigration, getLatestEnabledMigrationId } from "../index"
|
||||
import { getAppMigrationVersion } from "../appMigrationMetadata"
|
||||
|
||||
jest.mock<typeof migrations>("../migrations", () => ({
|
||||
|
@ -52,4 +53,29 @@ describe("migrations", () => {
|
|||
},
|
||||
})
|
||||
})
|
||||
|
||||
it("should disable all migrations after one that is disabled", () => {
|
||||
const MIGRATION_ID1 = "20231211105810_new-test",
|
||||
MIGRATION_ID2 = "20231211105812_new-test",
|
||||
MIGRATION_ID3 = "20231211105814_new-test"
|
||||
// create some migrations to test with
|
||||
const migrations: AppMigration[] = [
|
||||
{
|
||||
id: MIGRATION_ID1,
|
||||
func: async () => {},
|
||||
},
|
||||
{
|
||||
id: MIGRATION_ID2,
|
||||
func: async () => {},
|
||||
},
|
||||
{
|
||||
id: MIGRATION_ID3,
|
||||
func: async () => {},
|
||||
},
|
||||
]
|
||||
|
||||
expect(getLatestEnabledMigrationId(migrations)).toBe(MIGRATION_ID3)
|
||||
migrations[1].disabled = true
|
||||
expect(getLatestEnabledMigrationId(migrations)).toBe(MIGRATION_ID1)
|
||||
})
|
||||
})
|
||||
|
|
|
@ -59,6 +59,9 @@ class LinkDocumentImpl implements LinkDocument {
|
|||
this.doc1 = docA.tableId > docB.tableId ? docA : docB
|
||||
this.doc2 = docA.tableId > docB.tableId ? docB : docA
|
||||
}
|
||||
_rev?: string | undefined
|
||||
createdAt?: string | number | undefined
|
||||
updatedAt?: string | undefined
|
||||
}
|
||||
|
||||
export default LinkDocumentImpl
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import newid from "./newid"
|
||||
import { db as dbCore } from "@budibase/backend-core"
|
||||
import { context, db as dbCore } from "@budibase/backend-core"
|
||||
import {
|
||||
DatabaseQueryOpts,
|
||||
Datasource,
|
||||
|
@ -10,6 +10,7 @@ import {
|
|||
RelationshipFieldMetadata,
|
||||
SourceName,
|
||||
VirtualDocumentType,
|
||||
LinkDocument,
|
||||
} from "@budibase/types"
|
||||
|
||||
export { DocumentType, VirtualDocumentType } from "@budibase/types"
|
||||
|
@ -137,10 +138,24 @@ export function generateLinkID(
|
|||
/**
|
||||
* Gets parameters for retrieving link docs, this is a utility function for the getDocParams function.
|
||||
*/
|
||||
export function getLinkParams(otherProps: any = {}) {
|
||||
function getLinkParams(otherProps: Partial<DatabaseQueryOpts> = {}) {
|
||||
return getDocParams(DocumentType.LINK, null, otherProps)
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all the link docs document from the current app db.
|
||||
*/
|
||||
export async function allLinkDocs() {
|
||||
const db = context.getAppDB()
|
||||
|
||||
const response = await db.allDocs<LinkDocument>(
|
||||
getLinkParams({
|
||||
include_docs: true,
|
||||
})
|
||||
)
|
||||
return response.rows.map(row => row.doc!)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a new layout ID.
|
||||
* @returns The new layout ID which the layout doc can be stored under.
|
||||
|
|
|
@ -96,6 +96,7 @@ const environment = {
|
|||
DISABLE_THREADING: process.env.DISABLE_THREADING,
|
||||
DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
|
||||
DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING,
|
||||
DISABLE_APP_MIGRATIONS: process.env.SKIP_APP_MIGRATIONS || false,
|
||||
MULTI_TENANCY: process.env.MULTI_TENANCY,
|
||||
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
|
||||
SELF_HOSTED: process.env.SELF_HOSTED,
|
||||
|
|
|
@ -1,9 +1,16 @@
|
|||
import { UserCtx } from "@budibase/types"
|
||||
import { checkMissingMigrations } from "../appMigrations"
|
||||
import env from "../environment"
|
||||
|
||||
export default async (ctx: UserCtx, next: any) => {
|
||||
const { appId } = ctx
|
||||
|
||||
// migrations can be disabled via environment variable if you
|
||||
// need to completely disable migrations, e.g. for testing
|
||||
if (env.DISABLE_APP_MIGRATIONS) {
|
||||
return next()
|
||||
}
|
||||
|
||||
if (!appId) {
|
||||
return next()
|
||||
}
|
||||
|
|
|
@ -14,6 +14,7 @@ import {
|
|||
CONSTANT_INTERNAL_ROW_COLS,
|
||||
generateJunctionTableID,
|
||||
} from "../../../../db/utils"
|
||||
import { isEqual } from "lodash"
|
||||
|
||||
const FieldTypeMap: Record<FieldType, SQLiteType> = {
|
||||
[FieldType.BOOLEAN]: SQLiteType.NUMERIC,
|
||||
|
@ -107,8 +108,22 @@ async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
|
|||
|
||||
export async function syncDefinition(): Promise<void> {
|
||||
const db = context.getAppDB()
|
||||
let existing: SQLiteDefinition | undefined
|
||||
try {
|
||||
existing = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
|
||||
} catch (err: any) {
|
||||
if (err.status !== 404) {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
const definition = await buildBaseDefinition()
|
||||
if (existing) {
|
||||
definition._rev = existing._rev
|
||||
}
|
||||
// only write if something has changed
|
||||
if (!existing || !isEqual(existing.sql, definition.sql)) {
|
||||
await db.put(definition)
|
||||
}
|
||||
}
|
||||
|
||||
export async function addTable(table: Table) {
|
||||
|
|
|
@ -30,4 +30,7 @@ export interface SQLiteDefinition {
|
|||
}
|
||||
}
|
||||
|
||||
export type PreSaveSQLiteDefinition = Omit<SQLiteDefinition, "_rev">
|
||||
export interface PreSaveSQLiteDefinition
|
||||
extends Omit<SQLiteDefinition, "_rev"> {
|
||||
_rev?: string
|
||||
}
|
||||
|
|
|
@ -165,3 +165,13 @@ export interface Database {
|
|||
deleteIndex(...args: any[]): Promise<any>
|
||||
getIndexes(...args: any[]): Promise<any>
|
||||
}
|
||||
|
||||
export interface DBError extends Error {
|
||||
status: number
|
||||
statusCode: number
|
||||
reason: string
|
||||
name: string
|
||||
errid: string
|
||||
error: string
|
||||
description: string
|
||||
}
|
||||
|
|
|
@ -21,7 +21,9 @@ const generateTimestamp = () => {
|
|||
}
|
||||
|
||||
const createMigrationFile = () => {
|
||||
const migrationFilename = `${generateTimestamp()}_${title}`
|
||||
const migrationFilename = `${generateTimestamp()}_${title
|
||||
.replace(/-/g, "_")
|
||||
.replace(/ /g, "_")}`
|
||||
const migrationsDir = "../packages/server/src/appMigrations"
|
||||
|
||||
const template = `const migration = async () => {
|
||||
|
|
Loading…
Reference in a new issue