From 9fdba2b6e126b7420bdea7c14997f3938a341c70 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 19 May 2022 08:19:25 +0100 Subject: [PATCH] Migration for apps, automations, datasources, layouts, queries, roles, tables --- .../src/events/publishers/role.ts | 4 +- packages/backend-core/src/migrations/index.js | 4 +- .../src/migrations/functions/backfill/app.ts | 16 +++++- .../backfill/app/{app.ts => apps.ts} | 0 .../functions/backfill/app/automation.ts | 15 ------ .../functions/backfill/app/automations.ts | 26 ++++++++++ .../functions/backfill/app/datasource.ts | 10 ---- .../functions/backfill/app/datasources.ts | 22 ++++++++ .../functions/backfill/app/layout.ts | 10 ---- .../functions/backfill/app/layouts.ts | 22 ++++++++ .../functions/backfill/app/queries.ts | 33 ++++++++++++ .../functions/backfill/app/query.ts | 11 ---- .../migrations/functions/backfill/app/role.ts | 12 ----- .../functions/backfill/app/roles.ts | 22 ++++++++ .../backfill/app/{row.ts => rows.ts} | 0 .../backfill/app/{screen.ts => screens.ts} | 0 .../functions/backfill/app/table.ts | 3 -- .../functions/backfill/app/tables.ts | 22 ++++++++ .../functions/backfill/tests/app.spec.ts | 42 --------------- .../server/src/migrations/tests/index.spec.ts | 52 +++++++++++++++++++ .../types/src/documents/app/automation.ts | 7 ++- packages/types/src/documents/app/index.ts | 1 + packages/types/src/documents/app/query.ts | 4 +- packages/types/src/documents/app/role.ts | 4 +- packages/types/src/documents/app/user.ts | 3 ++ packages/types/src/documents/global/user.ts | 8 ++- packages/worker/package.json | 1 + packages/worker/src/sdk/users/events.ts | 23 +++++--- 28 files changed, 257 insertions(+), 120 deletions(-) rename packages/server/src/migrations/functions/backfill/app/{app.ts => apps.ts} (100%) delete mode 100644 packages/server/src/migrations/functions/backfill/app/automation.ts create mode 100644 packages/server/src/migrations/functions/backfill/app/automations.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/datasource.ts create mode 100644 packages/server/src/migrations/functions/backfill/app/datasources.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/layout.ts create mode 100644 packages/server/src/migrations/functions/backfill/app/layouts.ts create mode 100644 packages/server/src/migrations/functions/backfill/app/queries.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/query.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/role.ts create mode 100644 packages/server/src/migrations/functions/backfill/app/roles.ts rename packages/server/src/migrations/functions/backfill/app/{row.ts => rows.ts} (100%) rename packages/server/src/migrations/functions/backfill/app/{screen.ts => screens.ts} (100%) delete mode 100644 packages/server/src/migrations/functions/backfill/app/table.ts create mode 100644 packages/server/src/migrations/functions/backfill/app/tables.ts delete mode 100644 packages/server/src/migrations/functions/backfill/tests/app.spec.ts create mode 100644 packages/server/src/migrations/tests/index.spec.ts create mode 100644 packages/types/src/documents/app/user.ts diff --git a/packages/backend-core/src/events/publishers/role.ts b/packages/backend-core/src/events/publishers/role.ts index a7f7e60a32..b08d001c77 100644 --- a/packages/backend-core/src/events/publishers/role.ts +++ b/packages/backend-core/src/events/publishers/role.ts @@ -27,12 +27,12 @@ export function deleted(role: Role) { publishEvent(Event.ROLE_DELETED, properties) } -export function assigned(user: User, role: Role) { +export function assigned(user: User, role: string) { const properties: RoleAssignedEvent = {} publishEvent(Event.ROLE_ASSIGNED, properties) } -export function unassigned(user: User, role: Role) { +export function unassigned(user: User, role: string) { const properties: RoleUnassignedEvent = {} publishEvent(Event.ROLE_UNASSIGNED, properties) } diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js index ada1478ace..eb51613e60 100644 --- a/packages/backend-core/src/migrations/index.js +++ b/packages/backend-core/src/migrations/index.js @@ -27,7 +27,7 @@ exports.getMigrationsDoc = async db => { } } -const runMigration = async (migration, options = {}) => { +exports.runMigration = async (migration, options = {}) => { const tenantId = getTenantId() const migrationType = migration.type const migrationName = migration.name @@ -110,7 +110,7 @@ exports.runMigrations = async (migrations, options = {}) => { // for all migrations for (const migration of migrations) { // run the migration - await doInTenant(tenantId, () => runMigration(migration, options)) + await doInTenant(tenantId, () => exports.runMigration(migration, options)) } } console.log("Migrations complete") diff --git a/packages/server/src/migrations/functions/backfill/app.ts b/packages/server/src/migrations/functions/backfill/app.ts index 6080f75b1f..4b2d20b008 100644 --- a/packages/server/src/migrations/functions/backfill/app.ts +++ b/packages/server/src/migrations/functions/backfill/app.ts @@ -1,4 +1,10 @@ -import * as app from "./app/app" +import * as apps from "./app/apps" +import * as automations from "./app/automations" +import * as datasources from "./app/datasources" +import * as layouts from "./app/layouts" +import * as queries from "./app/queries" +import * as roles from "./app/roles" +import * as tables from "./app/tables" /** * Date: @@ -9,5 +15,11 @@ import * as app from "./app/app" */ export const run = async (appDb: any) => { - await app.backfill(appDb) + await apps.backfill(appDb) + await automations.backfill(appDb) + await datasources.backfill(appDb) + await layouts.backfill(appDb) + await queries.backfill(appDb) + await roles.backfill(appDb) + await tables.backfill(appDb) } diff --git a/packages/server/src/migrations/functions/backfill/app/app.ts b/packages/server/src/migrations/functions/backfill/app/apps.ts similarity index 100% rename from packages/server/src/migrations/functions/backfill/app/app.ts rename to packages/server/src/migrations/functions/backfill/app/apps.ts diff --git a/packages/server/src/migrations/functions/backfill/app/automation.ts b/packages/server/src/migrations/functions/backfill/app/automation.ts deleted file mode 100644 index 86792d072d..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/automation.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { events, db } from "@budibase/backend-core" -import { Automation, AutomationStep } from "@budibase/types" - -export const backfill = async (appDb: any) => { - const automations: Automation[] = [] - - for (const automation of automations) { - events.automation.created(automation) - - const steps: AutomationStep[] = [] - for (const step of steps) { - events.automation.stepCreated(automation, step) - } - } -} diff --git a/packages/server/src/migrations/functions/backfill/app/automations.ts b/packages/server/src/migrations/functions/backfill/app/automations.ts new file mode 100644 index 0000000000..1c1f0e1da4 --- /dev/null +++ b/packages/server/src/migrations/functions/backfill/app/automations.ts @@ -0,0 +1,26 @@ +import { events, db } from "@budibase/backend-core" +import { getAutomationParams } from "../../../../db/utils" +import { Automation } from "@budibase/types" + +const getAutomations = async (appDb: any): Promise => { + const response = await appDb.allDocs( + getAutomationParams(null, { + include_docs: true, + }) + ) + return response.rows.map((row: any) => row.doc) +} + +export const backfill = async (appDb: any) => { + if (db.isDevAppID(appDb.name)) { + const automations = await getAutomations(appDb) + + for (const automation of automations) { + events.automation.created(automation) + + for (const step of automation.definition.steps) { + events.automation.stepCreated(automation, step) + } + } + } +} diff --git a/packages/server/src/migrations/functions/backfill/app/datasource.ts b/packages/server/src/migrations/functions/backfill/app/datasource.ts deleted file mode 100644 index fa01c2e3ad..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/datasource.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { events, db } from "@budibase/backend-core" -import { Datasource } from "@budibase/types" - -export const backfill = async (appDb: any) => { - const datasources: Datasource[] = [] - - for (const datasource of datasources) { - events.datasource.created(datasource) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app/datasources.ts b/packages/server/src/migrations/functions/backfill/app/datasources.ts new file mode 100644 index 0000000000..7ebc92ab7c --- /dev/null +++ b/packages/server/src/migrations/functions/backfill/app/datasources.ts @@ -0,0 +1,22 @@ +import { events, db } from "@budibase/backend-core" +import { getDatasourceParams } from "../../../../db/utils" +import { Datasource } from "@budibase/types" + +const getDatasources = async (appDb: any): Promise => { + const response = await appDb.allDocs( + getDatasourceParams(null, { + include_docs: true, + }) + ) + return response.rows.map((row: any) => row.doc) +} + +export const backfill = async (appDb: any) => { + if (db.isDevAppID(appDb.name)) { + const datasources: Datasource[] = await getDatasources(appDb) + + for (const datasource of datasources) { + events.datasource.created(datasource) + } + } +} diff --git a/packages/server/src/migrations/functions/backfill/app/layout.ts b/packages/server/src/migrations/functions/backfill/app/layout.ts deleted file mode 100644 index c2e4aa0faf..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/layout.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { events, db } from "@budibase/backend-core" -import { Layout } from "@budibase/types" - -export const backfill = async (appDb: any) => { - const layouts: Layout[] = [] - - for (const layout of layouts) { - events.layout.created(layout) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app/layouts.ts b/packages/server/src/migrations/functions/backfill/app/layouts.ts new file mode 100644 index 0000000000..89c3aa9dc2 --- /dev/null +++ b/packages/server/src/migrations/functions/backfill/app/layouts.ts @@ -0,0 +1,22 @@ +import { events, db } from "@budibase/backend-core" +import { getLayoutParams } from "../../../../db/utils" +import { Layout } from "@budibase/types" + +const getLayouts = async (appDb: any): Promise => { + const response = await appDb.allDocs( + getLayoutParams(null, { + include_docs: true, + }) + ) + return response.rows.map((row: any) => row.doc) +} + +export const backfill = async (appDb: any) => { + if (db.isDevAppID(appDb.name)) { + const layouts: Layout[] = await getLayouts(appDb) + + for (const layout of layouts) { + events.layout.created(layout) + } + } +} diff --git a/packages/server/src/migrations/functions/backfill/app/queries.ts b/packages/server/src/migrations/functions/backfill/app/queries.ts new file mode 100644 index 0000000000..6c6ad985f2 --- /dev/null +++ b/packages/server/src/migrations/functions/backfill/app/queries.ts @@ -0,0 +1,33 @@ +import { events, db } from "@budibase/backend-core" +import { getQueryParams } from "../../../../db/utils" +import { Query, Datasource } from "@budibase/types" + +const getQueries = async (appDb: any): Promise => { + const response = await appDb.allDocs( + getQueryParams(null, { + include_docs: true, + }) + ) + return response.rows.map((row: any) => row.doc) +} + +const getDatasource = async ( + appDb: any, + datasourceId: string +): Promise => { + return appDb.get(datasourceId) +} + +export const backfill = async (appDb: any) => { + if (db.isDevAppID(appDb.name)) { + const queries: Query[] = await getQueries(appDb) + + for (const query of queries) { + const datasource: Datasource = await getDatasource( + appDb, + query.datasourceId + ) + events.query.created(datasource, query) + } + } +} diff --git a/packages/server/src/migrations/functions/backfill/app/query.ts b/packages/server/src/migrations/functions/backfill/app/query.ts deleted file mode 100644 index 7031db048d..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/query.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { events, db } from "@budibase/backend-core" -import { Query, Datasource } from "@budibase/types" - -export const backfill = async (appDb: any) => { - const queries: Query[] = [] - - for (const query of queries) { - const datasource: Datasource = {} - events.query.created(datasource, query) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app/role.ts b/packages/server/src/migrations/functions/backfill/app/role.ts deleted file mode 100644 index 9162d9a546..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/role.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { events, db } from "@budibase/backend-core" -import { Role, User } from "@budibase/types" - -export const backfill = async (appDb: any) => { - const roles: Role[] = [] - - for (const role of roles) { - events.role.created(role) - const user: User = {} - events.role.assigned(user, role) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app/roles.ts b/packages/server/src/migrations/functions/backfill/app/roles.ts new file mode 100644 index 0000000000..de2ce9d37b --- /dev/null +++ b/packages/server/src/migrations/functions/backfill/app/roles.ts @@ -0,0 +1,22 @@ +import { events, db } from "@budibase/backend-core" +import { getRoleParams } from "../../../../db/utils" +import { Role } from "@budibase/types" + +const getRoles = async (appDb: any): Promise => { + const response = await appDb.allDocs( + getRoleParams(null, { + include_docs: true, + }) + ) + return response.rows.map((row: any) => row.doc) +} + +export const backfill = async (appDb: any) => { + if (db.isDevAppID(appDb.name)) { + const roles = await getRoles(appDb) + + for (const role of roles) { + events.role.created(role) + } + } +} diff --git a/packages/server/src/migrations/functions/backfill/app/row.ts b/packages/server/src/migrations/functions/backfill/app/rows.ts similarity index 100% rename from packages/server/src/migrations/functions/backfill/app/row.ts rename to packages/server/src/migrations/functions/backfill/app/rows.ts diff --git a/packages/server/src/migrations/functions/backfill/app/screen.ts b/packages/server/src/migrations/functions/backfill/app/screens.ts similarity index 100% rename from packages/server/src/migrations/functions/backfill/app/screen.ts rename to packages/server/src/migrations/functions/backfill/app/screens.ts diff --git a/packages/server/src/migrations/functions/backfill/app/table.ts b/packages/server/src/migrations/functions/backfill/app/table.ts deleted file mode 100644 index 116e7bbbdf..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/table.ts +++ /dev/null @@ -1,3 +0,0 @@ -// TABLE_CREATED = "table:created", -// -// TABLE_DATA_IMPORTED = "table:data:imported", diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts new file mode 100644 index 0000000000..233474ffbd --- /dev/null +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -0,0 +1,22 @@ +import { events, db } from "@budibase/backend-core" +import { getTableParams } from "../../../../db/utils" +import { Table } from "@budibase/types" + +const getTables = async (appDb: any): Promise => { + const response = await appDb.allDocs( + getTableParams(null, { + include_docs: true, + }) + ) + return response.rows.map((row: any) => row.doc) +} + +export const backfill = async (appDb: any) => { + if (db.isDevAppID(appDb.name)) { + const tables = await getTables(appDb) + + for (const table of tables) { + events.table.created(table) + } + } +} diff --git a/packages/server/src/migrations/functions/backfill/tests/app.spec.ts b/packages/server/src/migrations/functions/backfill/tests/app.spec.ts deleted file mode 100644 index a8caebff1c..0000000000 --- a/packages/server/src/migrations/functions/backfill/tests/app.spec.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { tenancy, events } from "@budibase/backend-core" -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import { backfill } from "../app/app" - -describe("app backfill", () => { - const config = new TestConfig() - - beforeEach(async () => { - await config.init() - jest.clearAllMocks() - }) - - afterEach(() => { - config.end() - }) - - it("should backfill dev app", async () => { - await config.doInContext(null, async () => { - const db = tenancy.getAppDB({}) - - await backfill(db) - - expect(events.app.created).toBeCalledTimes(1) - expect(events.app.published).toBeCalledTimes(0) - }) - }) - - it("should backfill prod app", async () => { - await config.doInContext( - null, - async () => { - const db = tenancy.getAppDB({}) - - await backfill(db) - - // expect(events.app.created).toBeCalledTimes(0) - expect(events.app.published).toBeCalledTimes(1) - }, - { prod: true } - ) - }) -}) diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts new file mode 100644 index 0000000000..9a01778e29 --- /dev/null +++ b/packages/server/src/migrations/tests/index.spec.ts @@ -0,0 +1,52 @@ +import { events, migrations } from "@budibase/backend-core" +import TestConfig from "../../tests/utilities/TestConfiguration" +import structures from "../../tests/utilities/structures" +import { MIGRATIONS } from "../" + +jest.setTimeout(100000) + +describe("migrations", () => { + const config = new TestConfig() + + beforeAll(async () => { + await config.init() + }) + + afterAll(() => { + config.end() + }) + + describe("backfill", () => { + it("runs app db migration", async () => { + await config.doInContext(null, async () => { + await config.createAutomation() + await config.createAutomation(structures.newAutomation()) + await config.createDatasource() + await config.createDatasource() + await config.createLayout() + await config.createQuery() + await config.createQuery() + await config.createRole() + await config.createRole() + await config.createTable() + await config.createTable() + + jest.clearAllMocks() + const migration = MIGRATIONS.filter( + m => m.name === "event_app_backfill" + )[0] + await migrations.runMigration(migration) + + expect(events.app.created).toBeCalledTimes(1) + expect(events.app.published).toBeCalledTimes(1) + expect(events.automation.created).toBeCalledTimes(2) + expect(events.automation.stepCreated).toBeCalledTimes(1) + expect(events.datasource.created).toBeCalledTimes(2) + expect(events.layout.created).toBeCalledTimes(3) + expect(events.query.created).toBeCalledTimes(2) + expect(events.role.created).toBeCalledTimes(2) + expect(events.table.created).toBeCalledTimes(3) + }) + }) + }) +}) diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index 80a2afa543..4dd64835c6 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -1,4 +1,9 @@ -export interface Automation {} +export interface Automation { + definition: { + steps: AutomationStep[] + trigger: AutomationTrigger + } +} export interface AutomationStep {} diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts index c4d55173dc..26f498979d 100644 --- a/packages/types/src/documents/app/index.ts +++ b/packages/types/src/documents/app/index.ts @@ -9,3 +9,4 @@ export * from "./screen" export * from "./view" export * from "./document" export * from "./row" +export * from "./user" diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts index ba038af326..7edda3eba8 100644 --- a/packages/types/src/documents/app/query.ts +++ b/packages/types/src/documents/app/query.ts @@ -1 +1,3 @@ -export interface Query {} +export interface Query { + datasourceId: string +} diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts index 304e11b799..292ab71806 100644 --- a/packages/types/src/documents/app/role.ts +++ b/packages/types/src/documents/app/role.ts @@ -1 +1,3 @@ -export interface Role {} +import { Document } from "./document" + +export interface Role extends Document {} diff --git a/packages/types/src/documents/app/user.ts b/packages/types/src/documents/app/user.ts new file mode 100644 index 0000000000..233da6c454 --- /dev/null +++ b/packages/types/src/documents/app/user.ts @@ -0,0 +1,3 @@ +export interface UserMetadata { + roleId: string +} diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts index e918b910d0..43592582c7 100644 --- a/packages/types/src/documents/global/user.ts +++ b/packages/types/src/documents/global/user.ts @@ -1 +1,7 @@ -export interface User {} +export interface User { + roles: UserRoles +} + +export interface UserRoles { + [key: string]: string +} diff --git a/packages/worker/package.json b/packages/worker/package.json index 02609f081e..10f69d2fc5 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -64,6 +64,7 @@ "server-destroy": "^1.0.1" }, "devDependencies": { + "@budibase/types": "^1.0.126-alpha.0", "@types/jest": "^26.0.23", "@types/koa": "^2.13.3", "@types/koa-router": "^7.4.2", diff --git a/packages/worker/src/sdk/users/events.ts b/packages/worker/src/sdk/users/events.ts index 98fb68efda..76108f6853 100644 --- a/packages/worker/src/sdk/users/events.ts +++ b/packages/worker/src/sdk/users/events.ts @@ -1,4 +1,5 @@ -const { events } = require("@budibase/backend-core") +import { events } from "@budibase/backend-core" +import { User, UserRoles } from "@budibase/types" export const handleDeleteEvents = (user: any) => { events.user.deleted(user) @@ -12,23 +13,31 @@ export const handleDeleteEvents = (user: any) => { } } -const assignAppRoleEvents = (roles: any, existingRoles: any) => { +const assignAppRoleEvents = ( + user: User, + roles: UserRoles, + existingRoles: UserRoles +) => { for (const [appId, role] of Object.entries(roles)) { // app role in existing is not same as new if (!existingRoles || existingRoles[appId] !== role) { - events.role.assigned(role) + events.role.assigned(user, role) } } } -const unassignAppRoleEvents = (roles: any, existingRoles: any) => { +const unassignAppRoleEvents = ( + user: User, + roles: UserRoles, + existingRoles: UserRoles +) => { if (!existingRoles) { return } for (const [appId, role] of Object.entries(existingRoles)) { // app role in new is not same as existing if (!roles || roles[appId] !== role) { - events.role.unassigned(role) + events.role.unassigned(user, role) } } } @@ -37,8 +46,8 @@ const handleAppRoleEvents = (user: any, existingUser: any) => { const roles = user.roles const existingRoles = existingUser?.roles - assignAppRoleEvents(roles, existingRoles) - unassignAppRoleEvents(roles, existingRoles) + assignAppRoleEvents(user, roles, existingRoles) + unassignAppRoleEvents(user, roles, existingRoles) } export const handleSaveEvents = (user: any, existingUser: any) => {