From bf59ecf6b7e0724e43daacabfa8fa37bf8a85d95 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 29 Aug 2023 13:35:12 +0200 Subject: [PATCH 1/6] Split between internal and external views --- packages/server/src/api/controllers/datasource.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 19a38206dc..399d5f1d0c 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -17,7 +17,6 @@ import { FetchDatasourceInfoRequest, FetchDatasourceInfoResponse, IntegrationBase, - RestConfig, SourceName, UpdateDatasourceResponse, UserCtx, @@ -27,7 +26,6 @@ import { import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets" -import { areRESTVariablesValid } from "../../sdk/app/datasources/datasources" function getErrorTables(errors: any, errorType: string) { return Object.entries(errors) From 7e2ecc1b3afe6044e229a25026dc8f0e21b7ce28 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 29 Aug 2023 13:35:22 +0200 Subject: [PATCH 2/6] Split between internal and external views --- packages/server/src/sdk/app/views/external.ts | 77 +++++++++++++++++++ packages/server/src/sdk/app/views/index.ts | 69 ++++------------- packages/server/src/sdk/app/views/internal.ts | 77 +++++++++++++++++++ 3 files changed, 171 insertions(+), 52 deletions(-) create mode 100644 packages/server/src/sdk/app/views/external.ts create mode 100644 packages/server/src/sdk/app/views/internal.ts diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts new file mode 100644 index 0000000000..97b47fbcb5 --- /dev/null +++ b/packages/server/src/sdk/app/views/external.ts @@ -0,0 +1,77 @@ +import { ViewV2 } from "@budibase/types" +import { context, HTTPError } from "@budibase/backend-core" + +import sdk from "../../../sdk" +import * as utils from "../../../db/utils" +import { enrichSchema, isV2 } from "." + +export async function get( + viewId: string, + opts?: { enriched: boolean } +): Promise { + const { tableId } = utils.extractViewInfoFromID(viewId) + const table = await sdk.tables.getTable(tableId) + const views = Object.values(table.views!) + const found = views.find(v => isV2(v) && v.id === viewId) + if (!found) { + throw new Error("No view found") + } + if (opts?.enriched) { + return enrichSchema(found, table.schema) as ViewV2 + } else { + return found as ViewV2 + } +} + +export async function create( + tableId: string, + viewRequest: Omit +): Promise { + const view: ViewV2 = { + ...viewRequest, + id: utils.generateViewID(tableId), + version: 2, + } + + const db = context.getAppDB() + + const table = await sdk.tables.getTable(tableId) + table.views ??= {} + + table.views[view.name] = view + await db.put(table) + return view +} + +export async function update(tableId: string, view: ViewV2): Promise { + const db = context.getAppDB() + const table = await sdk.tables.getTable(tableId) + table.views ??= {} + + const existingView = Object.values(table.views).find( + v => isV2(v) && v.id === view.id + ) + if (!existingView) { + throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404) + } + + console.log("set to", view) + delete table.views[existingView.name] + table.views[view.name] = view + await db.put(table) + return view +} + +export async function remove(viewId: string): Promise { + const db = context.getAppDB() + + const view = await get(viewId) + const table = await sdk.tables.getTable(view?.tableId) + if (!view) { + throw new HTTPError(`View ${viewId} not found`, 404) + } + + delete table.views![view?.name] + await db.put(table) + return view +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index c81fab607a..927f82cc68 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,64 +1,38 @@ import { RenameColumn, TableSchema, View, ViewV2 } from "@budibase/types" -import { context, db as dbCore, HTTPError } from "@budibase/backend-core" +import { db as dbCore } from "@budibase/backend-core" import { cloneDeep } from "lodash" import sdk from "../../../sdk" import * as utils from "../../../db/utils" +import { isExternalTable } from "../../../integrations/utils" + +import * as internal from "./internal" +import * as external from "./external" + +function pickApi(tableId: any) { + if (isExternalTable(tableId)) { + return external + } + return internal +} export async function get( viewId: string, opts?: { enriched: boolean } ): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) - const table = await sdk.tables.getTable(tableId) - const views = Object.values(table.views!) - const found = views.find(v => isV2(v) && v.id === viewId) - if (!found) { - throw new Error("No view found") - } - if (opts?.enriched) { - return enrichSchema(found, table.schema) as ViewV2 - } else { - return found as ViewV2 - } + return pickApi(tableId).get(viewId, opts) } export async function create( tableId: string, viewRequest: Omit ): Promise { - const view: ViewV2 = { - ...viewRequest, - id: utils.generateViewID(tableId), - version: 2, - } - - const db = context.getAppDB() - const table = await sdk.tables.getTable(tableId) - table.views ??= {} - - table.views[view.name] = view - await db.put(table) - return view + return pickApi(tableId).create(tableId, viewRequest) } export async function update(tableId: string, view: ViewV2): Promise { - const db = context.getAppDB() - const table = await sdk.tables.getTable(tableId) - table.views ??= {} - - const existingView = Object.values(table.views).find( - v => isV2(v) && v.id === view.id - ) - if (!existingView) { - throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404) - } - - console.log("set to", view) - delete table.views[existingView.name] - table.views[view.name] = view - await db.put(table) - return view + return pickApi(tableId).update(tableId, view) } export function isV2(view: View | ViewV2): view is ViewV2 { @@ -66,17 +40,8 @@ export function isV2(view: View | ViewV2): view is ViewV2 { } export async function remove(viewId: string): Promise { - const db = context.getAppDB() - - const view = await get(viewId) - const table = await sdk.tables.getTable(view?.tableId) - if (!view) { - throw new HTTPError(`View ${viewId} not found`, 404) - } - - delete table.views![view?.name] - await db.put(table) - return view + const { tableId } = utils.extractViewInfoFromID(viewId) + return pickApi(tableId).remove(viewId) } export function allowedFields(view: View | ViewV2) { diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts new file mode 100644 index 0000000000..4f65d24bfa --- /dev/null +++ b/packages/server/src/sdk/app/views/internal.ts @@ -0,0 +1,77 @@ +import { ViewV2 } from "@budibase/types" +import { context, HTTPError } from "@budibase/backend-core" + +import sdk from "../../../sdk" +import * as utils from "../../../db/utils" +import { enrichSchema, isV2 } from "." + +export async function get( + viewId: string, + opts?: { enriched: boolean } +): Promise { + const { tableId } = utils.extractViewInfoFromID(viewId) + const table = await sdk.tables.getTable(tableId) + const views = Object.values(table.views!) + const found = views.find(v => isV2(v) && v.id === viewId) + if (!found) { + throw new Error("No view found") + } + if (opts?.enriched) { + return enrichSchema(found, table.schema) as ViewV2 + } else { + return found as ViewV2 + } +} + +export async function create( + tableId: string, + viewRequest: Omit +): Promise { + const view: ViewV2 = { + ...viewRequest, + id: utils.generateViewID(tableId), + version: 2, + } + + const db = context.getAppDB() + + const table = await sdk.tables.getTable(tableId) + table.views ??= {} + + table.views[view.name] = view + await db.put(table) + return view +} + +export async function update(tableId: string, view: ViewV2): Promise { + const db = context.getAppDB() + const table = await sdk.tables.getTable(tableId) + table.views ??= {} + + const existingView = Object.values(table.views).find( + v => isV2(v) && v.id === view.id + ) + if (!existingView) { + throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404) + } + + console.log("set to", view) + delete table.views[existingView.name] + table.views[view.name] = view + await db.put(table) + return view +} + +export async function remove(viewId: string): Promise { + const db = context.getAppDB() + + const view = await get(viewId) + const table = await sdk.tables.getTable(view?.tableId) + if (!view) { + throw new HTTPError(`View ${viewId} not found`, 404) + } + + delete table.views![view?.name] + await db.put(table) + return view +} From 6f2aa3287839adbc165e41cca18bdd35ca9ecaea Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 29 Aug 2023 15:39:56 +0200 Subject: [PATCH 3/6] Implement external --- packages/server/src/sdk/app/views/external.ts | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index 97b47fbcb5..bc240ae17b 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -4,13 +4,18 @@ import { context, HTTPError } from "@budibase/backend-core" import sdk from "../../../sdk" import * as utils from "../../../db/utils" import { enrichSchema, isV2 } from "." +import { breakExternalTableId } from "../../../integrations/utils" export async function get( viewId: string, opts?: { enriched: boolean } ): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) - const table = await sdk.tables.getTable(tableId) + + const { datasourceId, tableName } = breakExternalTableId(tableId) + const ds = await sdk.datasources.get(datasourceId!) + + const table = ds.entities![tableName!] const views = Object.values(table.views!) const found = views.find(v => isV2(v) && v.id === viewId) if (!found) { @@ -35,20 +40,23 @@ export async function create( const db = context.getAppDB() - const table = await sdk.tables.getTable(tableId) - table.views ??= {} - - table.views[view.name] = view - await db.put(table) + const { datasourceId, tableName } = breakExternalTableId(tableId) + const ds = await sdk.datasources.get(datasourceId!) + ds.entities![tableName!].views ??= {} + ds.entities![tableName!].views![view.name] = view + await db.put(ds) return view } export async function update(tableId: string, view: ViewV2): Promise { const db = context.getAppDB() - const table = await sdk.tables.getTable(tableId) - table.views ??= {} - const existingView = Object.values(table.views).find( + const { datasourceId, tableName } = breakExternalTableId(tableId) + const ds = await sdk.datasources.get(datasourceId!) + ds.entities![tableName!].views ??= {} + const views = ds.entities![tableName!].views! + + const existingView = Object.values(views).find( v => isV2(v) && v.id === view.id ) if (!existingView) { @@ -56,9 +64,9 @@ export async function update(tableId: string, view: ViewV2): Promise { } console.log("set to", view) - delete table.views[existingView.name] - table.views[view.name] = view - await db.put(table) + delete views[existingView.name] + views[view.name] = view + await db.put(ds) return view } @@ -66,12 +74,15 @@ export async function remove(viewId: string): Promise { const db = context.getAppDB() const view = await get(viewId) - const table = await sdk.tables.getTable(view?.tableId) + if (!view) { throw new HTTPError(`View ${viewId} not found`, 404) } - delete table.views![view?.name] - await db.put(table) + const { datasourceId, tableName } = breakExternalTableId(view.tableId) + const ds = await sdk.datasources.get(datasourceId!) + + delete ds.entities![tableName!].views![view?.name] + await db.put(ds) return view } From ed9e0ed2fffa7df7f5005211b2c96bd200f57ca4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 29 Aug 2023 16:13:44 +0200 Subject: [PATCH 4/6] Test external --- .../src/api/routes/tests/viewV2.spec.ts | 61 +++++++++++++------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 29e5f98c26..371e04b31d 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -10,6 +10,8 @@ import { ViewV2, } from "@budibase/types" import { generator } from "@budibase/backend-core/tests" +import { buildExternalTableId } from "../../../integrations/utils" +import { generateDatasourceID } from "../../../db/utils" function priceTable(): Table { return { @@ -32,27 +34,52 @@ function priceTable(): Table { } } -describe("/v2/views", () => { - const config = setup.getConfig() +const config = setup.getConfig() - afterAll(setup.afterAll) +beforeAll(async () => { + await config.init() +}) + +describe.each([ + // ["internal ds", () => config.createTable(priceTable())], + [ + "external ds", + async () => { + const datasource = await config.createDatasource({ + datasource: { + ...setup.structures.basicDatasource().datasource, + plus: true, + _id: generateDatasourceID({ plus: true }), + }, + }) + + return config.createTable({ + ...priceTable(), + sourceId: datasource._id, + type: "external", + }) + }, + ], +])("/v2/views (%s)", (_, tableBuilder) => { + let table: Table beforeAll(async () => { - await config.init() - await config.createTable(priceTable()) + table = await tableBuilder() }) + afterAll(setup.afterAll) + describe("create", () => { it("persist the view when the view is successfully created", async () => { const newView: CreateViewRequest = { name: generator.name(), - tableId: config.table!._id!, + tableId: table._id!, } const res = await config.api.viewV2.create(newView) expect(res).toEqual({ ...newView, - id: expect.stringMatching(new RegExp(`${config.table?._id!}_`)), + id: expect.stringMatching(new RegExp(`${table._id!}_`)), version: 2, }) }) @@ -60,7 +87,7 @@ describe("/v2/views", () => { it("can persist views with all fields", async () => { const newView: Required = { name: generator.name(), - tableId: config.table!._id!, + tableId: table._id!, primaryDisplay: generator.word(), query: [{ operator: "equal", field: "field", value: "value" }], sort: { @@ -87,7 +114,7 @@ describe("/v2/views", () => { it("persist only UI schema overrides", async () => { const newView: CreateViewRequest = { name: generator.name(), - tableId: config.table!._id!, + tableId: table._id!, schema: { Price: { name: "Price", @@ -124,7 +151,7 @@ describe("/v2/views", () => { it("will not throw an exception if the schema is 'deleting' non UI fields", async () => { const newView: CreateViewRequest = { name: generator.name(), - tableId: config.table!._id!, + tableId: table._id!, schema: { Price: { name: "Price", @@ -153,7 +180,7 @@ describe("/v2/views", () => { }) it("can update an existing view data", async () => { - const tableId = config.table!._id! + const tableId = table._id! await config.api.viewV2.update({ ...view, query: [{ operator: "equal", field: "newField", value: "thatValue" }], @@ -176,7 +203,7 @@ describe("/v2/views", () => { }) it("can update all fields", async () => { - const tableId = config.table!._id! + const tableId = table._id! const updatedData: Required = { version: view.version, @@ -210,7 +237,7 @@ describe("/v2/views", () => { [view.name]: { ...updatedData, schema: { - ...config.table!.schema, + ...table.schema, Category: expect.objectContaining({ visible: false, }), @@ -226,7 +253,7 @@ describe("/v2/views", () => { }) it("can update an existing view name", async () => { - const tableId = config.table!._id! + const tableId = table._id! await config.api.viewV2.update({ ...view, name: "View B" }) expect(await config.api.table.get(tableId)).toEqual( @@ -239,7 +266,7 @@ describe("/v2/views", () => { }) it("cannot update an unexisting views nor edit ids", async () => { - const tableId = config.table!._id! + const tableId = table._id! await config.api.viewV2.update( { ...view, id: generator.guid() }, { expectStatus: 404 } @@ -258,7 +285,7 @@ describe("/v2/views", () => { }) it("cannot update views with the wrong tableId", async () => { - const tableId = config.table!._id! + const tableId = table._id! await config.api.viewV2.update( { ...view, @@ -379,7 +406,7 @@ describe("/v2/views", () => { }) it("can delete an existing view", async () => { - const tableId = config.table!._id! + const tableId = table._id! const getPersistedView = async () => (await config.api.table.get(tableId)).views![view.name] From 24332f2f4516541c83b795c8293b0e0432606902 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 29 Aug 2023 16:39:19 +0200 Subject: [PATCH 5/6] Fix tests --- .../src/api/routes/tests/viewV2.spec.ts | 53 +++++++------------ 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 371e04b31d..a9c9f3a320 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -10,7 +10,6 @@ import { ViewV2, } from "@budibase/types" import { generator } from "@budibase/backend-core/tests" -import { buildExternalTableId } from "../../../integrations/utils" import { generateDatasourceID } from "../../../db/utils" function priceTable(): Table { @@ -41,7 +40,7 @@ beforeAll(async () => { }) describe.each([ - // ["internal ds", () => config.createTable(priceTable())], + ["internal ds", () => config.createTable(priceTable())], [ "external ds", async () => { @@ -175,7 +174,8 @@ describe.each([ let view: ViewV2 beforeEach(async () => { - await config.createTable(priceTable()) + table = await tableBuilder() + view = await config.api.viewV2.create({ name: "View A" }) }) @@ -186,19 +186,12 @@ describe.each([ query: [{ operator: "equal", field: "newField", value: "thatValue" }], }) - expect(await config.api.table.get(tableId)).toEqual({ - ...config.table, - views: { - [view.name]: { - ...view, - query: [ - { operator: "equal", field: "newField", value: "thatValue" }, - ], - schema: expect.anything(), - }, + expect((await config.api.table.get(tableId)).views).toEqual({ + [view.name]: { + ...view, + query: [{ operator: "equal", field: "newField", value: "thatValue" }], + schema: expect.anything(), }, - _rev: expect.any(String), - updatedAt: expect.any(String), }) }) @@ -231,24 +224,19 @@ describe.each([ } await config.api.viewV2.update(updatedData) - expect(await config.api.table.get(tableId)).toEqual({ - ...config.table, - views: { - [view.name]: { - ...updatedData, - schema: { - ...table.schema, - Category: expect.objectContaining({ - visible: false, - }), - Price: expect.objectContaining({ - visible: false, - }), - }, + expect((await config.api.table.get(tableId)).views).toEqual({ + [view.name]: { + ...updatedData, + schema: { + ...table.schema, + Category: expect.objectContaining({ + visible: false, + }), + Price: expect.objectContaining({ + visible: false, + }), }, }, - _rev: expect.any(String), - updatedAt: expect.any(String), }) }) @@ -401,7 +389,6 @@ describe.each([ let view: ViewV2 beforeAll(async () => { - await config.createTable(priceTable()) view = await config.api.viewV2.create() }) @@ -420,8 +407,6 @@ describe.each([ describe("fetch view (through table)", () => { it("should be able to fetch a view V2", async () => { - const table = await config.createTable(priceTable()) - const newView: CreateViewRequest = { name: generator.name(), tableId: table._id!, From 8e4138f50debad581bc835db60366cef2ccf5df7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 29 Aug 2023 18:32:18 +0200 Subject: [PATCH 6/6] Lint --- packages/server/src/sdk/app/views/internal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts index 4f65d24bfa..97b47fbcb5 100644 --- a/packages/server/src/sdk/app/views/internal.ts +++ b/packages/server/src/sdk/app/views/internal.ts @@ -1,4 +1,4 @@ -import { ViewV2 } from "@budibase/types" +import { ViewV2 } from "@budibase/types" import { context, HTTPError } from "@budibase/backend-core" import sdk from "../../../sdk"