From b335e492d79aa6cd5027253db3eb75cd2e668469 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 10 Jul 2023 17:42:27 +0100 Subject: [PATCH 01/30] Show all users editing an app in the app list --- .../src/components/common/NavItem.svelte | 2 +- .../src/components/start/AppRow.svelte | 6 ++-- .../builder/app/[application]/_layout.svelte | 2 +- .../src/components}/UserAvatars.svelte | 0 .../frontend-core/src/components/index.js | 1 + .../server/src/api/controllers/application.ts | 28 ++++++++++++++++++- packages/server/src/websockets/websocket.ts | 17 ++++++++--- packages/types/src/documents/app/app.ts | 2 ++ 8 files changed, 48 insertions(+), 10 deletions(-) rename packages/{builder/src/pages/builder/app/[application]/_components => frontend-core/src/components}/UserAvatars.svelte (100%) diff --git a/packages/builder/src/components/common/NavItem.svelte b/packages/builder/src/components/common/NavItem.svelte index 6f87c8c362..7d61751364 100644 --- a/packages/builder/src/components/common/NavItem.svelte +++ b/packages/builder/src/components/common/NavItem.svelte @@ -2,7 +2,7 @@ import { Icon } from "@budibase/bbui" import { createEventDispatcher, getContext } from "svelte" import { helpers } from "@budibase/shared-core" - import UserAvatars from "../../pages/builder/app/[application]/_components/UserAvatars.svelte" + import { UserAvatars } from "@budibase/frontend-core" export let icon export let withArrow = false diff --git a/packages/builder/src/components/start/AppRow.svelte b/packages/builder/src/components/start/AppRow.svelte index 50e6b8466a..2e7719987d 100644 --- a/packages/builder/src/components/start/AppRow.svelte +++ b/packages/builder/src/components/start/AppRow.svelte @@ -2,12 +2,12 @@ import { Heading, Body, Button, Icon } from "@budibase/bbui" import { processStringSync } from "@budibase/string-templates" import { goto } from "@roxi/routify" - import { UserAvatar } from "@budibase/frontend-core" + import { UserAvatars } from "@budibase/frontend-core" export let app export let lockedAction - $: editing = app?.lockedBy != null + $: editing = app.sessions?.length const handleDefaultClick = () => { if (window.innerWidth < 640) { @@ -41,7 +41,7 @@
{#if editing} Currently editing - + {:else if app.updatedAt} {processStringSync("Updated {{ duration time 'millisecond' }} ago", { time: new Date().getTime() - new Date(app.updatedAt).getTime(), diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 8e637d6fa2..7eb7b35c7a 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -26,7 +26,7 @@ import TourWrap from "components/portal/onboarding/TourWrap.svelte" import TourPopover from "components/portal/onboarding/TourPopover.svelte" import BuilderSidePanel from "./_components/BuilderSidePanel.svelte" - import UserAvatars from "./_components/UserAvatars.svelte" + import { UserAvatars } from "@budibase/frontend-core" import { TOUR_KEYS, TOURS } from "components/portal/onboarding/tours.js" import PreviewOverlay from "./_components/PreviewOverlay.svelte" diff --git a/packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte b/packages/frontend-core/src/components/UserAvatars.svelte similarity index 100% rename from packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte rename to packages/frontend-core/src/components/UserAvatars.svelte diff --git a/packages/frontend-core/src/components/index.js b/packages/frontend-core/src/components/index.js index 3005c85d01..01a7c78cb8 100644 --- a/packages/frontend-core/src/components/index.js +++ b/packages/frontend-core/src/components/index.js @@ -2,4 +2,5 @@ export { default as SplitPage } from "./SplitPage.svelte" export { default as TestimonialPage } from "./TestimonialPage.svelte" export { default as Testimonial } from "./Testimonial.svelte" export { default as UserAvatar } from "./UserAvatar.svelte" +export { default as UserAvatars } from "./UserAvatars.svelte" export { Grid } from "./grid" diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index c068a422b0..5e12dd4d76 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -49,6 +49,7 @@ import { MigrationType, PlanType, Screen, + SocketSession, UserCtx, } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" @@ -178,11 +179,12 @@ export const addSampleData = async (ctx: UserCtx) => { export async function fetch(ctx: UserCtx) { const dev = ctx.query && ctx.query.status === AppStatus.DEV const all = ctx.query && ctx.query.status === AppStatus.ALL - const apps = (await dbCore.getAllApps({ dev, all })) as App[] + let apps = (await dbCore.getAllApps({ dev, all })) as App[] const appIds = apps .filter((app: any) => app.status === "development") .map((app: any) => app.appId) + // get the locks for all the dev apps if (dev || all) { const locks = await getLocksById(appIds) @@ -197,6 +199,30 @@ export async function fetch(ctx: UserCtx) { } } + // Get all builder sessions in each app + const sessions = await builderSocket?.getRoomSessions(appIds) + if (sessions?.length) { + let appSessionMap: { [key: string]: SocketSession[] } = {} + sessions.forEach(session => { + const room = session.room + if (!room) { + return + } + if (!appSessionMap[room]) { + appSessionMap[room] = [] + } + appSessionMap[room].push(session) + }) + apps.forEach(app => { + const sessions = appSessionMap[app.appId] + if (sessions?.length) { + app.sessions = sessions + } else { + delete app.sessions + } + }) + } + ctx.body = await checkAppMetadata(apps) } diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 24cac3c37d..9dea67ef5f 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -125,9 +125,18 @@ export class BaseSocket { } // Gets an array of all redis keys of users inside a certain room - async getRoomSessionIds(room: string): Promise { - const keys = await this.redisClient?.get(this.getRoomKey(room)) - return keys || [] + async getRoomSessionIds(room: string | string[]): Promise { + if (Array.isArray(room)) { + const roomKeys = room.map(this.getRoomKey.bind(this)) + const roomSessionIdMap = await this.redisClient?.bulkGet(roomKeys) + let sessionIds: any[] = [] + Object.values(roomSessionIdMap || {}).forEach(roomSessionIds => { + sessionIds = sessionIds.concat(roomSessionIds) + }) + return sessionIds + } else { + return (await this.redisClient?.get(this.getRoomKey(room))) || [] + } } // Sets the list of redis keys for users inside a certain room. @@ -137,7 +146,7 @@ export class BaseSocket { } // Gets a list of all users inside a certain room - async getRoomSessions(room?: string): Promise { + async getRoomSessions(room?: string | string[]): Promise { if (room) { const sessionIds = await this.getRoomSessionIds(room) const keys = sessionIds.map(this.getSessionKey.bind(this)) diff --git a/packages/types/src/documents/app/app.ts b/packages/types/src/documents/app/app.ts index 258eaef297..f42422b557 100644 --- a/packages/types/src/documents/app/app.ts +++ b/packages/types/src/documents/app/app.ts @@ -1,4 +1,5 @@ import { User, Document } from "../" +import { SocketSession } from "../../sdk" export type AppMetadataErrors = { [key: string]: string[] } @@ -17,6 +18,7 @@ export interface App extends Document { customTheme?: AppCustomTheme revertableVersion?: string lockedBy?: User + sessions?: SocketSession[] navigation?: AppNavigation automationErrors?: AppMetadataErrors icon?: AppIcon From 5fd004b86ef6e7239b071c60d1749b2586081e99 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 11 Jul 2023 18:06:17 +0200 Subject: [PATCH 02/30] Remove view on deletion --- .../backend/TableNavigator/popovers/EditViewPopover.svelte | 3 +-- packages/builder/src/stores/backend/views.js | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte index b637f46f5e..99f19935a1 100644 --- a/packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte +++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte @@ -35,9 +35,8 @@ try { const isSelected = decodeURIComponent($params.viewName) === $views.selectedViewName - const name = view.name const id = view.tableId - await views.delete(name) + await views.delete(view) notifications.success("View deleted") if (isSelected) { $goto(`./table/${id}`) diff --git a/packages/builder/src/stores/backend/views.js b/packages/builder/src/stores/backend/views.js index 2f396096c8..b01f31c4bd 100644 --- a/packages/builder/src/stores/backend/views.js +++ b/packages/builder/src/stores/backend/views.js @@ -26,14 +26,12 @@ export function createViewsStore() { } const deleteView = async view => { - await API.deleteView(view) + await API.deleteView(view.name) // Update tables tables.update(state => { const table = state.list.find(table => table._id === view.tableId) - if (table) { - delete table.views[view.name] - } + delete table.views[view.name] return { ...state } }) } From 85aa1059722a584e65e6920814186362a05a109d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 13 Jul 2023 12:17:24 +0200 Subject: [PATCH 03/30] Type row tests --- .../routes/tests/{row.spec.js => row.spec.ts} | 221 +++++++++--------- .../src/tests/utilities/TestConfiguration.ts | 5 +- 2 files changed, 119 insertions(+), 107 deletions(-) rename packages/server/src/api/routes/tests/{row.spec.js => row.spec.ts} (80%) diff --git a/packages/server/src/api/routes/tests/row.spec.js b/packages/server/src/api/routes/tests/row.spec.ts similarity index 80% rename from packages/server/src/api/routes/tests/row.spec.js rename to packages/server/src/api/routes/tests/row.spec.ts index a8041dac3b..8e99c30246 100644 --- a/packages/server/src/api/routes/tests/row.spec.js +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1,27 +1,27 @@ -const tk = require( "timekeeper") +import tk from "timekeeper" const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString() tk.freeze(timestamp) - -const { outputProcessing } = require("../../../utilities/rowProcessor") -const setup = require("./utilities") +import { outputProcessing } from "../../../utilities/rowProcessor" +import * as setup from "./utilities" const { basicRow } = setup.structures -const { context, tenancy } = require("@budibase/backend-core") -const { - quotas, -} = require("@budibase/pro") -const { +import { context, tenancy } from "@budibase/backend-core" +import { quotas } from "@budibase/pro" +import { QuotaUsageType, StaticQuotaName, MonthlyQuotaName, -} = require("@budibase/types") -const { structures } = require("@budibase/backend-core/tests"); + Row, + Table, + FieldType, +} from "@budibase/types" +import { structures } from "@budibase/backend-core/tests" describe("/rows", () => { let request = setup.getRequest() let config = setup.getConfig() - let table - let row + let table: Table + let row: Row afterAll(setup.afterAll) @@ -29,12 +29,12 @@ describe("/rows", () => { await config.init() }) - beforeEach(async()=>{ + beforeEach(async () => { table = await config.createTable() - row = basicRow(table._id) + row = basicRow(table._id!) }) - const loadRow = async (id, tbl_Id, status = 200) => + const loadRow = async (id: string, tbl_Id: string, status = 200) => await request .get(`/api/${tbl_Id}/rows/${id}`) .set(config.defaultHeaders()) @@ -42,21 +42,28 @@ describe("/rows", () => { .expect(status) const getRowUsage = async () => { - const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS)) + const { total } = await config.doInContext(null, () => + quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS) + ) return total } const getQueryUsage = async () => { - const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES)) + const { total } = await config.doInContext(null, () => + quotas.getCurrentUsageValues( + QuotaUsageType.MONTHLY, + MonthlyQuotaName.QUERIES + ) + ) return total } - const assertRowUsage = async expected => { + const assertRowUsage = async (expected: number) => { const usage = await getRowUsage() expect(usage).toBe(expected) } - const assertQueryUsage = async expected => { + const assertQueryUsage = async (expected: number) => { const usage = await getQueryUsage() expect(usage).toBe(expected) } @@ -70,9 +77,11 @@ describe("/rows", () => { .post(`/api/${row.tableId}/rows`) .send(row) .set(config.defaultHeaders()) - .expect('Content-Type', /json/) + .expect("Content-Type", /json/) .expect(200) - expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`) + expect((res as any).res.statusMessage).toEqual( + `${table.name} saved successfully` + ) expect(res.body.name).toEqual("Test Contact") expect(res.body._rev).toBeDefined() await assertRowUsage(rowUsage + 1) @@ -86,12 +95,11 @@ describe("/rows", () => { const newTable = await config.createTable({ name: "TestTableAuto", type: "table", - key: "name", schema: { ...table.schema, "Row ID": { name: "Row ID", - type: "number", + type: FieldType.NUMBER, subtype: "autoID", icon: "ri-magic-line", autocolumn: true, @@ -104,28 +112,30 @@ describe("/rows", () => { }, }, }, - } + }, }) - const ids = [1,2,3] + const ids = [1, 2, 3] // Performing several create row requests should increment the autoID fields accordingly - const createRow = async (id) => { + const createRow = async (id: number) => { const res = await request .post(`/api/${newTable._id}/rows`) .send({ - name: "row_" + id + name: "row_" + id, }) .set(config.defaultHeaders()) - .expect('Content-Type', /json/) + .expect("Content-Type", /json/) .expect(200) - expect(res.res.statusMessage).toEqual(`${newTable.name} saved successfully`) + expect((res as any).res.statusMessage).toEqual( + `${newTable.name} saved successfully` + ) expect(res.body.name).toEqual("row_" + id) expect(res.body._rev).toBeDefined() expect(res.body["Row ID"]).toEqual(id) } - for (let i=0; i { .expect("Content-Type", /json/) .expect(200) - expect(res.res.statusMessage).toEqual( + expect((res as any).res.statusMessage).toEqual( `${table.name} updated successfully.` ) expect(res.body.name).toEqual("Updated Name") @@ -196,8 +206,8 @@ describe("/rows", () => { .expect(200) expect(res.body.length).toBe(2) - expect(res.body.find(r => r.name === newRow.name)).toBeDefined() - expect(res.body.find(r => r.name === row.name)).toBeDefined() + expect(res.body.find((r: Row) => r.name === newRow.name)).toBeDefined() + expect(res.body.find((r: Row) => r.name === row.name)).toBeDefined() await assertQueryUsage(queryUsage + 1) }) @@ -215,92 +225,91 @@ describe("/rows", () => { it("row values are coerced", async () => { const str = { - type: "string", + type: FieldType.STRING, + name: "str", constraints: { type: "string", presence: false }, } const attachment = { - type: "attachment", + type: FieldType.ATTACHMENT, + name: "attachment", constraints: { type: "array", presence: false }, } const bool = { - type: "boolean", + type: FieldType.BOOLEAN, + name: "boolean", constraints: { type: "boolean", presence: false }, } const number = { - type: "number", + type: FieldType.NUMBER, + name: "str", constraints: { type: "number", presence: false }, } const datetime = { - type: "datetime", + type: FieldType.DATETIME, + name: "datetime", constraints: { type: "string", presence: false, datetime: { earliest: "", latest: "" }, - } + }, } const arrayField = { - type: "array", + type: FieldType.ARRAY, constraints: { type: "array", presence: false, - inclusion: [ - "One", - "Two", - "Three", - ] + inclusion: ["One", "Two", "Three"], }, name: "Sample Tags", - sortable: false + sortable: false, } const optsField = { - fieldName: "Sample Opts", - name: "Sample Opts", - type: "options", - constraints: { - type: "string", - presence: false, - inclusion: [ "Alpha", "Beta", "Gamma" ] + fieldName: "Sample Opts", + name: "Sample Opts", + type: FieldType.OPTIONS, + constraints: { + type: "string", + presence: false, + inclusion: ["Alpha", "Beta", "Gamma"], + }, }, - }, - - table = await config.createTable({ - name: "TestTable2", - type: "table", - key: "name", - schema: { - name: str, - stringUndefined: str, - stringNull: str, - stringString: str, - numberEmptyString: number, - numberNull: number, - numberUndefined: number, - numberString: number, - numberNumber: number, - datetimeEmptyString: datetime, - datetimeNull: datetime, - datetimeUndefined: datetime, - datetimeString: datetime, - datetimeDate: datetime, - boolNull: bool, - boolEmpty: bool, - boolUndefined: bool, - boolString: bool, - boolBool: bool, - attachmentNull: attachment, - attachmentUndefined: attachment, - attachmentEmpty: attachment, - attachmentEmptyArrayStr: attachment, - arrayFieldEmptyArrayStr: arrayField, - arrayFieldArrayStrKnown: arrayField, - arrayFieldNull: arrayField, - arrayFieldUndefined: arrayField, - optsFieldEmptyStr: optsField, - optsFieldUndefined: optsField, - optsFieldNull: optsField, - optsFieldStrKnown: optsField - }, - }) + table = await config.createTable({ + name: "TestTable2", + type: "table", + schema: { + name: str, + stringUndefined: str, + stringNull: str, + stringString: str, + numberEmptyString: number, + numberNull: number, + numberUndefined: number, + numberString: number, + numberNumber: number, + datetimeEmptyString: datetime, + datetimeNull: datetime, + datetimeUndefined: datetime, + datetimeString: datetime, + datetimeDate: datetime, + boolNull: bool, + boolEmpty: bool, + boolUndefined: bool, + boolString: bool, + boolBool: bool, + attachmentNull: attachment, + attachmentUndefined: attachment, + attachmentEmpty: attachment, + attachmentEmptyArrayStr: attachment, + arrayFieldEmptyArrayStr: arrayField, + arrayFieldArrayStrKnown: arrayField, + arrayFieldNull: arrayField, + arrayFieldUndefined: arrayField, + optsFieldEmptyStr: optsField, + optsFieldUndefined: optsField, + optsFieldNull: optsField, + optsFieldStrKnown: optsField, + }, + }) row = { name: "Test Row", @@ -334,13 +343,13 @@ describe("/rows", () => { optsFieldEmptyStr: "", optsFieldUndefined: undefined, optsFieldNull: null, - optsFieldStrKnown: 'Alpha' + optsFieldStrKnown: "Alpha", } - const createdRow = await config.createRow(row); - const id = createdRow._id + const createdRow = await config.createRow(row) + const id = createdRow._id! - const saved = (await loadRow(id, table._id)).body + const saved = (await loadRow(id, table._id!)).body expect(saved.stringUndefined).toBe(undefined) expect(saved.stringNull).toBe("") @@ -365,15 +374,15 @@ describe("/rows", () => { expect(saved.attachmentNull).toEqual([]) expect(saved.attachmentUndefined).toBe(undefined) expect(saved.attachmentEmpty).toEqual([]) - expect(saved.attachmentEmptyArrayStr).toEqual([]) + expect(saved.attachmentEmptyArrayStr).toEqual([]) expect(saved.arrayFieldEmptyArrayStr).toEqual([]) expect(saved.arrayFieldNull).toEqual([]) expect(saved.arrayFieldUndefined).toEqual(undefined) expect(saved.optsFieldEmptyStr).toEqual(null) expect(saved.optsFieldUndefined).toEqual(undefined) expect(saved.optsFieldNull).toEqual(null) - expect(saved.arrayFieldArrayStrKnown).toEqual(['One']) - expect(saved.optsFieldStrKnown).toEqual('Alpha') + expect(saved.arrayFieldArrayStrKnown).toEqual(["One"]) + expect(saved.optsFieldStrKnown).toEqual("Alpha") }) }) @@ -396,13 +405,13 @@ describe("/rows", () => { .expect("Content-Type", /json/) .expect(200) - expect(res.res.statusMessage).toEqual( + expect((res as any).res.statusMessage).toEqual( `${table.name} updated successfully.` ) expect(res.body.name).toEqual("Updated Name") expect(res.body.description).toEqual(existing.description) - const savedRow = await loadRow(res.body._id, table._id) + const savedRow = await loadRow(res.body._id, table._id!) expect(savedRow.body.description).toEqual(existing.description) expect(savedRow.body.name).toEqual("Updated Name") @@ -504,7 +513,7 @@ describe("/rows", () => { .expect(200) expect(res.body.length).toEqual(2) - await loadRow(row1._id, table._id, 404) + await loadRow(row1._id!, table._id!, 404) await assertRowUsage(rowUsage - 2) await assertQueryUsage(queryUsage + 1) }) @@ -562,7 +571,7 @@ describe("/rows", () => { describe("fetchEnrichedRows", () => { it("should allow enriching some linked rows", async () => { const { table, firstRow, secondRow } = await tenancy.doInTenant( - setup.structures.TENANT_ID, + config.getTenantId(), async () => { const table = await config.createLinkedTable() const firstRow = await config.createRow({ @@ -624,7 +633,7 @@ describe("/rows", () => { await setup.switchToSelfHosted(async () => { context.doInAppContext(config.getAppId(), async () => { const enriched = await outputProcessing(table, [row]) - expect(enriched[0].attachment[0].url).toBe( + expect((enriched as Row[])[0].attachment[0].url).toBe( `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}` ) }) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 2078bf765a..f0e3678099 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -135,7 +135,10 @@ class TestConfiguration { } } - async doInContext(appId: string | null, task: any) { + async doInContext( + appId: string | null, + task: () => Promise + ): Promise { if (!appId) { appId = this.appId } From e78b71b9fca27ed3213857345d1ec421758e020e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 14 Jul 2023 17:07:24 +0200 Subject: [PATCH 04/30] Run CI checks on any PR --- .github/workflows/budibase_ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 659dbc4974..b644a6bdb9 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -12,9 +12,6 @@ on: - master - develop pull_request: - branches: - - master - - develop workflow_dispatch: env: From 29624eea1039a288fab391325c613783403463f9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 14 Jul 2023 17:08:49 +0200 Subject: [PATCH 05/30] Use nx deps to run client --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 83719d7589..3afe279e00 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "kill-all": "yarn run kill-builder && yarn run kill-server", "dev": "yarn run kill-all && lerna run --stream dev:builder --stream", "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", - "dev:server": "yarn run kill-server && yarn build --projects=@budibase/client && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", + "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", From d1b64bcd9cbf6faffdc6f4eb5b1866231721f704 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 14 Jul 2023 17:04:59 +0200 Subject: [PATCH 06/30] Moving files and functions --- .../server/src/api/controllers/datasource.ts | 3 +- .../api/controllers/row/ExternalRequest.ts | 3 +- .../src/api/controllers/row/external.ts | 151 +---------- .../server/src/api/controllers/row/index.ts | 10 +- .../src/api/controllers/row/internal.ts | 234 +---------------- .../server/src/api/controllers/row/utils.ts | 101 ++----- packages/server/src/api/controllers/user.ts | 4 +- .../server/src/api/controllers/view/index.ts | 3 +- packages/server/src/sdk/app/rows/index.ts | 4 + packages/server/src/sdk/app/rows/search.ts | 38 +++ .../src/sdk/app/rows/search/external.ts | 152 +++++++++++ .../src/sdk/app/rows/search/internal.ts | 246 ++++++++++++++++++ .../app/rows/search}/internalSearch.ts | 0 packages/server/src/sdk/app/rows/utils.ts | 47 ++++ 14 files changed, 529 insertions(+), 467 deletions(-) create mode 100644 packages/server/src/sdk/app/rows/search.ts create mode 100644 packages/server/src/sdk/app/rows/search/external.ts create mode 100644 packages/server/src/sdk/app/rows/search/internal.ts rename packages/server/src/{api/controllers/row => sdk/app/rows/search}/internalSearch.ts (100%) create mode 100644 packages/server/src/sdk/app/rows/utils.ts diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 069842a020..65902a542e 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -9,7 +9,6 @@ import { import { destroy as tableDestroy } from "./table/internal" import { BuildSchemaErrors, InvalidColumns } from "../../constants" import { getIntegration } from "../../integrations" -import { getDatasourceAndQuery } from "./row/utils" import { invalidateDynamicVariables } from "../../threads/utils" import { db as dbCore, context, events } from "@budibase/backend-core" import { @@ -442,7 +441,7 @@ export async function find(ctx: UserCtx) { export async function query(ctx: UserCtx) { const queryJson = ctx.request.body try { - ctx.body = await getDatasourceAndQuery(queryJson) + ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson) } catch (err: any) { ctx.throw(400, err) } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 0139147e35..9378eb534b 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -23,14 +23,13 @@ import { isRowId, isSQL, } from "../../../integrations/utils" -import { getDatasourceAndQuery } from "./utils" +import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" import { FieldTypes } from "../../../constants" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import { processDates, processFormulas } from "../../../utilities/rowProcessor" import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" -import { isEditableColumn } from "../../../sdk/app/tables/validation" export interface ManyRelationship { tableId?: string diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 44396df67b..5a9647c63d 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -1,30 +1,21 @@ -import { - FieldTypes, - NoEmptyFilterStrings, - SortDirection, -} from "../../../constants" +import { FieldTypes, NoEmptyFilterStrings } from "../../../constants" import { breakExternalTableId, breakRowIdField, } from "../../../integrations/utils" import { ExternalRequest, RunConfig } from "./ExternalRequest" -import * as exporters from "../view/exporters" -import { apiFileReturn } from "../../../utilities/fileSystem" import { + Ctx, Datasource, IncludeRelationship, Operation, - PaginationJson, Row, - SortJson, Table, UserCtx, } from "@budibase/types" import sdk from "../../../sdk" import * as utils from "./utils" -const { cleanExportRows } = require("./utils") - async function getRow( tableId: string, rowId: string, @@ -114,21 +105,6 @@ export async function save(ctx: UserCtx) { } } -export async function fetchView(ctx: UserCtx) { - // there are no views in external datasources, shouldn't ever be called - // for now just fetch - const split = ctx.params.viewName.split("all_") - ctx.params.tableId = split[1] ? split[1] : split[0] - return fetch(ctx) -} - -export async function fetch(ctx: UserCtx) { - const tableId = ctx.params.tableId - return handleRequest(Operation.READ, tableId, { - includeSqlRelationships: IncludeRelationship.INCLUDE, - }) -} - export async function find(ctx: UserCtx) { const id = ctx.params.rowId const tableId = ctx.params.tableId @@ -161,129 +137,6 @@ export async function bulkDestroy(ctx: UserCtx) { return { response: { ok: true }, rows: responses.map(resp => resp.row) } } -export async function search(ctx: UserCtx) { - const tableId = ctx.params.tableId - const { paginate, query, ...params } = ctx.request.body - let { bookmark, limit } = params - if (!bookmark && paginate) { - bookmark = 1 - } - let paginateObj = {} - - if (paginate) { - paginateObj = { - // add one so we can track if there is another page - limit: limit, - page: bookmark, - } - } else if (params && limit) { - paginateObj = { - limit: limit, - } - } - let sort: SortJson | undefined - if (params.sort) { - const direction = - params.sortOrder === "descending" - ? SortDirection.DESCENDING - : SortDirection.ASCENDING - sort = { - [params.sort]: { direction }, - } - } - try { - const rows = (await handleRequest(Operation.READ, tableId, { - filters: query, - sort, - paginate: paginateObj as PaginationJson, - includeSqlRelationships: IncludeRelationship.INCLUDE, - })) as Row[] - let hasNextPage = false - if (paginate && rows.length === limit) { - const nextRows = (await handleRequest(Operation.READ, tableId, { - filters: query, - sort, - paginate: { - limit: 1, - page: bookmark * limit + 1, - }, - includeSqlRelationships: IncludeRelationship.INCLUDE, - })) as Row[] - hasNextPage = nextRows.length > 0 - } - // need wrapper object for bookmarks etc when paginating - return { rows, hasNextPage, bookmark: bookmark + 1 } - } catch (err: any) { - if (err.message && err.message.includes("does not exist")) { - throw new Error( - `Table updated externally, please re-fetch - ${err.message}` - ) - } else { - throw err - } - } -} - -export async function exportRows(ctx: UserCtx) { - const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId) - const format = ctx.query.format - const { columns } = ctx.request.body - const datasource = await sdk.datasources.get(datasourceId!) - if (!datasource || !datasource.entities) { - ctx.throw(400, "Datasource has not been configured for plus API.") - } - - if (ctx.request.body.rows) { - ctx.request.body = { - query: { - oneOf: { - _id: ctx.request.body.rows.map((row: string) => { - const ids = JSON.parse( - decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") - ) - if (ids.length > 1) { - ctx.throw(400, "Export data does not support composite keys.") - } - return ids[0] - }), - }, - }, - } - } - - let result = await search(ctx) - let rows: Row[] = [] - - // Filter data to only specified columns if required - - if (columns && columns.length) { - for (let i = 0; i < result.rows.length; i++) { - rows[i] = {} - for (let column of columns) { - rows[i][column] = result.rows[i][column] - } - } - } else { - rows = result.rows - } - - if (!tableName) { - ctx.throw(400, "Could not find table name.") - } - let schema = datasource.entities[tableName].schema - let exportRows = cleanExportRows(rows, schema, format, columns) - - let headers = Object.keys(schema) - - // @ts-ignore - const exporter = exporters[format] - const filename = `export.${format}` - - // send down the file - ctx.attachment(filename) - return apiFileReturn(exporter(headers, exportRows)) -} - export async function fetchEnrichedRow(ctx: UserCtx) { const id = ctx.params.rowId const tableId = ctx.params.tableId diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 91270429a4..cff594d329 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -5,6 +5,7 @@ import { isExternalTable } from "../../../integrations/utils" import { Ctx } from "@budibase/types" import * as utils from "./utils" import { gridSocket } from "../../../websockets" +import sdk from "../../../sdk" function pickApi(tableId: any) { if (isExternalTable(tableId)) { @@ -64,14 +65,14 @@ export const save = async (ctx: any) => { } export async function fetchView(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), { + ctx.body = await quotas.addQuery(() => sdk.rows.fetchView(tableId, ctx), { datasourceId: tableId, }) } export async function fetch(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), { + ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId, ctx), { datasourceId: tableId, }) } @@ -120,7 +121,8 @@ export async function destroy(ctx: any) { export async function search(ctx: any) { const tableId = utils.getTableId(ctx) ctx.status = 200 - ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), { + + ctx.body = await quotas.addQuery(() => sdk.rows.search(tableId, ctx), { datasourceId: tableId, }) } @@ -150,7 +152,7 @@ export async function fetchEnrichedRow(ctx: any) { export const exportRows = async (ctx: any) => { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), { + ctx.body = await quotas.addQuery(() => sdk.rows.exportRows(tableId, ctx), { datasourceId: tableId, }) } diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 87d8cd7e9a..9808e93dfb 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -1,7 +1,6 @@ import * as linkRows from "../../../db/linkedRows" import { generateRowID, - getRowParams, getTableIDFromRowID, DocumentType, InternalTables, @@ -14,8 +13,6 @@ import { } from "../../../utilities/rowProcessor" import { FieldTypes } from "../../../constants" import * as utils from "./utils" -import { fullSearch, paginatedSearch } from "./internalSearch" -import { getGlobalUsersFromMetadata } from "../../../utilities/global" import * as inMemoryViews from "../../../db/inMemoryView" import env from "../../../environment" import { @@ -27,65 +24,7 @@ import { import { cloneDeep } from "lodash/fp" import { context, db as dbCore } from "@budibase/backend-core" import { finaliseRow, updateRelatedFormula } from "./staticFormula" -import { csv, json, jsonWithSchema, Format } from "../view/exporters" -import { apiFileReturn } from "../../../utilities/fileSystem" -import { - UserCtx, - Database, - LinkDocumentValue, - Row, - Table, -} from "@budibase/types" - -import { cleanExportRows } from "./utils" - -const CALCULATION_TYPES = { - SUM: "sum", - COUNT: "count", - STATS: "stats", -} - -async function getView(db: Database, viewName: string) { - let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc - let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc - let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView - let viewInfo, - migrate = false - try { - viewInfo = await mainGetter(db, viewName) - } catch (err: any) { - // check if it can be retrieved from design doc (needs migrated) - if (err.status !== 404) { - viewInfo = null - } else { - viewInfo = await secondaryGetter(db, viewName) - migrate = !!viewInfo - } - } - if (migrate) { - await migration(db, viewName) - } - if (!viewInfo) { - throw "View does not exist." - } - return viewInfo -} - -async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) { - let rows - if (tableId === InternalTables.USER_METADATA) { - await userController.fetchMetadata(ctx) - rows = ctx.body - } else { - const response = await db.allDocs( - getRowParams(tableId, null, { - include_docs: true, - }) - ) - rows = response.rows.map(row => row.doc) - } - return rows as Row[] -} +import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types" export async function patch(ctx: UserCtx) { const db = context.getAppDB() @@ -195,82 +134,6 @@ export async function save(ctx: UserCtx) { }) } -export async function fetchView(ctx: UserCtx) { - const viewName = decodeURIComponent(ctx.params.viewName) - - // if this is a table view being looked for just transfer to that - if (viewName.startsWith(DocumentType.TABLE)) { - ctx.params.tableId = viewName - return fetch(ctx) - } - - const db = context.getAppDB() - const { calculation, group, field } = ctx.query - const viewInfo = await getView(db, viewName) - let response - if (env.SELF_HOSTED) { - response = await db.query(`database/${viewName}`, { - include_docs: !calculation, - group: !!group, - }) - } else { - const tableId = viewInfo.meta.tableId - const data = await getRawTableData(ctx, db, tableId) - response = await inMemoryViews.runView( - viewInfo, - calculation as string, - !!group, - data - ) - } - - let rows - if (!calculation) { - response.rows = response.rows.map(row => row.doc) - let table - try { - table = await db.get(viewInfo.meta.tableId) - } catch (err) { - /* istanbul ignore next */ - table = { - schema: {}, - } - } - rows = await outputProcessing(table, response.rows) - } - - if (calculation === CALCULATION_TYPES.STATS) { - response.rows = response.rows.map(row => ({ - group: row.key, - field, - ...row.value, - avg: row.value.sum / row.value.count, - })) - rows = response.rows - } - - if ( - calculation === CALCULATION_TYPES.COUNT || - calculation === CALCULATION_TYPES.SUM - ) { - rows = response.rows.map(row => ({ - group: row.key, - field, - value: row.value, - })) - } - return rows -} - -export async function fetch(ctx: UserCtx) { - const db = context.getAppDB() - - const tableId = ctx.params.tableId - let table = await db.get(tableId) - let rows = await getRawTableData(ctx, db, tableId) - return outputProcessing(table, rows) -} - export async function find(ctx: UserCtx) { const db = dbCore.getDB(ctx.appId) const table = await db.get(ctx.params.tableId) @@ -354,101 +217,6 @@ export async function bulkDestroy(ctx: UserCtx) { return { response: { ok: true }, rows: processedRows } } -export async function search(ctx: UserCtx) { - // Fetch the whole table when running in cypress, as search doesn't work - if (!env.COUCH_DB_URL && env.isCypress()) { - return { rows: await fetch(ctx) } - } - - const { tableId } = ctx.params - const db = context.getAppDB() - const { paginate, query, ...params } = ctx.request.body - params.version = ctx.version - params.tableId = tableId - - let table - if (params.sort && !params.sortType) { - table = await db.get(tableId) - const schema = table.schema - const sortField = schema[params.sort] - params.sortType = sortField.type == "number" ? "number" : "string" - } - - let response - if (paginate) { - response = await paginatedSearch(query, params) - } else { - response = await fullSearch(query, params) - } - - // Enrich search results with relationships - if (response.rows && response.rows.length) { - // enrich with global users if from users table - if (tableId === InternalTables.USER_METADATA) { - response.rows = await getGlobalUsersFromMetadata(response.rows) - } - table = table || (await db.get(tableId)) - response.rows = await outputProcessing(table, response.rows) - } - - return response -} - -export async function exportRows(ctx: UserCtx) { - const db = context.getAppDB() - const table = await db.get(ctx.params.tableId) - const rowIds = ctx.request.body.rows - let format = ctx.query.format - if (typeof format !== "string") { - ctx.throw(400, "Format parameter is not valid") - } - const { columns, query } = ctx.request.body - - let result - if (rowIds) { - let response = ( - await db.allDocs({ - include_docs: true, - keys: rowIds, - }) - ).rows.map(row => row.doc) - - result = await outputProcessing(table, response) - } else if (query) { - let searchResponse = await search(ctx) - result = searchResponse.rows - } - - let rows: Row[] = [] - let schema = table.schema - - // Filter data to only specified columns if required - if (columns && columns.length) { - for (let i = 0; i < result.length; i++) { - rows[i] = {} - for (let column of columns) { - rows[i][column] = result[i][column] - } - } - } else { - rows = result - } - - let exportRows = cleanExportRows(rows, schema, format, columns) - if (format === Format.CSV) { - ctx.attachment("export.csv") - return apiFileReturn(csv(Object.keys(rows[0]), exportRows)) - } else if (format === Format.JSON) { - ctx.attachment("export.json") - return apiFileReturn(json(exportRows)) - } else if (format === Format.JSON_WITH_SCHEMA) { - ctx.attachment("export.json") - return apiFileReturn(jsonWithSchema(schema, exportRows)) - } else { - throw "Format not recognised" - } -} - export async function fetchEnrichedRow(ctx: UserCtx) { const db = context.getAppDB() const tableId = ctx.params.tableId diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index a213f14a08..6ff90f2b25 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -1,14 +1,19 @@ import { InternalTables } from "../../../db/utils" import * as userController from "../user" -import { FieldTypes } from "../../../constants" import { context } from "@budibase/backend-core" -import { makeExternalQuery } from "../../../integrations/base/query" -import { FieldType, Row, Table, UserCtx } from "@budibase/types" -import { Format } from "../view/exporters" +import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types" +import { FieldTypes } from "../../../constants" import sdk from "../../../sdk" -const validateJs = require("validate.js") -const { cloneDeep } = require("lodash/fp") +import validateJs from "validate.js" +import { cloneDeep } from "lodash/fp" + +function isForeignKey(key: string, table: Table) { + const relationships = Object.values(table.schema).filter( + column => column.type === FieldType.LINK + ) + return relationships.some(relationship => relationship.foreignKey === key) +} validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -20,19 +25,6 @@ validateJs.extend(validateJs.validators.datetime, { }, }) -function isForeignKey(key: string, table: Table) { - const relationships = Object.values(table.schema).filter( - column => column.type === FieldType.LINK - ) - return relationships.some(relationship => relationship.foreignKey === key) -} - -export async function getDatasourceAndQuery(json: any) { - const datasourceId = json.endpoint.datasourceId - const datasource = await sdk.datasources.get(datasourceId) - return makeExternalQuery(datasource, json) -} - export async function findRow(ctx: UserCtx, tableId: string, rowId: string) { const db = context.getAppDB() let row @@ -52,6 +44,18 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) { return row } +export function getTableId(ctx: Ctx) { + if (ctx.request.body && ctx.request.body.tableId) { + return ctx.request.body.tableId + } + if (ctx.params && ctx.params.tableId) { + return ctx.params.tableId + } + if (ctx.params && ctx.params.viewName) { + return ctx.params.viewName + } +} + export async function validate({ tableId, row, @@ -81,8 +85,8 @@ export async function validate({ continue } // special case for options, need to always allow unselected (empty) - if (type === FieldTypes.OPTIONS && constraints.inclusion) { - constraints.inclusion.push(null, "") + if (type === FieldTypes.OPTIONS && constraints?.inclusion) { + constraints.inclusion.push(null as any, "") } let res @@ -94,13 +98,13 @@ export async function validate({ } row[fieldName].map((val: any) => { if ( - !constraints.inclusion.includes(val) && - constraints.inclusion.length !== 0 + !constraints?.inclusion?.includes(val) && + constraints?.inclusion?.length !== 0 ) { errors[fieldName] = "Field not in list" } }) - } else if (constraints.presence && row[fieldName].length === 0) { + } else if (constraints?.presence && row[fieldName].length === 0) { // non required MultiSelect creates an empty array, which should not throw errors errors[fieldName] = [`${fieldName} is required`] } @@ -128,52 +132,3 @@ export async function validate({ } return { valid: Object.keys(errors).length === 0, errors } } - -export function cleanExportRows( - rows: any[], - schema: any, - format: string, - columns: string[] -) { - let cleanRows = [...rows] - - const relationships = Object.entries(schema) - .filter((entry: any[]) => entry[1].type === FieldTypes.LINK) - .map(entry => entry[0]) - - relationships.forEach(column => { - cleanRows.forEach(row => { - delete row[column] - }) - delete schema[column] - }) - - if (format === Format.CSV) { - // Intended to append empty values in export - const schemaKeys = Object.keys(schema) - for (let key of schemaKeys) { - if (columns?.length && columns.indexOf(key) > 0) { - continue - } - for (let row of cleanRows) { - if (row[key] == null) { - row[key] = undefined - } - } - } - } - - return cleanRows -} - -export function getTableId(ctx: any) { - if (ctx.request.body && ctx.request.body.tableId) { - return ctx.request.body.tableId - } - if (ctx.params && ctx.params.tableId) { - return ctx.params.tableId - } - if (ctx.params && ctx.params.viewName) { - return ctx.params.viewName - } -} diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index b66f11bc1c..d4b3fe7a02 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -3,10 +3,10 @@ import { InternalTables } from "../../db/utils" import { getGlobalUsers } from "../../utilities/global" import { getFullUser } from "../../utilities/users" import { context } from "@budibase/backend-core" -import { UserCtx } from "@budibase/types" +import { Ctx, UserCtx } from "@budibase/types" import sdk from "../../sdk" -export async function fetchMetadata(ctx: UserCtx) { +export async function fetchMetadata(ctx: Ctx) { const global = await getGlobalUsers() const metadata = await sdk.users.rawUserMetadata() const users = [] diff --git a/packages/server/src/api/controllers/view/index.ts b/packages/server/src/api/controllers/view/index.ts index b194f3cb5e..83fbe7a17b 100644 --- a/packages/server/src/api/controllers/view/index.ts +++ b/packages/server/src/api/controllers/view/index.ts @@ -15,7 +15,6 @@ import { TableSchema, View, } from "@budibase/types" -import { cleanExportRows } from "../row/utils" import { builderSocket } from "../../../websockets" const { cloneDeep, isEqual } = require("lodash") @@ -169,7 +168,7 @@ export async function exportView(ctx: Ctx) { schema = table.schema } - let exportRows = cleanExportRows(rows, schema, format, []) + let exportRows = sdk.rows.utils.cleanExportRows(rows, schema, format, []) if (format === Format.CSV) { ctx.attachment(`${viewName}.csv`) diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index 12a44ded67..1ba91b134f 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -1,7 +1,11 @@ import * as attachments from "./attachments" import * as rows from "./rows" +import * as search from "./search" +import * as utils from "./utils" export default { ...attachments, ...rows, + ...search, + utils: utils, } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts new file mode 100644 index 0000000000..c37494192b --- /dev/null +++ b/packages/server/src/sdk/app/rows/search.ts @@ -0,0 +1,38 @@ +import { Ctx, SearchFilters } from "@budibase/types" +import { isExternalTable } from "../../../integrations/utils" +import * as internal from "./search/internal" +import * as external from "./search/external" + +export interface SearchParams { + tableId: string + paginate: boolean + query?: SearchFilters + bookmark?: number + limit: number + sort?: string + sortOrder?: string + sortType?: string +} + +function pickApi(tableId: any) { + if (isExternalTable(tableId)) { + return external + } + return internal +} + +export async function search(tableId: string, ctx: Ctx) { + return pickApi(tableId).search(ctx) +} + +export async function exportRows(tableId: string, ctx: Ctx) { + return pickApi(tableId).exportRows(ctx) +} + +export async function fetch(tableId: string, ctx: Ctx) { + return pickApi(tableId).fetch(ctx) +} + +export async function fetchView(tableId: string, ctx: Ctx) { + return pickApi(tableId).fetchView(ctx) +} diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts new file mode 100644 index 0000000000..18e9d1a79b --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -0,0 +1,152 @@ +import { + SortJson, + SortDirection, + Operation, + PaginationJson, + IncludeRelationship, + Row, + Ctx, +} from "@budibase/types" +import sdk from "../../../../sdk" +import { handleRequest } from "../../../../api/controllers/row/external" +import { breakExternalTableId } from "../../../../integrations/utils" +import { cleanExportRows } from "../utils" +import { apiFileReturn } from "../../../../utilities/fileSystem" + +export async function search(ctx: Ctx) { + const tableId = ctx.params.tableId + const { paginate, query, ...params } = ctx.request.body + let { bookmark, limit } = params + if (!bookmark && paginate) { + bookmark = 1 + } + let paginateObj = {} + + if (paginate) { + paginateObj = { + // add one so we can track if there is another page + limit: limit, + page: bookmark, + } + } else if (params && limit) { + paginateObj = { + limit: limit, + } + } + let sort: SortJson | undefined + if (params.sort) { + const direction = + params.sortOrder === "descending" + ? SortDirection.DESCENDING + : SortDirection.ASCENDING + sort = { + [params.sort]: { direction }, + } + } + try { + const rows = (await handleRequest(Operation.READ, tableId, { + filters: query, + sort, + paginate: paginateObj as PaginationJson, + includeSqlRelationships: IncludeRelationship.INCLUDE, + })) as Row[] + let hasNextPage = false + if (paginate && rows.length === limit) { + const nextRows = (await handleRequest(Operation.READ, tableId, { + filters: query, + sort, + paginate: { + limit: 1, + page: bookmark * limit + 1, + }, + includeSqlRelationships: IncludeRelationship.INCLUDE, + })) as Row[] + hasNextPage = nextRows.length > 0 + } + // need wrapper object for bookmarks etc when paginating + return { rows, hasNextPage, bookmark: bookmark + 1 } + } catch (err: any) { + if (err.message && err.message.includes("does not exist")) { + throw new Error( + `Table updated externally, please re-fetch - ${err.message}` + ) + } else { + throw err + } + } +} + +export async function exportRows(ctx: Ctx) { + const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId) + const format = ctx.query.format as string + const { columns } = ctx.request.body + const datasource = await sdk.datasources.get(datasourceId!) + if (!datasource || !datasource.entities) { + ctx.throw(400, "Datasource has not been configured for plus API.") + } + + if (ctx.request.body.rows) { + ctx.request.body = { + query: { + oneOf: { + _id: ctx.request.body.rows.map((row: string) => { + const ids = JSON.parse( + decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") + ) + if (ids.length > 1) { + ctx.throw(400, "Export data does not support composite keys.") + } + return ids[0] + }), + }, + }, + } + } + + let result = await search(ctx) + let rows: Row[] = [] + + // Filter data to only specified columns if required + + if (columns && columns.length) { + for (let i = 0; i < result.rows.length; i++) { + rows[i] = {} + for (let column of columns) { + rows[i][column] = result.rows[i][column] + } + } + } else { + rows = result.rows + } + + if (!tableName) { + ctx.throw(400, "Could not find table name.") + } + let schema = datasource.entities[tableName].schema + let exportRows = cleanExportRows(rows, schema, format, columns) + + let headers = Object.keys(schema) + + // @ts-ignore + const exporter = exporters[format] + const filename = `export.${format}` + + // send down the file + ctx.attachment(filename) + return apiFileReturn(exporter(headers, exportRows)) +} + +export async function fetch(ctx: Ctx) { + const tableId = ctx.params.tableId + return handleRequest(Operation.READ, tableId, { + includeSqlRelationships: IncludeRelationship.INCLUDE, + }) +} + +export async function fetchView(ctx: Ctx) { + // there are no views in external datasources, shouldn't ever be called + // for now just fetch + const split = ctx.params.viewName.split("all_") + ctx.params.tableId = split[1] ? split[1] : split[0] + return fetch(ctx) +} diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts new file mode 100644 index 0000000000..f79bb691dd --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -0,0 +1,246 @@ +import { context } from "@budibase/backend-core" +import env from "../../../../environment" +import { fullSearch, paginatedSearch } from "./internalSearch" +import { + InternalTables, + getRowParams, + DocumentType, +} from "../../../../db/utils" +import { getGlobalUsersFromMetadata } from "../../../../utilities/global" +import { outputProcessing } from "../../../../utilities/rowProcessor" +import { Ctx, Database, Row, UserCtx } from "@budibase/types" +import { cleanExportRows } from "../utils" +import { + Format, + csv, + json, + jsonWithSchema, +} from "../../../../api/controllers/view/exporters" +import { apiFileReturn } from "../../../../utilities/fileSystem" +import * as userController from "../../../../api/controllers/user" +import * as inMemoryViews from "../../../../db/inMemoryView" +import { + migrateToInMemoryView, + migrateToDesignView, + getFromDesignDoc, + getFromMemoryDoc, +} from "../../../../api/controllers/view/utils" + +export async function search(ctx: Ctx) { + // Fetch the whole table when running in cypress, as search doesn't work + if (!env.COUCH_DB_URL && env.isCypress()) { + return { rows: await fetch(ctx) } + } + + const { tableId } = ctx.params + const db = context.getAppDB() + const { paginate, query, ...params } = ctx.request.body + params.version = ctx.version + params.tableId = tableId + + let table + if (params.sort && !params.sortType) { + table = await db.get(tableId) + const schema = table.schema + const sortField = schema[params.sort] + params.sortType = sortField.type == "number" ? "number" : "string" + } + + let response + if (paginate) { + response = await paginatedSearch(query, params) + } else { + response = await fullSearch(query, params) + } + + // Enrich search results with relationships + if (response.rows && response.rows.length) { + // enrich with global users if from users table + if (tableId === InternalTables.USER_METADATA) { + response.rows = await getGlobalUsersFromMetadata(response.rows) + } + table = table || (await db.get(tableId)) + response.rows = await outputProcessing(table, response.rows) + } + + return response +} + +export async function exportRows(ctx: Ctx) { + const db = context.getAppDB() + const table = await db.get(ctx.params.tableId) + const rowIds = ctx.request.body.rows + let format = ctx.query.format + if (typeof format !== "string") { + ctx.throw(400, "Format parameter is not valid") + } + const { columns, query } = ctx.request.body + + let result + if (rowIds) { + let response = ( + await db.allDocs({ + include_docs: true, + keys: rowIds, + }) + ).rows.map(row => row.doc) + + result = await outputProcessing(table, response) + } else if (query) { + let searchResponse = await search(ctx) + result = searchResponse.rows + } + + let rows: Row[] = [] + let schema = table.schema + + // Filter data to only specified columns if required + if (columns && columns.length) { + for (let i = 0; i < result.length; i++) { + rows[i] = {} + for (let column of columns) { + rows[i][column] = result[i][column] + } + } + } else { + rows = result + } + + let exportRows = cleanExportRows(rows, schema, format, columns) + if (format === Format.CSV) { + ctx.attachment("export.csv") + return apiFileReturn(csv(Object.keys(rows[0]), exportRows)) + } else if (format === Format.JSON) { + ctx.attachment("export.json") + return apiFileReturn(json(exportRows)) + } else if (format === Format.JSON_WITH_SCHEMA) { + ctx.attachment("export.json") + return apiFileReturn(jsonWithSchema(schema, exportRows)) + } else { + throw "Format not recognised" + } +} + +export async function fetch(ctx: Ctx) { + const db = context.getAppDB() + + const tableId = ctx.params.tableId + let table = await db.get(tableId) + let rows = await getRawTableData(ctx, db, tableId) + return outputProcessing(table, rows) +} + +async function getRawTableData(ctx: Ctx, db: Database, tableId: string) { + let rows + if (tableId === InternalTables.USER_METADATA) { + await userController.fetchMetadata(ctx) + rows = ctx.body + } else { + const response = await db.allDocs( + getRowParams(tableId, null, { + include_docs: true, + }) + ) + rows = response.rows.map(row => row.doc) + } + return rows as Row[] +} + +export async function fetchView(ctx: Ctx) { + const viewName = decodeURIComponent(ctx.params.viewName) + + // if this is a table view being looked for just transfer to that + if (viewName.startsWith(DocumentType.TABLE)) { + ctx.params.tableId = viewName + return fetch(ctx) + } + + const db = context.getAppDB() + const { calculation, group, field } = ctx.query + const viewInfo = await getView(db, viewName) + let response + if (env.SELF_HOSTED) { + response = await db.query(`database/${viewName}`, { + include_docs: !calculation, + group: !!group, + }) + } else { + const tableId = viewInfo.meta.tableId + const data = await getRawTableData(ctx, db, tableId) + response = await inMemoryViews.runView( + viewInfo, + calculation as string, + !!group, + data + ) + } + + let rows + if (!calculation) { + response.rows = response.rows.map(row => row.doc) + let table + try { + table = await db.get(viewInfo.meta.tableId) + } catch (err) { + /* istanbul ignore next */ + table = { + schema: {}, + } + } + rows = await outputProcessing(table, response.rows) + } + + if (calculation === CALCULATION_TYPES.STATS) { + response.rows = response.rows.map(row => ({ + group: row.key, + field, + ...row.value, + avg: row.value.sum / row.value.count, + })) + rows = response.rows + } + + if ( + calculation === CALCULATION_TYPES.COUNT || + calculation === CALCULATION_TYPES.SUM + ) { + rows = response.rows.map(row => ({ + group: row.key, + field, + value: row.value, + })) + } + return rows +} + +const CALCULATION_TYPES = { + SUM: "sum", + COUNT: "count", + STATS: "stats", +} + +async function getView(db: Database, viewName: string) { + let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc + let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc + let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView + let viewInfo, + migrate = false + try { + viewInfo = await mainGetter(db, viewName) + } catch (err: any) { + // check if it can be retrieved from design doc (needs migrated) + if (err.status !== 404) { + viewInfo = null + } else { + viewInfo = await secondaryGetter(db, viewName) + migrate = !!viewInfo + } + } + if (migrate) { + await migration(db, viewName) + } + if (!viewInfo) { + throw "View does not exist." + } + return viewInfo +} diff --git a/packages/server/src/api/controllers/row/internalSearch.ts b/packages/server/src/sdk/app/rows/search/internalSearch.ts similarity index 100% rename from packages/server/src/api/controllers/row/internalSearch.ts rename to packages/server/src/sdk/app/rows/search/internalSearch.ts diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts new file mode 100644 index 0000000000..1f5be1479b --- /dev/null +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -0,0 +1,47 @@ +import { FieldTypes } from "../../../constants" +import { makeExternalQuery } from "../../../integrations/base/query" +import { Format } from "../../../api/controllers/view/exporters" +import sdk from "../.." + +export async function getDatasourceAndQuery(json: any) { + const datasourceId = json.endpoint.datasourceId + const datasource = await sdk.datasources.get(datasourceId) + return makeExternalQuery(datasource, json) +} + +export function cleanExportRows( + rows: any[], + schema: any, + format: string, + columns: string[] +) { + let cleanRows = [...rows] + + const relationships = Object.entries(schema) + .filter((entry: any[]) => entry[1].type === FieldTypes.LINK) + .map(entry => entry[0]) + + relationships.forEach(column => { + cleanRows.forEach(row => { + delete row[column] + }) + delete schema[column] + }) + + if (format === Format.CSV) { + // Intended to append empty values in export + const schemaKeys = Object.keys(schema) + for (let key of schemaKeys) { + if (columns?.length && columns.indexOf(key) > 0) { + continue + } + for (let row of cleanRows) { + if (row[key] == null) { + row[key] = undefined + } + } + } + } + + return cleanRows +} From 9fbad3721839a45812cdf1f9560909c4501cc35a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 10:51:52 +0200 Subject: [PATCH 07/30] Fix tests --- .../src/api/controllers/row/external.ts | 1 - .../src/sdk/app/rows/search/external.ts | 31 ++++++++++++++++-- .../tests => sdk/tests/rows}/row.spec.ts | 32 ++++++++----------- 3 files changed, 42 insertions(+), 22 deletions(-) rename packages/server/src/{api/controllers/tests => sdk/tests/rows}/row.spec.ts (77%) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 5a9647c63d..36329b3469 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -5,7 +5,6 @@ import { } from "../../../integrations/utils" import { ExternalRequest, RunConfig } from "./ExternalRequest" import { - Ctx, Datasource, IncludeRelationship, Operation, diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 18e9d1a79b..c48f984a2d 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -7,11 +7,13 @@ import { Row, Ctx, } from "@budibase/types" +import * as exporters from "../../../../api/controllers/view/exporters" import sdk from "../../../../sdk" import { handleRequest } from "../../../../api/controllers/row/external" import { breakExternalTableId } from "../../../../integrations/utils" import { cleanExportRows } from "../utils" import { apiFileReturn } from "../../../../utilities/fileSystem" +import { utils } from "@budibase/shared-core" export async function search(ctx: Ctx) { const tableId = ctx.params.tableId @@ -85,6 +87,15 @@ export async function exportRows(ctx: Ctx) { ctx.throw(400, "Datasource has not been configured for plus API.") } + if (!exporters.isFormat(format)) { + ctx.throw( + 400, + `Format ${format} not valid. Valid values: ${Object.values( + exporters.Format + )}` + ) + } + if (ctx.request.body.rows) { ctx.request.body = { query: { @@ -127,13 +138,27 @@ export async function exportRows(ctx: Ctx) { let headers = Object.keys(schema) - // @ts-ignore - const exporter = exporters[format] + let content + switch (format) { + case exporters.Format.CSV: + content = exporters.csv(headers, exportRows) + break + case exporters.Format.JSON: + content = exporters.json(exportRows) + break + case exporters.Format.JSON_WITH_SCHEMA: + content = exporters.jsonWithSchema(schema, exportRows) + break + default: + utils.unreachable(format) + break + } + const filename = `export.${format}` // send down the file ctx.attachment(filename) - return apiFileReturn(exporter(headers, exportRows)) + return apiFileReturn(content) } export async function fetch(ctx: Ctx) { diff --git a/packages/server/src/api/controllers/tests/row.spec.ts b/packages/server/src/sdk/tests/rows/row.spec.ts similarity index 77% rename from packages/server/src/api/controllers/tests/row.spec.ts rename to packages/server/src/sdk/tests/rows/row.spec.ts index 19e86d08be..68140345b7 100644 --- a/packages/server/src/api/controllers/tests/row.spec.ts +++ b/packages/server/src/sdk/tests/rows/row.spec.ts @@ -1,14 +1,14 @@ -import { exportRows } from "../row/external" -import sdk from "../../../sdk" -import { ExternalRequest } from "../row/ExternalRequest" +import { exportRows } from "../../app/rows/search/external" +import sdk from "../.." +import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest" -// @ts-ignore -sdk.datasources = { - get: jest.fn(), -} +const mockDatasourcesGet = jest.fn() +sdk.datasources.get = mockDatasourcesGet -jest.mock("../row/ExternalRequest") -jest.mock("../view/exporters", () => ({ +jest.mock("../../../api/controllers/row/ExternalRequest") + +jest.mock("../../../api/controllers/view/exporters", () => ({ + ...jest.requireActual("../../../api/controllers/view/exporters"), csv: jest.fn(), Format: { CSV: "csv", @@ -31,14 +31,15 @@ function getUserCtx() { throw "Err" }), attachment: jest.fn(), - } + } as any } describe("external row controller", () => { describe("exportRows", () => { beforeAll(() => { - //@ts-ignore - jest.spyOn(ExternalRequest.prototype, "run").mockImplementation(() => []) + jest + .spyOn(ExternalRequest.prototype, "run") + .mockImplementation(() => Promise.resolve([])) }) afterEach(() => { @@ -48,7 +49,6 @@ describe("external row controller", () => { it("should throw a 400 if no datasource entities are present", async () => { let userCtx = getUserCtx() try { - //@ts-ignore await exportRows(userCtx) } catch (e) { expect(userCtx.throw).toHaveBeenCalledWith( @@ -59,8 +59,7 @@ describe("external row controller", () => { }) it("should handle single quotes from a row ID", async () => { - //@ts-ignore - sdk.datasources.get.mockImplementation(() => ({ + mockDatasourcesGet.mockImplementation(async () => ({ entities: { tablename: { schema: {}, @@ -72,7 +71,6 @@ describe("external row controller", () => { rows: ["['d001']"], } - //@ts-ignore await exportRows(userCtx) expect(userCtx.request.body).toEqual({ @@ -90,7 +88,6 @@ describe("external row controller", () => { rows: ["[123]", "['d001'%2C'10111']"], } try { - //@ts-ignore await exportRows(userCtx) } catch (e) { expect(userCtx.throw).toHaveBeenCalledWith( @@ -107,7 +104,6 @@ describe("external row controller", () => { rows: ["[123]"], } try { - //@ts-ignore await exportRows(userCtx) } catch (e) { expect(userCtx.throw).toHaveBeenCalledWith( From e336ba4b5e171961ab0fe593ac18c58c269e0e29 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 11:13:21 +0200 Subject: [PATCH 08/30] Type and fix tests --- .../api/routes/tests/internalSearch.spec.js | 197 -------------- .../api/routes/tests/internalSearch.spec.ts | 248 ++++++++++++++++++ packages/types/src/sdk/search.ts | 2 +- 3 files changed, 249 insertions(+), 198 deletions(-) delete mode 100644 packages/server/src/api/routes/tests/internalSearch.spec.js create mode 100644 packages/server/src/api/routes/tests/internalSearch.spec.ts diff --git a/packages/server/src/api/routes/tests/internalSearch.spec.js b/packages/server/src/api/routes/tests/internalSearch.spec.js deleted file mode 100644 index 8d57d2ee1c..0000000000 --- a/packages/server/src/api/routes/tests/internalSearch.spec.js +++ /dev/null @@ -1,197 +0,0 @@ -const fetch = require("node-fetch") -fetch.mockSearch() -const search = require("../../controllers/row/internalSearch") -// this will be mocked out for _search endpoint -const PARAMS = { - tableId: "ta_12345679abcdef", - version: "1", - bookmark: null, - sort: null, - sortOrder: "ascending", - sortType: "string", -} - -function checkLucene(resp, expected, params = PARAMS) { - const query = resp.rows[0].query - const json = JSON.parse(query) - if (PARAMS.sort) { - expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) - } - if (PARAMS.bookmark) { - expect(json.bookmark).toBe(PARAMS.bookmark) - } - expect(json.include_docs).toBe(true) - expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`) - expect(json.limit).toBe(params.limit || 50) -} - -describe("internal search", () => { - it("default query", async () => { - const response = await search.paginatedSearch({ - }, PARAMS) - checkLucene(response, `*:*`) - }) - - it("test equal query", async () => { - const response = await search.paginatedSearch({ - equal: { - "column": "1", - } - }, PARAMS) - checkLucene(response, `*:* AND column:"1"`) - }) - - it("test notEqual query", async () => { - const response = await search.paginatedSearch({ - notEqual: { - "column": "1", - } - }, PARAMS) - checkLucene(response, `*:* AND !column:"1"`) - }) - - it("test OR query", async () => { - const response = await search.paginatedSearch({ - allOr: true, - equal: { - "column": "2", - }, - notEqual: { - "column": "1", - } - }, PARAMS) - checkLucene(response, `(column:"2" OR !column:"1")`) - }) - - it("test AND query", async () => { - const response = await search.paginatedSearch({ - equal: { - "column": "2", - }, - notEqual: { - "column": "1", - } - }, PARAMS) - checkLucene(response, `(*:* AND column:"2" AND !column:"1")`) - }) - - it("test pagination query", async () => { - const updatedParams = { - ...PARAMS, - limit: 100, - bookmark: "awd", - sort: "column", - } - const response = await search.paginatedSearch({ - string: { - "column": "2", - }, - }, updatedParams) - checkLucene(response, `*:* AND column:2*`, updatedParams) - }) - - it("test range query", async () => { - const response = await search.paginatedSearch({ - range: { - "column": { low: 1, high: 2 }, - }, - }, PARAMS) - checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS) - }) - - it("test empty query", async () => { - const response = await search.paginatedSearch({ - empty: { - "column": "", - }, - }, PARAMS) - checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS) - }) - - it("test notEmpty query", async () => { - const response = await search.paginatedSearch({ - notEmpty: { - "column": "", - }, - }, PARAMS) - checkLucene(response, `*:* AND column:["" TO *]`, PARAMS) - }) - - it("test oneOf query", async () => { - const response = await search.paginatedSearch({ - oneOf: { - "column": ["a", "b"], - }, - }, PARAMS) - checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS) - }) - - it("test contains query", async () => { - const response = await search.paginatedSearch({ - contains: { - "column": "a", - "colArr": [1, 2, 3], - }, - }, PARAMS) - checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS) - }) - - it("test multiple of same column", async () => { - const response = await search.paginatedSearch({ - allOr: true, - equal: { - "1:column": "a", - "2:column": "b", - "3:column": "c", - }, - }, PARAMS) - checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS) - }) - - it("check a weird case for lucene building", async () => { - const response = await search.paginatedSearch({ - equal: { - "1:1:column": "a", - }, - }, PARAMS) - checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS) - }) - - it("test containsAny query", async () => { - const response = await search.paginatedSearch({ - containsAny: { - "column": ["a", "b", "c"] - }, - }, PARAMS) - checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS) - }) - - it("test notContains query", async () => { - const response = await search.paginatedSearch({ - notContains: { - "column": ["a", "b", "c"] - }, - }, PARAMS) - checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS) - }) - - it("test equal without version query", async () => { - PARAMS.version = null - const response = await search.paginatedSearch({ - equal: { - "column": "1", - } - }, PARAMS) - - const query = response.rows[0].query - const json = JSON.parse(query) - if (PARAMS.sort) { - expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) - } - if (PARAMS.bookmark) { - expect(json.bookmark).toBe(PARAMS.bookmark) - } - expect(json.include_docs).toBe(true) - expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`) - }) -}) \ No newline at end of file diff --git a/packages/server/src/api/routes/tests/internalSearch.spec.ts b/packages/server/src/api/routes/tests/internalSearch.spec.ts new file mode 100644 index 0000000000..fdae513cb6 --- /dev/null +++ b/packages/server/src/api/routes/tests/internalSearch.spec.ts @@ -0,0 +1,248 @@ +const nodeFetch = require("node-fetch") +nodeFetch.mockSearch() +import { SearchParams } from "@budibase/backend-core" +import * as search from "../../../sdk/app/rows/search/internalSearch" +import { Row } from "@budibase/types" + +// this will be mocked out for _search endpoint +const PARAMS: SearchParams = { + tableId: "ta_12345679abcdef", + version: "1", + bookmark: undefined, + sort: undefined, + sortOrder: "ascending", + sortType: "string", +} + +function checkLucene(resp: any, expected: any, params = PARAMS) { + const query = resp.rows[0].query + const json = JSON.parse(query) + if (PARAMS.sort) { + expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) + } + if (PARAMS.bookmark) { + expect(json.bookmark).toBe(PARAMS.bookmark) + } + expect(json.include_docs).toBe(true) + expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`) + expect(json.limit).toBe(params.limit || 50) +} + +describe("internal search", () => { + it("default query", async () => { + const response = await search.paginatedSearch({}, PARAMS) + checkLucene(response, `*:*`) + }) + + it("test equal query", async () => { + const response = await search.paginatedSearch( + { + equal: { + column: "1", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:"1"`) + }) + + it("test notEqual query", async () => { + const response = await search.paginatedSearch( + { + notEqual: { + column: "1", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND !column:"1"`) + }) + + it("test OR query", async () => { + const response = await search.paginatedSearch( + { + allOr: true, + equal: { + column: "2", + }, + notEqual: { + column: "1", + }, + }, + PARAMS + ) + checkLucene(response, `(column:"2" OR !column:"1")`) + }) + + it("test AND query", async () => { + const response = await search.paginatedSearch( + { + equal: { + column: "2", + }, + notEqual: { + column: "1", + }, + }, + PARAMS + ) + checkLucene(response, `(*:* AND column:"2" AND !column:"1")`) + }) + + it("test pagination query", async () => { + const updatedParams = { + ...PARAMS, + limit: 100, + bookmark: "awd", + sort: "column", + } + const response = await search.paginatedSearch( + { + string: { + column: "2", + }, + }, + updatedParams + ) + checkLucene(response, `*:* AND column:2*`, updatedParams) + }) + + it("test range query", async () => { + const response = await search.paginatedSearch( + { + range: { + column: { low: 1, high: 2 }, + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS) + }) + + it("test empty query", async () => { + const response = await search.paginatedSearch( + { + empty: { + column: "", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS) + }) + + it("test notEmpty query", async () => { + const response = await search.paginatedSearch( + { + notEmpty: { + column: "", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:["" TO *]`, PARAMS) + }) + + it("test oneOf query", async () => { + const response = await search.paginatedSearch( + { + oneOf: { + column: ["a", "b"], + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS) + }) + + it("test contains query", async () => { + const response = await search.paginatedSearch( + { + contains: { + column: "a", + colArr: [1, 2, 3], + }, + }, + PARAMS + ) + checkLucene( + response, + `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, + PARAMS + ) + }) + + it("test multiple of same column", async () => { + const response = await search.paginatedSearch( + { + allOr: true, + equal: { + "1:column": "a", + "2:column": "b", + "3:column": "c", + }, + }, + PARAMS + ) + checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS) + }) + + it("check a weird case for lucene building", async () => { + const response = await search.paginatedSearch( + { + equal: { + "1:1:column": "a", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS) + }) + + it("test containsAny query", async () => { + const response = await search.paginatedSearch( + { + containsAny: { + column: ["a", "b", "c"], + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS) + }) + + it("test notContains query", async () => { + const response = await search.paginatedSearch( + { + notContains: { + column: ["a", "b", "c"], + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS) + }) + + it("test equal without version query", async () => { + PARAMS.version = undefined + const response = await search.paginatedSearch( + { + equal: { + column: "1", + }, + }, + PARAMS + ) + + const query = response.rows[0].query + const json = JSON.parse(query) + if (PARAMS.sort) { + expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) + } + if (PARAMS.bookmark) { + expect(json.bookmark).toBe(PARAMS.bookmark) + } + expect(json.include_docs).toBe(true) + expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`) + }) +}) diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index d4c5135038..ae9aec66a2 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -32,7 +32,7 @@ export interface SearchFilters { [key: string]: any[] } contains?: { - [key: string]: any[] + [key: string]: any[] | any } notContains?: { [key: string]: any[] From 18d2802888c5b1465f2157acba9744a9609f4fad Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 11:14:34 +0200 Subject: [PATCH 09/30] Update nodejs version --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 094292d096..da92e03885 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -nodejs 14.20.1 +nodejs 14.21.3 python 3.10.0 \ No newline at end of file From 20635ae2ff9957d16fe42c28250052b80a0443e4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 12:18:57 +0200 Subject: [PATCH 10/30] Type and fix test --- .../tests/{queryRows.spec.js => queryRows.spec.ts} | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) rename packages/server/src/automations/tests/{queryRows.spec.js => queryRows.spec.ts} (93%) diff --git a/packages/server/src/automations/tests/queryRows.spec.js b/packages/server/src/automations/tests/queryRows.spec.ts similarity index 93% rename from packages/server/src/automations/tests/queryRows.spec.js rename to packages/server/src/automations/tests/queryRows.spec.ts index 1ce7460806..bee90df08e 100644 --- a/packages/server/src/automations/tests/queryRows.spec.js +++ b/packages/server/src/automations/tests/queryRows.spec.ts @@ -1,6 +1,6 @@ // lucene searching not supported in test due to use of PouchDB -let rows = [] -jest.mock("../../api/controllers/row/internalSearch", () => ({ +let rows: Row[] = [] +jest.mock("../../sdk/app/rows/search/internalSearch", () => ({ fullSearch: jest.fn(() => { return { rows, @@ -8,12 +8,13 @@ jest.mock("../../api/controllers/row/internalSearch", () => ({ }), paginatedSearch: jest.fn(), })) -const setup = require("./utilities") +import { Row, Table } from "@budibase/types" +import * as setup from "./utilities" const NAME = "Test" describe("Test a query step automation", () => { - let table + let table: Table let config = setup.getConfig() beforeAll(async () => { @@ -87,8 +88,8 @@ describe("Test a query step automation", () => { filters: {}, "filters-def": [ { - value: null - } + value: null, + }, ], sortColumn: "name", sortOrder: "ascending", From 940da62b980c1712142fe9ed91f95aa5fe08e5af Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 14 Jul 2023 18:14:53 +0200 Subject: [PATCH 11/30] Clean --- packages/server/src/api/controllers/row/internal.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 9808e93dfb..a69aabd3c2 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -2,7 +2,6 @@ import * as linkRows from "../../../db/linkedRows" import { generateRowID, getTableIDFromRowID, - DocumentType, InternalTables, } from "../../../db/utils" import * as userController from "../user" @@ -13,14 +12,6 @@ import { } from "../../../utilities/rowProcessor" import { FieldTypes } from "../../../constants" import * as utils from "./utils" -import * as inMemoryViews from "../../../db/inMemoryView" -import env from "../../../environment" -import { - migrateToInMemoryView, - migrateToDesignView, - getFromDesignDoc, - getFromMemoryDoc, -} from "../view/utils" import { cloneDeep } from "lodash/fp" import { context, db as dbCore } from "@budibase/backend-core" import { finaliseRow, updateRelatedFormula } from "./staticFormula" From 77b3547af74635d87b1d9d2943167fdd676ddb2a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 09:53:15 +0200 Subject: [PATCH 12/30] Remove usercontroller dependency from internal --- packages/server/src/api/controllers/user.ts | 16 +--------------- .../src/sdk/app/rows/search/internal.ts | 13 ++++++------- packages/server/src/sdk/users/utils.ts | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index d4b3fe7a02..82a776e9a6 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -7,21 +7,7 @@ import { Ctx, UserCtx } from "@budibase/types" import sdk from "../../sdk" export async function fetchMetadata(ctx: Ctx) { - const global = await getGlobalUsers() - const metadata = await sdk.users.rawUserMetadata() - const users = [] - for (let user of global) { - // find the metadata that matches up to the global ID - const info = metadata.find(meta => meta._id.includes(user._id)) - // remove these props, not for the correct DB - users.push({ - ...user, - ...info, - tableId: InternalTables.USER_METADATA, - // make sure the ID is always a local ID, not a global one - _id: generateUserMetadataID(user._id), - }) - } + const users = await sdk.users.fetchMetadata() ctx.body = users } diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index f79bb691dd..f4529888a3 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -8,7 +8,7 @@ import { } from "../../../../db/utils" import { getGlobalUsersFromMetadata } from "../../../../utilities/global" import { outputProcessing } from "../../../../utilities/rowProcessor" -import { Ctx, Database, Row, UserCtx } from "@budibase/types" +import { Ctx, Database, Row } from "@budibase/types" import { cleanExportRows } from "../utils" import { Format, @@ -17,7 +17,6 @@ import { jsonWithSchema, } from "../../../../api/controllers/view/exporters" import { apiFileReturn } from "../../../../utilities/fileSystem" -import * as userController from "../../../../api/controllers/user" import * as inMemoryViews from "../../../../db/inMemoryView" import { migrateToInMemoryView, @@ -25,6 +24,7 @@ import { getFromDesignDoc, getFromMemoryDoc, } from "../../../../api/controllers/view/utils" +import sdk from "src/sdk" export async function search(ctx: Ctx) { // Fetch the whole table when running in cypress, as search doesn't work @@ -126,15 +126,14 @@ export async function fetch(ctx: Ctx) { const tableId = ctx.params.tableId let table = await db.get(tableId) - let rows = await getRawTableData(ctx, db, tableId) + let rows = await getRawTableData(db, tableId) return outputProcessing(table, rows) } -async function getRawTableData(ctx: Ctx, db: Database, tableId: string) { +async function getRawTableData(db: Database, tableId: string) { let rows if (tableId === InternalTables.USER_METADATA) { - await userController.fetchMetadata(ctx) - rows = ctx.body + rows = await sdk.users.fetchMetadata() } else { const response = await db.allDocs( getRowParams(tableId, null, { @@ -166,7 +165,7 @@ export async function fetchView(ctx: Ctx) { }) } else { const tableId = viewInfo.meta.tableId - const data = await getRawTableData(ctx, db, tableId) + const data = await getRawTableData(db, tableId) response = await inMemoryViews.runView( viewInfo, calculation as string, diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index 619a7199cd..08084caad1 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -64,6 +64,25 @@ export async function rawUserMetadata(db?: Database) { ).rows.map(row => row.doc) } +export async function fetchMetadata() { + const global = await getGlobalUsers() + const metadata = await rawUserMetadata() + const users = [] + for (let user of global) { + // find the metadata that matches up to the global ID + const info = metadata.find(meta => meta._id.includes(user._id)) + // remove these props, not for the correct DB + users.push({ + ...user, + ...info, + tableId: InternalTables.USER_METADATA, + // make sure the ID is always a local ID, not a global one + _id: generateUserMetadataID(user._id), + }) + } + return users +} + export async function syncGlobalUsers() { // sync user metadata const dbs = [context.getDevAppDB(), context.getProdAppDB()] From 81c6eab05ef433f61e8e87e0acbf98a6e0f4d8ae Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 11:05:27 +0200 Subject: [PATCH 13/30] Fix references --- packages/server/src/sdk/app/rows/search/internal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index f4529888a3..6545761283 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -24,7 +24,7 @@ import { getFromDesignDoc, getFromMemoryDoc, } from "../../../../api/controllers/view/utils" -import sdk from "src/sdk" +import sdk from "../../../../sdk" export async function search(ctx: Ctx) { // Fetch the whole table when running in cypress, as search doesn't work From e7f1bcab9e063120795f6c7349358e09eb94309c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 14:16:12 +0200 Subject: [PATCH 14/30] Remove ctx from fetch --- packages/server/src/api/controllers/row/index.ts | 2 +- packages/server/src/sdk/app/rows/search.ts | 4 ++-- packages/server/src/sdk/app/rows/search/external.ts | 7 +++---- packages/server/src/sdk/app/rows/search/internal.ts | 13 +++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index cff594d329..a05c640c33 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -72,7 +72,7 @@ export async function fetchView(ctx: any) { export async function fetch(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId, ctx), { + ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId), { datasourceId: tableId, }) } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index c37494192b..56dea32025 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -29,8 +29,8 @@ export async function exportRows(tableId: string, ctx: Ctx) { return pickApi(tableId).exportRows(ctx) } -export async function fetch(tableId: string, ctx: Ctx) { - return pickApi(tableId).fetch(ctx) +export async function fetch(tableId: string) { + return pickApi(tableId).fetch(tableId) } export async function fetchView(tableId: string, ctx: Ctx) { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index c48f984a2d..abed06eb31 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -161,8 +161,7 @@ export async function exportRows(ctx: Ctx) { return apiFileReturn(content) } -export async function fetch(ctx: Ctx) { - const tableId = ctx.params.tableId +export async function fetch(tableId: string) { return handleRequest(Operation.READ, tableId, { includeSqlRelationships: IncludeRelationship.INCLUDE, }) @@ -172,6 +171,6 @@ export async function fetchView(ctx: Ctx) { // there are no views in external datasources, shouldn't ever be called // for now just fetch const split = ctx.params.viewName.split("all_") - ctx.params.tableId = split[1] ? split[1] : split[0] - return fetch(ctx) + const tableId = split[1] ? split[1] : split[0] + return fetch(tableId) } diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 6545761283..d3d15deb53 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -27,12 +27,13 @@ import { import sdk from "../../../../sdk" export async function search(ctx: Ctx) { + const { tableId } = ctx.params + // Fetch the whole table when running in cypress, as search doesn't work if (!env.COUCH_DB_URL && env.isCypress()) { - return { rows: await fetch(ctx) } + return { rows: await fetch(tableId) } } - const { tableId } = ctx.params const db = context.getAppDB() const { paginate, query, ...params } = ctx.request.body params.version = ctx.version @@ -121,13 +122,13 @@ export async function exportRows(ctx: Ctx) { } } -export async function fetch(ctx: Ctx) { +export async function fetch(tableId: string) { const db = context.getAppDB() - const tableId = ctx.params.tableId let table = await db.get(tableId) let rows = await getRawTableData(db, tableId) - return outputProcessing(table, rows) + const result = await outputProcessing(table, rows) + return result } async function getRawTableData(db: Database, tableId: string) { @@ -151,7 +152,7 @@ export async function fetchView(ctx: Ctx) { // if this is a table view being looked for just transfer to that if (viewName.startsWith(DocumentType.TABLE)) { ctx.params.tableId = viewName - return fetch(ctx) + return fetch(viewName) } const db = context.getAppDB() From 1bd8bdf84c285e72dd57733a71b1b99146db3f47 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 15:29:41 +0200 Subject: [PATCH 15/30] Clean ctx from fetchview --- .../server/src/api/controllers/row/index.ts | 18 +++++++++++++++--- .../server/src/api/controllers/view/index.ts | 5 ++++- packages/server/src/sdk/app/rows/search.ts | 14 ++++++++++++-- .../server/src/sdk/app/rows/search/external.ts | 4 ++-- .../server/src/sdk/app/rows/search/internal.ts | 10 +++++----- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index a05c640c33..ed441f7784 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -65,9 +65,21 @@ export const save = async (ctx: any) => { } export async function fetchView(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => sdk.rows.fetchView(tableId, ctx), { - datasourceId: tableId, - }) + const viewName = decodeURIComponent(ctx.params.viewName) + + const { calculation, group, field } = ctx.query + + ctx.body = await quotas.addQuery( + () => + sdk.rows.fetchView(tableId, viewName, { + calculation, + group, + field, + }), + { + datasourceId: tableId, + } + ) } export async function fetch(ctx: any) { diff --git a/packages/server/src/api/controllers/view/index.ts b/packages/server/src/api/controllers/view/index.ts index 83fbe7a17b..a087634292 100644 --- a/packages/server/src/api/controllers/view/index.ts +++ b/packages/server/src/api/controllers/view/index.ts @@ -162,7 +162,10 @@ export async function exportView(ctx: Ctx) { let rows = ctx.body as Row[] let schema: TableSchema = view && view.meta && view.meta.schema - const tableId = ctx.params.tableId || view.meta.tableId + const tableId = + ctx.params.tableId || + view?.meta?.tableId || + (viewName.startsWith(DocumentType.TABLE) && viewName) const table: Table = await sdk.tables.getTable(tableId) if (!schema) { schema = table.schema diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 56dea32025..f8f50df2e6 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -14,6 +14,12 @@ export interface SearchParams { sortType?: string } +export interface ViewParams { + calculation: string + group: string + field: string +} + function pickApi(tableId: any) { if (isExternalTable(tableId)) { return external @@ -33,6 +39,10 @@ export async function fetch(tableId: string) { return pickApi(tableId).fetch(tableId) } -export async function fetchView(tableId: string, ctx: Ctx) { - return pickApi(tableId).fetchView(ctx) +export async function fetchView( + tableId: string, + viewName: string, + params: ViewParams +) { + return pickApi(tableId).fetchView(viewName, params) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index abed06eb31..e6d910b1c9 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -167,10 +167,10 @@ export async function fetch(tableId: string) { }) } -export async function fetchView(ctx: Ctx) { +export async function fetchView(viewName: string) { // there are no views in external datasources, shouldn't ever be called // for now just fetch - const split = ctx.params.viewName.split("all_") + const split = viewName.split("all_") const tableId = split[1] ? split[1] : split[0] return fetch(tableId) } diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index d3d15deb53..7caa5083c9 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -146,17 +146,17 @@ async function getRawTableData(db: Database, tableId: string) { return rows as Row[] } -export async function fetchView(ctx: Ctx) { - const viewName = decodeURIComponent(ctx.params.viewName) - +export async function fetchView( + viewName: string, + options: { calculation: string; group: string; field: string } +) { // if this is a table view being looked for just transfer to that if (viewName.startsWith(DocumentType.TABLE)) { - ctx.params.tableId = viewName return fetch(viewName) } const db = context.getAppDB() - const { calculation, group, field } = ctx.query + const { calculation, group, field } = options const viewInfo = await getView(db, viewName) let response if (env.SELF_HOSTED) { From 1584ee4c9554b62aa39da747a779969bfa8f805c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 17 Jul 2023 14:45:13 +0100 Subject: [PATCH 16/30] Use shorthand typing --- packages/server/src/api/controllers/application.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 5e12dd4d76..666dacc28c 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -202,7 +202,7 @@ export async function fetch(ctx: UserCtx) { // Get all builder sessions in each app const sessions = await builderSocket?.getRoomSessions(appIds) if (sessions?.length) { - let appSessionMap: { [key: string]: SocketSession[] } = {} + let appSessionMap: Record = {} sessions.forEach(session => { const room = session.room if (!room) { From 1a8da1c0afba51d1dcfbda92da67d026d70cbae2 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 17 Jul 2023 14:56:09 +0100 Subject: [PATCH 17/30] Move app enrichment of user sessions to users SDK --- .../server/src/api/controllers/application.ts | 27 ++----------- packages/server/src/sdk/users/index.ts | 2 + packages/server/src/sdk/users/sessions.ts | 38 +++++++++++++++++++ 3 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 packages/server/src/sdk/users/sessions.ts diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 666dacc28c..2c53d3ad35 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -199,31 +199,10 @@ export async function fetch(ctx: UserCtx) { } } - // Get all builder sessions in each app - const sessions = await builderSocket?.getRoomSessions(appIds) - if (sessions?.length) { - let appSessionMap: Record = {} - sessions.forEach(session => { - const room = session.room - if (!room) { - return - } - if (!appSessionMap[room]) { - appSessionMap[room] = [] - } - appSessionMap[room].push(session) - }) - apps.forEach(app => { - const sessions = appSessionMap[app.appId] - if (sessions?.length) { - app.sessions = sessions - } else { - delete app.sessions - } - }) - } + // Enrich apps with all builder user sessions + const enrichedApps = await sdk.users.sessions.enrichApps(apps) - ctx.body = await checkAppMetadata(apps) + ctx.body = await checkAppMetadata(enrichedApps) } export async function fetchAppDefinition(ctx: UserCtx) { diff --git a/packages/server/src/sdk/users/index.ts b/packages/server/src/sdk/users/index.ts index c8431f7e61..c03eab7c2a 100644 --- a/packages/server/src/sdk/users/index.ts +++ b/packages/server/src/sdk/users/index.ts @@ -1,5 +1,7 @@ import * as utils from "./utils" +import * as sessions from "./sessions" export default { ...utils, + sessions, } diff --git a/packages/server/src/sdk/users/sessions.ts b/packages/server/src/sdk/users/sessions.ts new file mode 100644 index 0000000000..c413242277 --- /dev/null +++ b/packages/server/src/sdk/users/sessions.ts @@ -0,0 +1,38 @@ +import { builderSocket } from "../../websockets" +import { App, SocketSession } from "@budibase/types" + +export const enrichApps = async (apps: App[]) => { + // Sessions can only exist for dev app IDs + const devAppIds = apps + .filter((app: any) => app.status === "development") + .map((app: any) => app.appId) + + // Get all sessions for all apps and enrich app list + const sessions = await builderSocket?.getRoomSessions(devAppIds) + if (sessions?.length) { + let appSessionMap: Record = {} + sessions.forEach(session => { + const room = session.room + if (!room) { + return + } + if (!appSessionMap[room]) { + appSessionMap[room] = [] + } + appSessionMap[room].push(session) + }) + return apps.map(app => { + // Shallow clone to avoid mutating original reference + let enriched = { ...app } + const sessions = appSessionMap[app.appId] + if (sessions?.length) { + enriched.sessions = sessions + } else { + delete enriched.sessions + } + return enriched + }) + } else { + return apps + } +} From 90bf4655eaa924408cf095b6fa0dff4d1de3e136 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 15:57:12 +0200 Subject: [PATCH 18/30] Remove ctx from export rows (search not implemented) --- .../server/src/api/controllers/row/index.ts | 34 +++++++++++-- packages/server/src/sdk/app/rows/search.ts | 20 +++++++- .../src/sdk/app/rows/search/external.ts | 50 +++++++++---------- .../src/sdk/app/rows/search/internal.ts | 32 ++++++------ packages/server/src/sdk/app/rows/utils.ts | 3 +- 5 files changed, 92 insertions(+), 47 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index ed441f7784..78d64610b0 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -6,6 +6,8 @@ import { Ctx } from "@budibase/types" import * as utils from "./utils" import { gridSocket } from "../../../websockets" import sdk from "../../../sdk" +import * as exporters from "../view/exporters" +import { apiFileReturn } from "../../../utilities/fileSystem" function pickApi(tableId: any) { if (isExternalTable(tableId)) { @@ -164,7 +166,33 @@ export async function fetchEnrichedRow(ctx: any) { export const exportRows = async (ctx: any) => { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => sdk.rows.exportRows(tableId, ctx), { - datasourceId: tableId, - }) + + const format = ctx.query.format + + const { rows, columns, query } = ctx.request.body + if (typeof format !== "string" || !exporters.isFormat(format)) { + ctx.throw( + 400, + `Format ${format} not valid. Valid values: ${Object.values( + exporters.Format + )}` + ) + } + + ctx.body = await quotas.addQuery( + async () => { + const { fileName, content } = await sdk.rows.exportRows({ + tableId, + format, + rowIds: rows, + columns, + query, + }) + ctx.attachment(fileName) + return apiFileReturn(content) + }, + { + datasourceId: tableId, + } + ) } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index f8f50df2e6..b3b01d5d6c 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -2,6 +2,7 @@ import { Ctx, SearchFilters } from "@budibase/types" import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" +import { Format } from "../../../api/controllers/view/exporters" export interface SearchParams { tableId: string @@ -31,8 +32,23 @@ export async function search(tableId: string, ctx: Ctx) { return pickApi(tableId).search(ctx) } -export async function exportRows(tableId: string, ctx: Ctx) { - return pickApi(tableId).exportRows(ctx) +export interface ExportRowsParams { + tableId: string + format: Format + rowIds: string[] + columns: string[] + query: string +} + +export interface ExportRowsResult { + fileName: string + content: string +} + +export async function exportRows( + options: ExportRowsParams +): Promise { + return pickApi(options.tableId).exportRows(options) } export async function fetch(tableId: string) { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index e6d910b1c9..fe175d9d53 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -14,6 +14,7 @@ import { breakExternalTableId } from "../../../../integrations/utils" import { cleanExportRows } from "../utils" import { apiFileReturn } from "../../../../utilities/fileSystem" import { utils } from "@budibase/shared-core" +import { ExportRowsParams, ExportRowsResult } from "../search" export async function search(ctx: Ctx) { const tableId = ctx.params.tableId @@ -78,34 +79,30 @@ export async function search(ctx: Ctx) { } } -export async function exportRows(ctx: Ctx) { - const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId) - const format = ctx.query.format as string - const { columns } = ctx.request.body +export async function exportRows( + options: ExportRowsParams +): Promise { + const { tableId, format, columns, rowIds } = options + const { datasourceId, tableName } = breakExternalTableId(tableId) + const datasource = await sdk.datasources.get(datasourceId!) if (!datasource || !datasource.entities) { - ctx.throw(400, "Datasource has not been configured for plus API.") + throw ctx.throw(400, "Datasource has not been configured for plus API.") } - if (!exporters.isFormat(format)) { - ctx.throw( - 400, - `Format ${format} not valid. Valid values: ${Object.values( - exporters.Format - )}` - ) - } - - if (ctx.request.body.rows) { + if (rowIds?.length) { ctx.request.body = { query: { oneOf: { - _id: ctx.request.body.rows.map((row: string) => { + _id: rowIds.map((row: string) => { const ids = JSON.parse( decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") ) if (ids.length > 1) { - ctx.throw(400, "Export data does not support composite keys.") + throw ctx.throw( + 400, + "Export data does not support composite keys." + ) } return ids[0] }), @@ -131,14 +128,14 @@ export async function exportRows(ctx: Ctx) { } if (!tableName) { - ctx.throw(400, "Could not find table name.") + throw ctx.throw(400, "Could not find table name.") } - let schema = datasource.entities[tableName].schema + const schema = datasource.entities[tableName].schema let exportRows = cleanExportRows(rows, schema, format, columns) let headers = Object.keys(schema) - let content + let content: string switch (format) { case exporters.Format.CSV: content = exporters.csv(headers, exportRows) @@ -150,15 +147,14 @@ export async function exportRows(ctx: Ctx) { content = exporters.jsonWithSchema(schema, exportRows) break default: - utils.unreachable(format) - break + throw utils.unreachable(format) } - const filename = `export.${format}` - - // send down the file - ctx.attachment(filename) - return apiFileReturn(content) + const fileName = `export.${format}` + return { + fileName, + content, + } } export async function fetch(tableId: string) { diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 7caa5083c9..0407d1b630 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -25,6 +25,7 @@ import { getFromMemoryDoc, } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" +import { ExportRowsParams, ExportRowsResult } from "../search" export async function search(ctx: Ctx) { const { tableId } = ctx.params @@ -67,15 +68,12 @@ export async function search(ctx: Ctx) { return response } -export async function exportRows(ctx: Ctx) { +export async function exportRows( + options: ExportRowsParams +): Promise { + const { tableId, format, rowIds, columns, query } = options const db = context.getAppDB() - const table = await db.get(ctx.params.tableId) - const rowIds = ctx.request.body.rows - let format = ctx.query.format - if (typeof format !== "string") { - ctx.throw(400, "Format parameter is not valid") - } - const { columns, query } = ctx.request.body + const table = await db.get(tableId) let result if (rowIds) { @@ -109,14 +107,20 @@ export async function exportRows(ctx: Ctx) { let exportRows = cleanExportRows(rows, schema, format, columns) if (format === Format.CSV) { - ctx.attachment("export.csv") - return apiFileReturn(csv(Object.keys(rows[0]), exportRows)) + return { + fileName: "export.csv", + content: csv(Object.keys(rows[0]), exportRows), + } } else if (format === Format.JSON) { - ctx.attachment("export.json") - return apiFileReturn(json(exportRows)) + return { + fileName: "export.json", + content: json(exportRows), + } } else if (format === Format.JSON_WITH_SCHEMA) { - ctx.attachment("export.json") - return apiFileReturn(jsonWithSchema(schema, exportRows)) + return { + fileName: "export.json", + content: jsonWithSchema(schema, exportRows), + } } else { throw "Format not recognised" } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 1f5be1479b..d9f64af163 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -1,3 +1,4 @@ +import { TableSchema } from "@budibase/types" import { FieldTypes } from "../../../constants" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" @@ -11,7 +12,7 @@ export async function getDatasourceAndQuery(json: any) { export function cleanExportRows( rows: any[], - schema: any, + schema: TableSchema, format: string, columns: string[] ) { From 9271210b48e1455e28b24c417a8dd94c6e78abd9 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 17 Jul 2023 16:41:48 +0100 Subject: [PATCH 19/30] Update application.ts --- packages/server/src/api/controllers/application.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 2c53d3ad35..a2448a0384 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -179,7 +179,7 @@ export const addSampleData = async (ctx: UserCtx) => { export async function fetch(ctx: UserCtx) { const dev = ctx.query && ctx.query.status === AppStatus.DEV const all = ctx.query && ctx.query.status === AppStatus.ALL - let apps = (await dbCore.getAllApps({ dev, all })) as App[] + const apps = (await dbCore.getAllApps({ dev, all })) as App[] const appIds = apps .filter((app: any) => app.status === "development") From ccb5143383412f4efc48974af0eb1dd994bffc33 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 18:45:40 +0200 Subject: [PATCH 20/30] Remove context from search --- .../server/src/api/controllers/row/index.ts | 9 ++- packages/server/src/sdk/app/rows/search.ts | 18 +++--- .../src/sdk/app/rows/search/external.ts | 56 +++++++++---------- .../src/sdk/app/rows/search/internal.ts | 33 +++++++---- 4 files changed, 67 insertions(+), 49 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 78d64610b0..673f2adb53 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -134,9 +134,14 @@ export async function destroy(ctx: any) { export async function search(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.status = 200 - ctx.body = await quotas.addQuery(() => sdk.rows.search(tableId, ctx), { + const searchParams = { + ...ctx.request.body, + tableId, + } + + ctx.status = 200 + ctx.body = await quotas.addQuery(() => sdk.rows.search(searchParams), { datasourceId: tableId, }) } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index b3b01d5d6c..b43af78740 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,4 +1,4 @@ -import { Ctx, SearchFilters } from "@budibase/types" +import { SearchFilters } from "@budibase/types" import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" @@ -6,13 +6,15 @@ import { Format } from "../../../api/controllers/view/exporters" export interface SearchParams { tableId: string - paginate: boolean - query?: SearchFilters - bookmark?: number - limit: number + paginate?: boolean + query: SearchFilters + bookmark?: string + limit?: number sort?: string sortOrder?: string sortType?: string + version?: string + disableEscaping?: boolean } export interface ViewParams { @@ -28,8 +30,8 @@ function pickApi(tableId: any) { return internal } -export async function search(tableId: string, ctx: Ctx) { - return pickApi(tableId).search(ctx) +export async function search(options: SearchParams) { + return pickApi(options.tableId).search(options) } export interface ExportRowsParams { @@ -37,7 +39,7 @@ export interface ExportRowsParams { format: Format rowIds: string[] columns: string[] - query: string + query: SearchFilters } export interface ExportRowsResult { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index fe175d9d53..3317b0e398 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -5,22 +5,23 @@ import { PaginationJson, IncludeRelationship, Row, - Ctx, + SearchFilters, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import sdk from "../../../../sdk" import { handleRequest } from "../../../../api/controllers/row/external" import { breakExternalTableId } from "../../../../integrations/utils" import { cleanExportRows } from "../utils" -import { apiFileReturn } from "../../../../utilities/fileSystem" import { utils } from "@budibase/shared-core" -import { ExportRowsParams, ExportRowsResult } from "../search" +import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" +import { HTTPError } from "@budibase/backend-core" -export async function search(ctx: Ctx) { - const tableId = ctx.params.tableId - const { paginate, query, ...params } = ctx.request.body - let { bookmark, limit } = params - if (!bookmark && paginate) { +export async function search(options: SearchParams) { + const { tableId } = options + const { paginate, query, ...params } = options + const { limit } = params + let bookmark = (params.bookmark && parseInt(params.bookmark)) || undefined + if (paginate && bookmark) { bookmark = 1 } let paginateObj = {} @@ -60,14 +61,14 @@ export async function search(ctx: Ctx) { sort, paginate: { limit: 1, - page: bookmark * limit + 1, + page: bookmark! * limit + 1, }, includeSqlRelationships: IncludeRelationship.INCLUDE, })) as Row[] hasNextPage = nextRows.length > 0 } // need wrapper object for bookmarks etc when paginating - return { rows, hasNextPage, bookmark: bookmark + 1 } + return { rows, hasNextPage, bookmark: (bookmark || 0) + 1 } } catch (err: any) { if (err.message && err.message.includes("does not exist")) { throw new Error( @@ -87,31 +88,30 @@ export async function exportRows( const datasource = await sdk.datasources.get(datasourceId!) if (!datasource || !datasource.entities) { - throw ctx.throw(400, "Datasource has not been configured for plus API.") + throw new HTTPError("Datasource has not been configured for plus API.", 400) } + let query: SearchFilters = {} if (rowIds?.length) { - ctx.request.body = { - query: { - oneOf: { - _id: rowIds.map((row: string) => { - const ids = JSON.parse( - decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") + query = { + oneOf: { + _id: rowIds.map((row: string) => { + const ids = JSON.parse( + decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") + ) + if (ids.length > 1) { + throw new HTTPError( + "Export data does not support composite keys.", + 400 ) - if (ids.length > 1) { - throw ctx.throw( - 400, - "Export data does not support composite keys." - ) - } - return ids[0] - }), - }, + } + return ids[0] + }), }, } } - let result = await search(ctx) + let result = await search({ tableId, query }) let rows: Row[] = [] // Filter data to only specified columns if required @@ -128,7 +128,7 @@ export async function exportRows( } if (!tableName) { - throw ctx.throw(400, "Could not find table name.") + throw new HTTPError("Could not find table name.", 400) } const schema = datasource.entities[tableName].schema let exportRows = cleanExportRows(rows, schema, format, columns) diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 0407d1b630..70274de34a 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -1,4 +1,7 @@ -import { context } from "@budibase/backend-core" +import { + context, + SearchParams as InternalSearchParams, +} from "@budibase/backend-core" import env from "../../../../environment" import { fullSearch, paginatedSearch } from "./internalSearch" import { @@ -8,7 +11,7 @@ import { } from "../../../../db/utils" import { getGlobalUsersFromMetadata } from "../../../../utilities/global" import { outputProcessing } from "../../../../utilities/rowProcessor" -import { Ctx, Database, Row } from "@budibase/types" +import { Database, Row } from "@budibase/types" import { cleanExportRows } from "../utils" import { Format, @@ -16,7 +19,6 @@ import { json, jsonWithSchema, } from "../../../../api/controllers/view/exporters" -import { apiFileReturn } from "../../../../utilities/fileSystem" import * as inMemoryViews from "../../../../db/inMemoryView" import { migrateToInMemoryView, @@ -25,10 +27,10 @@ import { getFromMemoryDoc, } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" -import { ExportRowsParams, ExportRowsResult } from "../search" +import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" -export async function search(ctx: Ctx) { - const { tableId } = ctx.params +export async function search(options: SearchParams) { + const { tableId } = options // Fetch the whole table when running in cypress, as search doesn't work if (!env.COUCH_DB_URL && env.isCypress()) { @@ -36,16 +38,25 @@ export async function search(ctx: Ctx) { } const db = context.getAppDB() - const { paginate, query, ...params } = ctx.request.body - params.version = ctx.version - params.tableId = tableId + const { paginate, query } = options + + const params: InternalSearchParams = { + tableId: options.tableId, + sort: options.sort, + sortOrder: options.sortOrder, + sortType: options.sortType, + limit: options.limit, + bookmark: options.bookmark, + version: options.version, + disableEscaping: options.disableEscaping, + } let table if (params.sort && !params.sortType) { table = await db.get(tableId) const schema = table.schema const sortField = schema[params.sort] - params.sortType = sortField.type == "number" ? "number" : "string" + params.sortType = sortField.type === "number" ? "number" : "string" } let response @@ -86,7 +97,7 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { - let searchResponse = await search(ctx) + let searchResponse = await search({ tableId, query }) result = searchResponse.rows } From 07607c0fd2effe8a6799e5ffebac1239b457756e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 19:07:54 +0200 Subject: [PATCH 21/30] Fix tests --- packages/server/src/sdk/app/rows/search/external.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 3317b0e398..223e1dcd7b 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -20,8 +20,8 @@ export async function search(options: SearchParams) { const { tableId } = options const { paginate, query, ...params } = options const { limit } = params - let bookmark = (params.bookmark && parseInt(params.bookmark)) || undefined - if (paginate && bookmark) { + let bookmark = (params.bookmark && parseInt(params.bookmark)) || null + if (paginate && !bookmark) { bookmark = 1 } let paginateObj = {} @@ -68,7 +68,7 @@ export async function search(options: SearchParams) { hasNextPage = nextRows.length > 0 } // need wrapper object for bookmarks etc when paginating - return { rows, hasNextPage, bookmark: (bookmark || 0) + 1 } + return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 } } catch (err: any) { if (err.message && err.message.includes("does not exist")) { throw new Error( From deb256a013dddb90113c317d81250ae74a8a0162 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 19:50:06 +0200 Subject: [PATCH 22/30] Refactor and clean export tests --- .../src/api/controllers/row/external.ts | 1 + packages/server/src/sdk/app/rows/search.ts | 4 +- .../src/sdk/app/rows/search/external.ts | 10 +- packages/server/src/sdk/app/rows/utils.ts | 2 +- .../server/src/sdk/tests/rows/row.spec.ts | 115 ++++++++---------- 5 files changed, 59 insertions(+), 73 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 36329b3469..2122ada068 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -49,6 +49,7 @@ export async function handleRequest( } } } + return new ExternalRequest(operation, tableId, opts?.datasource).run( opts || {} ) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index b43af78740..87a1662a54 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -37,8 +37,8 @@ export async function search(options: SearchParams) { export interface ExportRowsParams { tableId: string format: Format - rowIds: string[] - columns: string[] + rowIds?: string[] + columns?: string[] query: SearchFilters } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 223e1dcd7b..a9da764e88 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -86,11 +86,6 @@ export async function exportRows( const { tableId, format, columns, rowIds } = options const { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource = await sdk.datasources.get(datasourceId!) - if (!datasource || !datasource.entities) { - throw new HTTPError("Datasource has not been configured for plus API.", 400) - } - let query: SearchFilters = {} if (rowIds?.length) { query = { @@ -111,6 +106,11 @@ export async function exportRows( } } + const datasource = await sdk.datasources.get(datasourceId!) + if (!datasource || !datasource.entities) { + throw new HTTPError("Datasource has not been configured for plus API.", 400) + } + let result = await search({ tableId, query }) let rows: Row[] = [] diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index d9f64af163..6a037a4ade 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -14,7 +14,7 @@ export function cleanExportRows( rows: any[], schema: TableSchema, format: string, - columns: string[] + columns?: string[] ) { let cleanRows = [...rows] diff --git a/packages/server/src/sdk/tests/rows/row.spec.ts b/packages/server/src/sdk/tests/rows/row.spec.ts index 68140345b7..08c5746f2e 100644 --- a/packages/server/src/sdk/tests/rows/row.spec.ts +++ b/packages/server/src/sdk/tests/rows/row.spec.ts @@ -1,6 +1,10 @@ import { exportRows } from "../../app/rows/search/external" import sdk from "../.." import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest" +import { ExportRowsParams } from "../../app/rows/search" +import { Format } from "../../../api/controllers/view/exporters" +import { HTTPError } from "@budibase/backend-core" +import { Operation } from "@budibase/types" const mockDatasourcesGet = jest.fn() sdk.datasources.get = mockDatasourcesGet @@ -16,30 +20,21 @@ jest.mock("../../../api/controllers/view/exporters", () => ({ })) jest.mock("../../../utilities/fileSystem") -function getUserCtx() { - return { - params: { - tableId: "datasource__tablename", - }, - query: { - format: "csv", - }, - request: { - body: {}, - }, - throw: jest.fn(() => { - throw "Err" - }), - attachment: jest.fn(), - } as any -} - -describe("external row controller", () => { +describe("external row sdk", () => { describe("exportRows", () => { + function getExportOptions(): ExportRowsParams { + return { + tableId: "datasource__tablename", + format: Format.CSV, + query: {}, + } + } + + const externalRequestCall = jest.fn() beforeAll(() => { jest .spyOn(ExternalRequest.prototype, "run") - .mockImplementation(() => Promise.resolve([])) + .mockImplementation(externalRequestCall.mockResolvedValue([])) }) afterEach(() => { @@ -47,15 +42,10 @@ describe("external row controller", () => { }) it("should throw a 400 if no datasource entities are present", async () => { - let userCtx = getUserCtx() - try { - await exportRows(userCtx) - } catch (e) { - expect(userCtx.throw).toHaveBeenCalledWith( - 400, - "Datasource has not been configured for plus API." - ) - } + const exportOptions = getExportOptions() + await expect(exportRows(exportOptions)).rejects.toThrowError( + new HTTPError("Datasource has not been configured for plus API.", 400) + ) }) it("should handle single quotes from a row ID", async () => { @@ -66,51 +56,46 @@ describe("external row controller", () => { }, }, })) - let userCtx = getUserCtx() - userCtx.request.body = { - rows: ["['d001']"], - } + const exportOptions = getExportOptions() + exportOptions.rowIds = ["['d001']"] - await exportRows(userCtx) + await exportRows(exportOptions) - expect(userCtx.request.body).toEqual({ - query: { - oneOf: { - _id: ["d001"], + expect(ExternalRequest).toBeCalledTimes(1) + expect(ExternalRequest).toBeCalledWith( + Operation.READ, + exportOptions.tableId, + undefined + ) + + expect(externalRequestCall).toBeCalledTimes(1) + expect(externalRequestCall).toBeCalledWith( + expect.objectContaining({ + filters: { + oneOf: { + _id: ["d001"], + }, }, - }, - }) + }) + ) }) it("should throw a 400 if any composite keys are present", async () => { - let userCtx = getUserCtx() - userCtx.request.body = { - rows: ["[123]", "['d001'%2C'10111']"], - } - try { - await exportRows(userCtx) - } catch (e) { - expect(userCtx.throw).toHaveBeenCalledWith( - 400, - "Export data does not support composite keys." - ) - } + const exportOptions = getExportOptions() + exportOptions.rowIds = ["[123]", "['d001'%2C'10111']"] + await expect(exportRows(exportOptions)).rejects.toThrowError( + new HTTPError("Export data does not support composite keys.", 400) + ) }) it("should throw a 400 if no table name was found", async () => { - let userCtx = getUserCtx() - userCtx.params.tableId = "datasource__" - userCtx.request.body = { - rows: ["[123]"], - } - try { - await exportRows(userCtx) - } catch (e) { - expect(userCtx.throw).toHaveBeenCalledWith( - 400, - "Could not find table name." - ) - } + const exportOptions = getExportOptions() + exportOptions.tableId = "datasource__" + exportOptions.rowIds = ["[123]"] + + await expect(exportRows(exportOptions)).rejects.toThrowError( + new HTTPError("Could not find table name.", 400) + ) }) }) }) From 2da66da5e35820022c4edc31fadb7991dfbac391 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 17 Jul 2023 19:11:35 +0000 Subject: [PATCH 23/30] Bump version to 2.8.12-alpha.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 4733073856..e6a6bbdcd3 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.8.12-alpha.0", + "version": "2.8.12-alpha.1", "npmClient": "yarn", "packages": [ "packages/*" From 1fbff5c6ef633d07267a71f6a863e363fe2e7e83 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 18 Jul 2023 07:13:13 +0000 Subject: [PATCH 24/30] Bump version to 2.8.12-alpha.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index e6a6bbdcd3..dd403724b2 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.8.12-alpha.1", + "version": "2.8.12-alpha.2", "npmClient": "yarn", "packages": [ "packages/*" From c4c62e5c6ff73bbffe93286344780533ba83a452 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 18 Jul 2023 10:48:55 +0200 Subject: [PATCH 25/30] Remove any from couchdb.get response --- packages/types/src/sdk/db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 58b3b7e5bc..e07cc82fd2 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -89,7 +89,7 @@ export interface Database { exists(): Promise checkSetup(): Promise> - get(id?: string): Promise + get(id?: string): Promise remove( id: string | Document, rev?: string From 1072292b9f95f6ff5e8f5563011fe9d1fcb879a7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 18 Jul 2023 10:49:39 +0200 Subject: [PATCH 26/30] Type breaking changes as any --- packages/backend-core/src/auth/auth.ts | 2 +- packages/backend-core/src/cache/user.ts | 2 +- packages/backend-core/src/db/searchIndexes/searchIndexes.ts | 2 +- packages/backend-core/src/users.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index fb2fd2cf51..0100a2d0e2 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -159,7 +159,7 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) { try { const db = getGlobalDB() - const dbUser = await db.get(userId) + const dbUser = await db.get(userId) //Do not overwrite the refresh token if a valid one is not provided. if (typeof details.refreshToken !== "string") { diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index b514c3af9b..8281bfca62 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -12,7 +12,7 @@ const EXPIRY_SECONDS = 3600 */ async function populateFromDB(userId: string, tenantId: string) { const db = tenancy.getTenantDB(tenantId) - const user = await db.get(userId) + const user = await db.get(userId) user.budibaseAccess = true if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { const account = await accounts.getAccount(user.email) diff --git a/packages/backend-core/src/db/searchIndexes/searchIndexes.ts b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts index f03259b47f..b953e3516e 100644 --- a/packages/backend-core/src/db/searchIndexes/searchIndexes.ts +++ b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts @@ -5,7 +5,7 @@ export async function createUserIndex() { const db = getGlobalDB() let designDoc try { - designDoc = await db.get("_design/database") + designDoc = await db.get("_design/database") } catch (err: any) { if (err.status === 404) { designDoc = { _id: "_design/database" } diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 166136df3c..b49058f546 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -67,9 +67,9 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => { export async function getById(id: string, opts?: GetOpts): Promise { const db = context.getGlobalDB() - let user = await db.get(id) + let user = await db.get(id) if (opts?.cleanup) { - user = removeUserPassword(user) + user = removeUserPassword(user) as User } return user } From 2698e47ec4be211631f4f157db68e44f5320a6cc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 18 Jul 2023 11:41:51 +0200 Subject: [PATCH 27/30] Type internal db.get --- packages/pro | 2 +- .../server/src/api/controllers/analytics.ts | 4 +-- .../server/src/api/controllers/apikeys.ts | 2 +- .../server/src/api/controllers/application.ts | 10 +++---- packages/server/src/api/controllers/auth.ts | 3 ++- .../server/src/api/controllers/automation.ts | 15 ++++++----- packages/server/src/api/controllers/backup.ts | 4 +-- .../server/src/api/controllers/component.ts | 4 +-- .../server/src/api/controllers/datasource.ts | 4 +-- .../src/api/controllers/deploy/index.ts | 10 +++---- packages/server/src/api/controllers/dev.ts | 5 ++-- .../src/api/controllers/public/utils.ts | 2 +- .../server/src/api/controllers/query/index.ts | 7 ++--- packages/server/src/api/controllers/role.ts | 17 ++++++------ .../src/api/controllers/row/internal.ts | 17 ++++++------ .../src/api/controllers/row/staticFormula.ts | 7 ++--- packages/server/src/api/controllers/screen.ts | 6 ++--- .../src/api/controllers/static/index.ts | 7 ++--- .../src/api/controllers/table/external.ts | 4 +-- .../src/api/controllers/table/internal.ts | 12 ++++----- packages/server/src/api/controllers/user.ts | 6 ++--- .../server/src/api/controllers/view/index.ts | 7 ++--- .../server/src/api/controllers/view/utils.ts | 26 +++++++++---------- .../server/src/api/controllers/webhook.ts | 2 +- packages/server/src/automations/utils.ts | 11 +++++--- .../src/db/linkedRows/LinkController.ts | 8 +++--- packages/server/src/db/views/staticViews.ts | 6 ++--- packages/server/src/middleware/builder.ts | 4 +-- .../server/src/sdk/app/applications/sync.ts | 2 +- .../src/sdk/app/datasources/datasources.ts | 4 +-- .../src/sdk/app/rows/search/internal.ts | 16 ++++++------ packages/server/src/sdk/users/crud.ts | 7 +++++ packages/server/src/sdk/users/index.ts | 2 ++ packages/server/src/threads/query.ts | 3 ++- packages/server/src/utilities/global.ts | 2 +- packages/server/src/utilities/index.ts | 4 +-- .../src/api/controllers/global/email.ts | 6 ++--- .../src/api/controllers/global/roles.ts | 2 +- .../worker/src/api/controllers/global/self.ts | 2 +- .../src/api/controllers/global/users.ts | 2 +- 40 files changed, 144 insertions(+), 120 deletions(-) create mode 100644 packages/server/src/sdk/users/crud.ts diff --git a/packages/pro b/packages/pro index 544c7e067d..1929d93660 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 544c7e067de69832469cde673e59501480d6d98a +Subproject commit 1929d93660e95f4b190d1936d0b3c65c4ad6a1f5 diff --git a/packages/server/src/api/controllers/analytics.ts b/packages/server/src/api/controllers/analytics.ts index dc7c4185bb..eee86a534b 100644 --- a/packages/server/src/api/controllers/analytics.ts +++ b/packages/server/src/api/controllers/analytics.ts @@ -1,5 +1,5 @@ import { events } from "@budibase/backend-core" -import { AnalyticsPingRequest, PingSource } from "@budibase/types" +import { AnalyticsPingRequest, App, PingSource } from "@budibase/types" import { DocumentType, isDevAppID } from "../../db/utils" import { context } from "@budibase/backend-core" @@ -16,7 +16,7 @@ export const ping = async (ctx: any) => { switch (body.source) { case PingSource.APP: { const db = context.getAppDB({ skip_setup: true }) - const appInfo = await db.get(DocumentType.APP_METADATA) + const appInfo = await db.get(DocumentType.APP_METADATA) let appId = context.getAppId() if (isDevAppID(appId)) { diff --git a/packages/server/src/api/controllers/apikeys.ts b/packages/server/src/api/controllers/apikeys.ts index 02b61954c3..2a02078483 100644 --- a/packages/server/src/api/controllers/apikeys.ts +++ b/packages/server/src/api/controllers/apikeys.ts @@ -6,7 +6,7 @@ const KEYS_DOC = dbCore.StaticDatabases.GLOBAL.docs.apiKeys async function getBuilderMainDoc() { const db = tenancy.getGlobalDB() try { - return await db.get(KEYS_DOC) + return await db.get(KEYS_DOC) } catch (err) { // doesn't exist yet, nothing to get return { diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index a2448a0384..418ccf637e 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -222,7 +222,7 @@ export async function fetchAppDefinition(ctx: UserCtx) { export async function fetchAppPackage(ctx: UserCtx) { const db = context.getAppDB() - let application = await db.get(DocumentType.APP_METADATA) + let application = await db.get(DocumentType.APP_METADATA) const layouts = await getLayouts() let screens = await getScreens() const license = await licensing.cache.getCachedLicense() @@ -458,7 +458,7 @@ export async function update(ctx: UserCtx) { export async function updateClient(ctx: UserCtx) { // Get current app version const db = context.getAppDB() - const application = await db.get(DocumentType.APP_METADATA) + const application = await db.get(DocumentType.APP_METADATA) const currentVersion = application.version // Update client library and manifest @@ -482,7 +482,7 @@ export async function updateClient(ctx: UserCtx) { export async function revertClient(ctx: UserCtx) { // Check app can be reverted const db = context.getAppDB() - const application = await db.get(DocumentType.APP_METADATA) + const application = await db.get(DocumentType.APP_METADATA) if (!application.revertableVersion) { ctx.throw(400, "There is no version to revert to") } @@ -535,7 +535,7 @@ async function destroyApp(ctx: UserCtx) { const db = dbCore.getDB(devAppId) // standard app deletion flow - const app = await db.get(DocumentType.APP_METADATA) + const app = await db.get(DocumentType.APP_METADATA) const result = await db.destroy() await quotas.removeApp() await events.app.deleted(app) @@ -598,7 +598,7 @@ export async function sync(ctx: UserCtx) { export async function updateAppPackage(appPackage: any, appId: any) { return context.doInAppContext(appId, async () => { const db = context.getAppDB() - const application = await db.get(DocumentType.APP_METADATA) + const application = await db.get(DocumentType.APP_METADATA) const newAppPackage = { ...application, ...appPackage } if (appPackage._rev !== application._rev) { diff --git a/packages/server/src/api/controllers/auth.ts b/packages/server/src/api/controllers/auth.ts index 15b99bf2f8..bca6c4e64e 100644 --- a/packages/server/src/api/controllers/auth.ts +++ b/packages/server/src/api/controllers/auth.ts @@ -4,6 +4,7 @@ import { getFullUser } from "../../utilities/users" import { roles, context } from "@budibase/backend-core" import { groups } from "@budibase/pro" import { ContextUser, User, Row, UserCtx } from "@budibase/types" +import sdk from "../../sdk" const PUBLIC_ROLE = roles.BUILTIN_ROLE_IDS.PUBLIC @@ -41,7 +42,7 @@ export async function fetchSelf(ctx: UserCtx) { // remove the full roles structure delete user.roles try { - const userTable = await db.get(InternalTables.USER_METADATA) + const userTable = await sdk.tables.getTable(InternalTables.USER_METADATA) // specifically needs to make sure is enriched ctx.body = await outputProcessing(userTable, user as Row) } catch (err: any) { diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index 25d915e995..a4697f99f2 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -16,6 +16,7 @@ import { setTestFlag, clearTestFlag } from "../../utilities/redis" import { context, cache, events } from "@budibase/backend-core" import { automations, features } from "@budibase/pro" import { + App, Automation, AutomationActionStepId, AutomationResults, @@ -152,7 +153,7 @@ export async function update(ctx: BBContext) { return } - const oldAutomation = await db.get(automation._id) + const oldAutomation = await db.get(automation._id) automation = cleanAutomationInputs(automation) automation = await checkForWebhooks({ oldAuto: oldAutomation, @@ -210,7 +211,7 @@ export async function find(ctx: BBContext) { export async function destroy(ctx: BBContext) { const db = context.getAppDB() const automationId = ctx.params.id - const oldAutomation = await db.get(automationId) + const oldAutomation = await db.get(automationId) await checkForWebhooks({ oldAuto: oldAutomation, }) @@ -229,7 +230,7 @@ export async function clearLogError(ctx: BBContext) { const { automationId, appId } = ctx.request.body await context.doInAppContext(appId, async () => { const db = context.getProdAppDB() - const metadata = await db.get(DocumentType.APP_METADATA) + const metadata = await db.get(DocumentType.APP_METADATA) if (!automationId) { delete metadata.automationErrors } else if ( @@ -267,7 +268,7 @@ export async function getDefinitionList(ctx: BBContext) { export async function trigger(ctx: BBContext) { const db = context.getAppDB() - let automation = await db.get(ctx.params.id) + let automation = await db.get(ctx.params.id) let hasCollectStep = sdk.automations.utils.checkForCollectStep(automation) if (hasCollectStep && (await features.isSyncAutomationsEnabled())) { @@ -312,8 +313,8 @@ function prepareTestInput(input: any) { export async function test(ctx: BBContext) { const db = context.getAppDB() - let automation = await db.get(ctx.params.id) - await setTestFlag(automation._id) + let automation = await db.get(ctx.params.id) + await setTestFlag(automation._id!) const testInput = prepareTestInput(ctx.request.body) const response = await triggers.externalTrigger( automation, @@ -328,7 +329,7 @@ export async function test(ctx: BBContext) { ...ctx.request.body, occurredAt: new Date().getTime(), }) - await clearTestFlag(automation._id) + await clearTestFlag(automation._id!) ctx.body = response await events.automation.tested(automation) } diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index 2a7921d354..8e1881a5fc 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -1,7 +1,7 @@ import sdk from "../../sdk" import { events, context, db } from "@budibase/backend-core" import { DocumentType } from "../../db/utils" -import { Ctx } from "@budibase/types" +import { App, Ctx } from "@budibase/types" interface ExportAppDumpRequest { excludeRows: boolean @@ -29,7 +29,7 @@ export async function exportAppDump(ctx: Ctx) { await context.doInAppContext(appId, async () => { const appDb = context.getAppDB() - const app = await appDb.get(DocumentType.APP_METADATA) + const app = await appDb.get(DocumentType.APP_METADATA) await events.app.exported(app) }) } diff --git a/packages/server/src/api/controllers/component.ts b/packages/server/src/api/controllers/component.ts index 52b1ac66bf..1d0ccf8784 100644 --- a/packages/server/src/api/controllers/component.ts +++ b/packages/server/src/api/controllers/component.ts @@ -1,5 +1,5 @@ import { DocumentType } from "../../db/utils" -import { Plugin } from "@budibase/types" +import { App, Plugin } from "@budibase/types" import { db as dbCore, context, tenancy } from "@budibase/backend-core" import { getComponentLibraryManifest } from "../../utilities/fileSystem" import { UserCtx } from "@budibase/types" @@ -7,7 +7,7 @@ import { UserCtx } from "@budibase/types" export async function fetchAppComponentDefinitions(ctx: UserCtx) { try { const db = context.getAppDB() - const app = await db.get(DocumentType.APP_METADATA) + const app = await db.get(DocumentType.APP_METADATA) let componentManifests = await Promise.all( app.componentLibraries.map(async (library: any) => { diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 65902a542e..8ef2e8789a 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -433,7 +433,7 @@ export async function destroy(ctx: UserCtx) { export async function find(ctx: UserCtx) { const db = context.getAppDB() - const datasource = await db.get(ctx.params.datasourceId) + const datasource = await sdk.datasources.get(ctx.params.datasourceId) ctx.body = await sdk.datasources.removeSecretSingle(datasource) } @@ -449,7 +449,7 @@ export async function query(ctx: UserCtx) { export async function getExternalSchema(ctx: UserCtx) { const db = context.getAppDB() - const datasource = await db.get(ctx.params.datasourceId) + const datasource = await sdk.datasources.get(ctx.params.datasourceId) const enrichedDatasource = await getAndMergeDatasource(datasource) const connector = await getConnector(enrichedDatasource) diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index a49434bbd1..a2b164ffc6 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -7,7 +7,7 @@ import { enableCronTrigger, } from "../../../automations/utils" import { backups } from "@budibase/pro" -import { AppBackupTrigger } from "@budibase/types" +import { App, AppBackupTrigger } from "@budibase/types" import sdk from "../../../sdk" import { builderSocket } from "../../../websockets" @@ -44,7 +44,7 @@ async function storeDeploymentHistory(deployment: any) { let deploymentDoc try { // theres only one deployment doc per app database - deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) + deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) } catch (err) { deploymentDoc = { _id: DocumentType.DEPLOYMENTS, history: {} } } @@ -113,7 +113,7 @@ export async function fetchDeployments(ctx: any) { export async function deploymentProgress(ctx: any) { try { const db = context.getAppDB() - const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) + const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) ctx.body = deploymentDoc[ctx.params.deploymentId] } catch (err) { ctx.throw( @@ -165,9 +165,9 @@ export const publishApp = async function (ctx: any) { // app metadata is excluded as it is likely to be in conflict // replicate the app metadata document manually const db = context.getProdAppDB() - const appDoc = await devDb.get(DocumentType.APP_METADATA) + const appDoc = await devDb.get(DocumentType.APP_METADATA) try { - const prodAppDoc = await db.get(DocumentType.APP_METADATA) + const prodAppDoc = await db.get(DocumentType.APP_METADATA) appDoc._rev = prodAppDoc._rev } catch (err) { delete appDoc._rev diff --git a/packages/server/src/api/controllers/dev.ts b/packages/server/src/api/controllers/dev.ts index e80d9d7ea1..cd32fa9359 100644 --- a/packages/server/src/api/controllers/dev.ts +++ b/packages/server/src/api/controllers/dev.ts @@ -6,6 +6,7 @@ import { clearLock as redisClearLock } from "../../utilities/redis" import { DocumentType } from "../../db/utils" import { context, env as envCore } from "@budibase/backend-core" import { events, db as dbCore, cache } from "@budibase/backend-core" +import { App } from "@budibase/types" async function redirect(ctx: any, method: string, path: string = "global") { const { devPath } = ctx.params @@ -81,7 +82,7 @@ export async function revert(ctx: any) { if (!exists) { throw new Error("App must be deployed to be reverted.") } - const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) + const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) if ( !deploymentDoc.history || Object.keys(deploymentDoc.history).length === 0 @@ -104,7 +105,7 @@ export async function revert(ctx: any) { // update appID in reverted app to be dev version again const db = context.getAppDB() - const appDoc = await db.get(DocumentType.APP_METADATA) + const appDoc = await db.get(DocumentType.APP_METADATA) appDoc.appId = appId appDoc.instance._id = appId await db.put(appDoc) diff --git a/packages/server/src/api/controllers/public/utils.ts b/packages/server/src/api/controllers/public/utils.ts index a51b6b5fff..1272fcb36a 100644 --- a/packages/server/src/api/controllers/public/utils.ts +++ b/packages/server/src/api/controllers/public/utils.ts @@ -14,7 +14,7 @@ export async function addRev( id = DocumentType.APP_METADATA } const db = context.getAppDB() - const dbDoc = await db.get(id) + const dbDoc = await db.get(id) body._rev = dbDoc._rev // update ID in case it is an app ID body._id = id diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 356a2eb5c0..4c1cad6b5b 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -9,6 +9,7 @@ import { quotas } from "@budibase/pro" import { events, context, utils, constants } from "@budibase/backend-core" import sdk from "../../../sdk" import { QueryEvent } from "../../../threads/definitions" +import { Query } from "@budibase/types" const Runner = new Thread(ThreadType.QUERY, { timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000, @@ -206,7 +207,7 @@ async function execute( ) { const db = context.getAppDB() - const query = await db.get(ctx.params.queryId) + const query = await db.get(ctx.params.queryId) const { datasource, envVars } = await sdk.datasources.getWithEnvVars( query.datasourceId ) @@ -275,7 +276,7 @@ export async function executeV2( const removeDynamicVariables = async (queryId: any) => { const db = context.getAppDB() - const query = await db.get(queryId) + const query = await db.get(queryId) const datasource = await sdk.datasources.get(query.datasourceId) const dynamicVariables = datasource.config?.dynamicVariables as any[] @@ -298,7 +299,7 @@ export async function destroy(ctx: any) { const db = context.getAppDB() const queryId = ctx.params.queryId await removeDynamicVariables(queryId) - const query = await db.get(queryId) + const query = await db.get(queryId) const datasource = await sdk.datasources.get(query.datasourceId) await db.remove(ctx.params.queryId, ctx.params.revId) ctx.message = `Query deleted.` diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index a2369ef2da..5ab1ad3502 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -1,6 +1,7 @@ import { roles, context, events, db as dbCore } from "@budibase/backend-core" import { getUserMetadataParams, InternalTables } from "../../db/utils" -import { UserCtx, Database } from "@budibase/types" +import { UserCtx, Database, UserRoles, Role } from "@budibase/types" +import sdk from "../../sdk" const UpdateRolesOptions = { CREATED: "created", @@ -13,23 +14,23 @@ async function updateRolesOnUserTable( updateOption: string, roleVersion: string | undefined ) { - const table = await db.get(InternalTables.USER_METADATA) + const table = await sdk.tables.getTable(InternalTables.USER_METADATA) const schema = table.schema const remove = updateOption === UpdateRolesOptions.REMOVED let updated = false for (let prop of Object.keys(schema)) { if (prop === "roleId") { updated = true - const constraints = schema[prop].constraints + const constraints = schema[prop].constraints! const updatedRoleId = roleVersion === roles.RoleIDVersion.NAME ? roles.getExternalRoleID(roleId, roleVersion) : roleId - const indexOfRoleId = constraints.inclusion.indexOf(updatedRoleId) + const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId) if (remove && indexOfRoleId !== -1) { - constraints.inclusion.splice(indexOfRoleId, 1) + constraints.inclusion!.splice(indexOfRoleId, 1) } else if (!remove && indexOfRoleId === -1) { - constraints.inclusion.push(updatedRoleId) + constraints.inclusion!.push(updatedRoleId) } break } @@ -69,7 +70,7 @@ export async function save(ctx: UserCtx) { let dbRole if (!isCreate) { - dbRole = await db.get(_id) + dbRole = await db.get(_id) } if (dbRole && dbRole.name !== name && isNewVersion) { ctx.throw(400, "Cannot change custom role name") @@ -105,7 +106,7 @@ export async function destroy(ctx: UserCtx) { // make sure has the prefix (if it has it then it won't be added) roleId = dbCore.generateRoleID(roleId) } - const role = await db.get(roleId) + const role = await db.get(roleId) // first check no users actively attached to role const users = ( await db.allDocs( diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index a69aabd3c2..0367346832 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -16,6 +16,7 @@ import { cloneDeep } from "lodash/fp" import { context, db as dbCore } from "@budibase/backend-core" import { finaliseRow, updateRelatedFormula } from "./staticFormula" import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types" +import sdk from "../../../sdk" export async function patch(ctx: UserCtx) { const db = context.getAppDB() @@ -24,7 +25,7 @@ export async function patch(ctx: UserCtx) { const isUserTable = tableId === InternalTables.USER_METADATA let oldRow try { - let dbTable = await db.get(tableId) + let dbTable = await sdk.tables.getTable(tableId) oldRow = await outputProcessing( dbTable, await utils.findRow(ctx, tableId, inputs._id) @@ -40,7 +41,7 @@ export async function patch(ctx: UserCtx) { throw "Row does not exist" } } - let dbTable = await db.get(tableId) + let dbTable = await sdk.tables.getTable(tableId) // need to build up full patch fields before coerce let combinedRow: any = cloneDeep(oldRow) for (let key of Object.keys(inputs)) { @@ -95,7 +96,7 @@ export async function save(ctx: UserCtx) { } // this returns the table and row incase they have been updated - const dbTable = await db.get(inputs.tableId) + const dbTable = await sdk.tables.getTable(inputs.tableId) // need to copy the table so it can be differenced on way out const tableClone = cloneDeep(dbTable) @@ -127,7 +128,7 @@ export async function save(ctx: UserCtx) { export async function find(ctx: UserCtx) { const db = dbCore.getDB(ctx.appId) - const table = await db.get(ctx.params.tableId) + const table = await sdk.tables.getTable(ctx.params.tableId) let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId) row = await outputProcessing(table, row) return row @@ -136,13 +137,13 @@ export async function find(ctx: UserCtx) { export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const { _id } = ctx.request.body - let row = await db.get(_id) + let row = await db.get(_id) let _rev = ctx.request.body._rev || row._rev if (row.tableId !== ctx.params.tableId) { throw "Supplied tableId doesn't match the row's tableId" } - const table = await db.get(row.tableId) + const table = await sdk.tables.getTable(row.tableId) // update the row to include full relationships before deleting them row = await outputProcessing(table, row, { squash: false }) // now remove the relationships @@ -172,7 +173,7 @@ export async function destroy(ctx: UserCtx) { export async function bulkDestroy(ctx: UserCtx) { const db = context.getAppDB() const tableId = ctx.params.tableId - const table = await db.get(tableId) + const table = await sdk.tables.getTable(tableId) let { rows } = ctx.request.body // before carrying out any updates, make sure the rows are ready to be returned @@ -214,7 +215,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const rowId = ctx.params.rowId // need table to work out where links go in row let [table, row] = await Promise.all([ - db.get(tableId), + sdk.tables.getTable(tableId), utils.findRow(ctx, tableId, rowId), ]) // get the link docs diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 9f2c64c7b4..b5d520f771 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -8,8 +8,9 @@ import { FieldTypes, FormulaTypes } from "../../../constants" import { context } from "@budibase/backend-core" import { Table, Row } from "@budibase/types" import * as linkRows from "../../../db/linkedRows" -const { isEqual } = require("lodash") -const { cloneDeep } = require("lodash/fp") +import sdk from "../../../sdk" +import { isEqual } from "lodash" +import { cloneDeep } from "lodash/fp" /** * This function runs through a list of enriched rows, looks at the rows which @@ -148,7 +149,7 @@ export async function finaliseRow( await db.put(table) } catch (err: any) { if (err.status === 409) { - const updatedTable = await db.get(table._id) + const updatedTable = await sdk.tables.getTable(table._id) let response = processAutoColumn(null, updatedTable, row, { reprocessing: true, }) diff --git a/packages/server/src/api/controllers/screen.ts b/packages/server/src/api/controllers/screen.ts index ddfec91c0c..446fe2e5fa 100644 --- a/packages/server/src/api/controllers/screen.ts +++ b/packages/server/src/api/controllers/screen.ts @@ -7,7 +7,7 @@ import { roles, } from "@budibase/backend-core" import { updateAppPackage } from "./application" -import { Plugin, ScreenProps, BBContext } from "@budibase/types" +import { Plugin, ScreenProps, BBContext, Screen } from "@budibase/types" import { builderSocket } from "../../websockets" export async function fetch(ctx: BBContext) { @@ -64,7 +64,7 @@ export async function save(ctx: BBContext) { }) // Update the app metadata - const application = await db.get(DocumentType.APP_METADATA) + const application = await db.get(DocumentType.APP_METADATA) let usedPlugins = application.usedPlugins || [] requiredPlugins.forEach((plugin: Plugin) => { @@ -104,7 +104,7 @@ export async function save(ctx: BBContext) { export async function destroy(ctx: BBContext) { const db = context.getAppDB() const id = ctx.params.screenId - const screen = await db.get(id) + const screen = await db.get(id) await db.remove(id, ctx.params.screenRev) diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index fc17cd46e0..3ea07bcc20 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -1,7 +1,7 @@ require("svelte/register") import { join } from "../../../utilities/centralPath" -const uuid = require("uuid") +import uuid from "uuid" import { ObjectStoreBuckets } from "../../../constants" import { processString } from "@budibase/string-templates" import { @@ -16,6 +16,7 @@ import AWS from "aws-sdk" import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" +import { App } from "@budibase/types" const send = require("koa-send") @@ -110,7 +111,7 @@ export const serveApp = async function (ctx: any) { let db try { db = context.getAppDB({ skip_setup: true }) - const appInfo = await db.get(DocumentType.APP_METADATA) + const appInfo = await db.get(DocumentType.APP_METADATA) let appId = context.getAppId() if (!env.isJest()) { @@ -177,7 +178,7 @@ export const serveApp = async function (ctx: any) { export const serveBuilderPreview = async function (ctx: any) { const db = context.getAppDB({ skip_setup: true }) - const appInfo = await db.get(DocumentType.APP_METADATA) + const appInfo = await db.get(DocumentType.APP_METADATA) if (!env.isJest()) { let appId = context.getAppId() diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 9029d0468a..f008ff43fb 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -323,7 +323,7 @@ export async function save(ctx: UserCtx) { // Since tables are stored inside datasources, we need to notify clients // that the datasource definition changed - const updatedDatasource = await db.get(datasource._id) + const updatedDatasource = await sdk.datasources.get(datasource._id!) builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) return tableToSave @@ -354,7 +354,7 @@ export async function destroy(ctx: UserCtx) { // Since tables are stored inside datasources, we need to notify clients // that the datasource definition changed - const updatedDatasource = await db.get(datasource._id) + const updatedDatasource = await sdk.datasources.get(datasource._id!) builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) return tableToDelete diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index a688befcbe..39073caf7f 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -15,7 +15,7 @@ import { isEqual } from "lodash" import { cloneDeep } from "lodash/fp" import sdk from "../../../sdk" -function checkAutoColumns(table: Table, oldTable: Table) { +function checkAutoColumns(table: Table, oldTable?: Table) { if (!table.schema) { return table } @@ -46,7 +46,7 @@ export async function save(ctx: any) { // if the table obj had an _id then it will have been retrieved let oldTable if (ctx.request.body && ctx.request.body._id) { - oldTable = await db.get(ctx.request.body._id) + oldTable = await sdk.tables.getTable(ctx.request.body._id) } // check all types are correct @@ -70,8 +70,8 @@ export async function save(ctx: any) { if (oldTable && oldTable.schema) { for (let propKey of Object.keys(tableToSave.schema)) { let oldColumn = oldTable.schema[propKey] - if (oldColumn && oldColumn.type === "internal") { - oldColumn.type = "auto" + if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) { + oldColumn.type = FieldTypes.AUTO } } } @@ -138,7 +138,7 @@ export async function save(ctx: any) { export async function destroy(ctx: any) { const db = context.getAppDB() - const tableToDelete = await db.get(ctx.params.tableId) + const tableToDelete = await sdk.tables.getTable(ctx.params.tableId) // Delete all rows for that table const rowsData = await db.allDocs( @@ -160,7 +160,7 @@ export async function destroy(ctx: any) { }) // don't remove the table itself until very end - await db.remove(tableToDelete._id, tableToDelete._rev) + await db.remove(tableToDelete._id!, tableToDelete._rev) // remove table search index if (!env.isTest() || env.COUCH_DB_URL) { diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index 82a776e9a6..1a2a3850ce 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -36,8 +36,8 @@ export async function updateMetadata(ctx: UserCtx) { export async function destroyMetadata(ctx: UserCtx) { const db = context.getAppDB() try { - const dbUser = await db.get(ctx.params.id) - await db.remove(dbUser._id, dbUser._rev) + const dbUser = await sdk.users.get(ctx.params.id) + await db.remove(dbUser._id!, dbUser._rev) } catch (err) { // error just means the global user has no config in this app } @@ -60,7 +60,7 @@ export async function setFlag(ctx: UserCtx) { const db = context.getAppDB() let doc try { - doc = await db.get(flagDocId) + doc = await db.get(flagDocId) } catch (err) { doc = { _id: flagDocId } } diff --git a/packages/server/src/api/controllers/view/index.ts b/packages/server/src/api/controllers/view/index.ts index a087634292..99c4224c62 100644 --- a/packages/server/src/api/controllers/view/index.ts +++ b/packages/server/src/api/controllers/view/index.ts @@ -27,7 +27,8 @@ export async function save(ctx: Ctx) { const db = context.getAppDB() const { originalName, ...viewToSave } = ctx.request.body - const existingTable = await db.get(ctx.request.body.tableId) + const existingTable = await sdk.tables.getTable(ctx.request.body.tableId) + existingTable.views ??= {} const table = cloneDeep(existingTable) const groupByField: any = Object.values(table.schema).find( @@ -120,8 +121,8 @@ export async function destroy(ctx: Ctx) { const db = context.getAppDB() const viewName = decodeURIComponent(ctx.params.viewName) const view = await deleteView(viewName) - const table = await db.get(view.meta.tableId) - delete table.views[viewName] + const table = await sdk.tables.getTable(view.meta.tableId) + delete table.views![viewName] await db.put(table) await events.view.deleted(view) diff --git a/packages/server/src/api/controllers/view/utils.ts b/packages/server/src/api/controllers/view/utils.ts index ef7ce772f9..189c4ede51 100644 --- a/packages/server/src/api/controllers/view/utils.ts +++ b/packages/server/src/api/controllers/view/utils.ts @@ -13,7 +13,7 @@ import { Database } from "@budibase/types" export async function getView(viewName: string) { const db = context.getAppDB() if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") return designDoc.views[viewName] } else { // This is a table view, don't read the view from the DB @@ -22,7 +22,7 @@ export async function getView(viewName: string) { } try { - const viewDoc = await db.get(generateMemoryViewID(viewName)) + const viewDoc = await db.get(generateMemoryViewID(viewName)) return viewDoc.view } catch (err: any) { // Return null when PouchDB doesn't found the view @@ -39,7 +39,7 @@ export async function getViews() { const db = context.getAppDB() const response = [] if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") for (let name of Object.keys(designDoc.views)) { // Only return custom views, not built ins const viewNames = Object.values(ViewName) as string[] @@ -76,7 +76,7 @@ export async function saveView( ) { const db = context.getAppDB() if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") designDoc.views = { ...designDoc.views, [viewName]: viewTemplate, @@ -96,9 +96,9 @@ export async function saveView( tableId: viewTemplate.meta.tableId, } try { - const old = await db.get(id) + const old = await db.get(id) if (originalId) { - const originalDoc = await db.get(originalId) + const originalDoc = await db.get(originalId) await db.remove(originalDoc._id, originalDoc._rev) } if (old && old._rev) { @@ -114,14 +114,14 @@ export async function saveView( export async function deleteView(viewName: string) { const db = context.getAppDB() if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") const view = designDoc.views[viewName] delete designDoc.views[viewName] await db.put(designDoc) return view } else { const id = generateMemoryViewID(viewName) - const viewDoc = await db.get(id) + const viewDoc = await db.get(id) await db.remove(viewDoc._id, viewDoc._rev) return viewDoc.view } @@ -129,7 +129,7 @@ export async function deleteView(viewName: string) { export async function migrateToInMemoryView(db: Database, viewName: string) { // delete the view initially - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") // run the view back through the view builder to update it const view = viewBuilder(designDoc.views[viewName].meta) delete designDoc.views[viewName] @@ -138,15 +138,15 @@ export async function migrateToInMemoryView(db: Database, viewName: string) { } export async function migrateToDesignView(db: Database, viewName: string) { - let view = await db.get(generateMemoryViewID(viewName)) - const designDoc = await db.get("_design/database") + let view = await db.get(generateMemoryViewID(viewName)) + const designDoc = await db.get("_design/database") designDoc.views[viewName] = viewBuilder(view.view.meta) await db.put(designDoc) await db.remove(view._id, view._rev) } export async function getFromDesignDoc(db: Database, viewName: string) { - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") let view = designDoc.views[viewName] if (view == null) { throw { status: 404, message: "Unable to get view" } @@ -155,7 +155,7 @@ export async function getFromDesignDoc(db: Database, viewName: string) { } export async function getFromMemoryDoc(db: Database, viewName: string) { - let view = await db.get(generateMemoryViewID(viewName)) + let view = await db.get(generateMemoryViewID(viewName)) if (view) { view = view.view } else { diff --git a/packages/server/src/api/controllers/webhook.ts b/packages/server/src/api/controllers/webhook.ts index ee92c70628..d667134f4d 100644 --- a/packages/server/src/api/controllers/webhook.ts +++ b/packages/server/src/api/controllers/webhook.ts @@ -77,7 +77,7 @@ export async function trigger(ctx: BBContext) { if (webhook.bodySchema) { validate(ctx.request.body, webhook.bodySchema) } - const target = await db.get(webhook.action.target) + const target = await db.get(webhook.action.target) if (webhook.action.type === WebhookActionType.AUTOMATION) { // trigger with both the pure request and then expand it // incase the user has produced a schema to bind to diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index dc2f02c821..526197cda8 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -8,7 +8,12 @@ import { db as dbCore, context } from "@budibase/backend-core" import { getAutomationMetadataParams } from "../db/utils" import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" -import { Automation, AutomationJob, WebhookActionType } from "@budibase/types" +import { + Automation, + AutomationJob, + Webhook, + WebhookActionType, +} from "@budibase/types" import sdk from "../sdk" const REBOOT_CRON = "@reboot" @@ -206,13 +211,13 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) { try { let db = context.getAppDB() // need to get the webhook to get the rev - const webhook = await db.get(oldTrigger.webhookId) + const webhook = await db.get(oldTrigger.webhookId) // might be updating - reset the inputs to remove the URLs if (newTrigger) { delete newTrigger.webhookId newTrigger.inputs = {} } - await sdk.automations.webhook.destroy(webhook._id, webhook._rev) + await sdk.automations.webhook.destroy(webhook._id!, webhook._rev!) } catch (err) { // don't worry about not being able to delete, if it doesn't exist all good } diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts index 4a154b8f19..c3b3b324b5 100644 --- a/packages/server/src/db/linkedRows/LinkController.ts +++ b/packages/server/src/db/linkedRows/LinkController.ts @@ -182,7 +182,7 @@ class LinkController { }) // if 1:N, ensure that this ID is not already attached to another record - const linkedTable = await this._db.get(field.tableId) + const linkedTable = await this._db.get(field.tableId) const linkedSchema = linkedTable.schema[field.fieldName!] // We need to map the global users to metadata in each app for relationships @@ -311,7 +311,7 @@ class LinkController { }) ) // remove schema from other table - let linkedTable = await this._db.get(field.tableId) + let linkedTable = await this._db.get
(field.tableId) if (field.fieldName) { delete linkedTable.schema[field.fieldName] } @@ -337,7 +337,7 @@ class LinkController { // table for some reason let linkedTable try { - linkedTable = await this._db.get(field.tableId) + linkedTable = await this._db.get
(field.tableId) } catch (err) { /* istanbul ignore next */ continue @@ -416,7 +416,7 @@ class LinkController { const field = schema[fieldName] try { if (field.type === FieldTypes.LINK && field.fieldName) { - const linkedTable = await this._db.get(field.tableId) + const linkedTable = await this._db.get
(field.tableId) delete linkedTable.schema[field.fieldName] await this._db.put(linkedTable) } diff --git a/packages/server/src/db/views/staticViews.ts b/packages/server/src/db/views/staticViews.ts index 730e5b1e66..8952a4d6a1 100644 --- a/packages/server/src/db/views/staticViews.ts +++ b/packages/server/src/db/views/staticViews.ts @@ -22,7 +22,7 @@ const SCREEN_PREFIX = DocumentType.SCREEN + SEPARATOR */ export async function createLinkView() { const db = context.getAppDB() - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") const view = { map: function (doc: LinkDocument) { // everything in this must remain constant as its going to Pouch, no external variables @@ -58,7 +58,7 @@ export async function createLinkView() { export async function createRoutingView() { const db = context.getAppDB() - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") const view = { // if using variables in a map function need to inject them before use map: `function(doc) { @@ -79,7 +79,7 @@ export async function createRoutingView() { async function searchIndex(indexName: string, fnString: string) { const db = context.getAppDB() - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") designDoc.indexes = { [indexName]: { index: fnString, diff --git a/packages/server/src/middleware/builder.ts b/packages/server/src/middleware/builder.ts index b9481e60e2..881ec843a4 100644 --- a/packages/server/src/middleware/builder.ts +++ b/packages/server/src/middleware/builder.ts @@ -10,7 +10,7 @@ import { setDebounce, } from "../utilities/redis" import { db as dbCore, cache } from "@budibase/backend-core" -import { UserCtx, Database } from "@budibase/types" +import { UserCtx, Database, App } from "@budibase/types" const DEBOUNCE_TIME_SEC = 30 @@ -51,7 +51,7 @@ async function updateAppUpdatedAt(ctx: UserCtx) { } await dbCore.doWithDB(appId, async (db: Database) => { try { - const metadata = await db.get(DocumentType.APP_METADATA) + const metadata = await db.get(DocumentType.APP_METADATA) metadata.updatedAt = new Date().toISOString() metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!) diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index d8807251a1..6e1e6747e1 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -37,7 +37,7 @@ async function syncUsersToApp( let metadata try { - metadata = await db.get(metadataId) + metadata = await db.get(metadataId) } catch (err: any) { if (err.status !== 404) { throw err diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 430f90c159..171ec42042 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -62,7 +62,7 @@ export async function get( opts?: { enriched: boolean } ): Promise { const appDb = context.getAppDB() - const datasource = await appDb.get(datasourceId) + const datasource = await appDb.get(datasourceId) if (opts?.enriched) { return (await enrichDatasourceWithValues(datasource)).datasource } else { @@ -72,7 +72,7 @@ export async function get( export async function getWithEnvVars(datasourceId: string) { const appDb = context.getAppDB() - const datasource = await appDb.get(datasourceId) + const datasource = await appDb.get(datasourceId) return enrichDatasourceWithValues(datasource) } diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 70274de34a..fb0e35b650 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -11,7 +11,7 @@ import { } from "../../../../db/utils" import { getGlobalUsersFromMetadata } from "../../../../utilities/global" import { outputProcessing } from "../../../../utilities/rowProcessor" -import { Database, Row } from "@budibase/types" +import { Database, Row, Table } from "@budibase/types" import { cleanExportRows } from "../utils" import { Format, @@ -37,7 +37,6 @@ export async function search(options: SearchParams) { return { rows: await fetch(tableId) } } - const db = context.getAppDB() const { paginate, query } = options const params: InternalSearchParams = { @@ -53,7 +52,7 @@ export async function search(options: SearchParams) { let table if (params.sort && !params.sortType) { - table = await db.get(tableId) + table = await sdk.tables.getTable(tableId) const schema = table.schema const sortField = schema[params.sort] params.sortType = sortField.type === "number" ? "number" : "string" @@ -72,7 +71,7 @@ export async function search(options: SearchParams) { if (tableId === InternalTables.USER_METADATA) { response.rows = await getGlobalUsersFromMetadata(response.rows) } - table = table || (await db.get(tableId)) + table = table || (await sdk.tables.getTable(tableId)) response.rows = await outputProcessing(table, response.rows) } @@ -84,7 +83,7 @@ export async function exportRows( ): Promise { const { tableId, format, rowIds, columns, query } = options const db = context.getAppDB() - const table = await db.get(tableId) + const table = await sdk.tables.getTable(tableId) let result if (rowIds) { @@ -140,7 +139,7 @@ export async function exportRows( export async function fetch(tableId: string) { const db = context.getAppDB() - let table = await db.get(tableId) + let table = await sdk.tables.getTable(tableId) let rows = await getRawTableData(db, tableId) const result = await outputProcessing(table, rows) return result @@ -193,12 +192,13 @@ export async function fetchView( let rows if (!calculation) { response.rows = response.rows.map(row => row.doc) - let table + let table: Table try { - table = await db.get(viewInfo.meta.tableId) + table = await sdk.tables.getTable(viewInfo.meta.tableId) } catch (err) { /* istanbul ignore next */ table = { + name: "", schema: {}, } } diff --git a/packages/server/src/sdk/users/crud.ts b/packages/server/src/sdk/users/crud.ts new file mode 100644 index 0000000000..bb043f9976 --- /dev/null +++ b/packages/server/src/sdk/users/crud.ts @@ -0,0 +1,7 @@ +import { context } from "@budibase/backend-core" +import { User } from "@budibase/types" + +export function get(userId: string) { + const db = context.getAppDB() + return db.get(userId) +} diff --git a/packages/server/src/sdk/users/index.ts b/packages/server/src/sdk/users/index.ts index c03eab7c2a..234e12a529 100644 --- a/packages/server/src/sdk/users/index.ts +++ b/packages/server/src/sdk/users/index.ts @@ -1,7 +1,9 @@ import * as utils from "./utils" import * as sessions from "./sessions" +import * as crud from "./crud" export default { ...utils, + ...crud, sessions, } diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 9901ddffa6..135c533122 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -11,6 +11,7 @@ import { cloneDeep } from "lodash/fp" import { isSQL } from "../integrations/utils" import { interpolateSQL } from "../integrations/queries/sql" +import { Query } from "@budibase/types" class QueryRunner { datasource: any @@ -167,7 +168,7 @@ class QueryRunner { async runAnotherQuery(queryId: string, parameters: any) { const db = context.getAppDB() - const query = await db.get(queryId) + const query = await db.get(queryId) const datasource = await sdk.datasources.get(query.datasourceId, { enriched: true, }) diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index 8366794b18..c9869b4920 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -90,7 +90,7 @@ export async function getCachedSelf(ctx: UserCtx, appId: string) { export async function getRawGlobalUser(userId: string) { const db = tenancy.getGlobalDB() - return db.get(getGlobalIDFromUserMetadataID(userId)) + return db.get(getGlobalIDFromUserMetadataID(userId)) } export async function getGlobalUser(userId: string) { diff --git a/packages/server/src/utilities/index.ts b/packages/server/src/utilities/index.ts index 822ac68631..15e59f3b66 100644 --- a/packages/server/src/utilities/index.ts +++ b/packages/server/src/utilities/index.ts @@ -41,7 +41,7 @@ export async function updateEntityMetadata( // read it to see if it exists, we'll overwrite it no matter what let rev, metadata: Document try { - const oldMetadata = await db.get(id) + const oldMetadata = await db.get(id) rev = oldMetadata._rev metadata = updateFn(oldMetadata) } catch (err) { @@ -75,7 +75,7 @@ export async function deleteEntityMetadata(type: string, entityId: string) { const id = generateMetadataID(type, entityId) let rev try { - const metadata = await db.get(id) + const metadata = await db.get(id) if (metadata) { rev = metadata._rev } diff --git a/packages/worker/src/api/controllers/global/email.ts b/packages/worker/src/api/controllers/global/email.ts index e352ef0b87..837f79035a 100644 --- a/packages/worker/src/api/controllers/global/email.ts +++ b/packages/worker/src/api/controllers/global/email.ts @@ -1,6 +1,6 @@ import { sendEmail as sendEmailFn } from "../../../utilities/email" import { tenancy } from "@budibase/backend-core" -import { BBContext } from "@budibase/types" +import { BBContext, User } from "@budibase/types" export async function sendEmail(ctx: BBContext) { let { @@ -16,10 +16,10 @@ export async function sendEmail(ctx: BBContext) { automation, invite, } = ctx.request.body - let user + let user: any if (userId) { const db = tenancy.getGlobalDB() - user = await db.get(userId) + user = await db.get(userId) } const response = await sendEmailFn(email, purpose, { workspaceId, diff --git a/packages/worker/src/api/controllers/global/roles.ts b/packages/worker/src/api/controllers/global/roles.ts index e0cb52e1b0..572c3328b6 100644 --- a/packages/worker/src/api/controllers/global/roles.ts +++ b/packages/worker/src/api/controllers/global/roles.ts @@ -35,7 +35,7 @@ export async function find(ctx: BBContext) { const appId = ctx.params.appId await context.doInAppContext(dbCore.getDevAppID(appId), async () => { const db = context.getAppDB() - const app = await db.get(dbCore.DocumentType.APP_METADATA) + const app = await db.get(dbCore.DocumentType.APP_METADATA) ctx.body = { roles: await roles.getAllRoles(), name: app.name, diff --git a/packages/worker/src/api/controllers/global/self.ts b/packages/worker/src/api/controllers/global/self.ts index d0232bee60..2ca5b6919c 100644 --- a/packages/worker/src/api/controllers/global/self.ts +++ b/packages/worker/src/api/controllers/global/self.ts @@ -44,7 +44,7 @@ export async function generateAPIKey(ctx: any) { const id = dbCore.generateDevInfoID(userId) let devInfo try { - devInfo = await db.get(id) + devInfo = await db.get(id) } catch (err) { devInfo = { _id: id, userId } } diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 320f7be01a..39a1279925 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -412,7 +412,7 @@ export const inviteAccept = async ( const saved = await userSdk.save(request) const db = tenancy.getGlobalDB() - const user = await db.get(saved._id) + const user = await db.get(saved._id) await events.user.inviteAccepted(user) return saved }) From 58c59eba5e7d653848d168c05f69bbc958fecd0a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 18 Jul 2023 12:00:02 +0200 Subject: [PATCH 28/30] Remove unused db usages --- packages/server/src/api/controllers/datasource.ts | 2 -- packages/server/src/api/controllers/table/internal.ts | 1 - packages/server/src/api/controllers/table/utils.ts | 10 ++-------- packages/server/src/automations/utils.ts | 2 +- packages/server/src/integrations/tests/mysql.spec.ts | 2 +- packages/server/src/sdk/app/tables/index.ts | 1 - 6 files changed, 4 insertions(+), 14 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 8ef2e8789a..00ae2ea1d7 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -432,7 +432,6 @@ export async function destroy(ctx: UserCtx) { } export async function find(ctx: UserCtx) { - const db = context.getAppDB() const datasource = await sdk.datasources.get(ctx.params.datasourceId) ctx.body = await sdk.datasources.removeSecretSingle(datasource) } @@ -448,7 +447,6 @@ export async function query(ctx: UserCtx) { } export async function getExternalSchema(ctx: UserCtx) { - const db = context.getAppDB() const datasource = await sdk.datasources.get(ctx.params.datasourceId) const enrichedDatasource = await getAndMergeDatasource(datasource) const connector = await getConnector(enrichedDatasource) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 39073caf7f..eab77e9169 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -184,7 +184,6 @@ export async function destroy(ctx: any) { } export async function bulkImport(ctx: any) { - const db = context.getAppDB() const table = await sdk.tables.getTable(ctx.params.tableId) const { rows, identifierFields } = ctx.request.body await handleDataImport(ctx.user, table, rows, identifierFields) diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 923608e9a2..54ee1b91ca 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -20,16 +20,10 @@ import viewTemplate from "../view/viewBuilder" import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" import { events, context } from "@budibase/backend-core" -import { - ContextUser, - Database, - Datasource, - SourceName, - Table, -} from "@budibase/types" +import { ContextUser, Datasource, SourceName, Table } from "@budibase/types" export async function clearColumns(table: any, columnNames: any) { - const db: Database = context.getAppDB() + const db = context.getAppDB() const rows = await db.allDocs( getRowParams(table._id, null, { include_docs: true, diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 526197cda8..194eff5b32 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -209,7 +209,7 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) { oldTrigger.webhookId ) { try { - let db = context.getAppDB() + const db = context.getAppDB() // need to get the webhook to get the rev const webhook = await db.get(oldTrigger.webhookId) // might be updating - reset the inputs to remove the URLs diff --git a/packages/server/src/integrations/tests/mysql.spec.ts b/packages/server/src/integrations/tests/mysql.spec.ts index e4a496d80d..c3c3b8343e 100644 --- a/packages/server/src/integrations/tests/mysql.spec.ts +++ b/packages/server/src/integrations/tests/mysql.spec.ts @@ -102,7 +102,7 @@ describe("MySQL Integration", () => { ) }) - it("parses strings matching a valid date format", async () => { + it.skip("parses strings matching a valid date format", async () => { const sql = "select * from users;" await config.integration.read({ sql, diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index c7de1b327c..65010aeaa4 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -28,7 +28,6 @@ async function getAllInternalTables(db?: Database): Promise { async function getAllExternalTables( datasourceId: any ): Promise> { - const db = context.getAppDB() const datasource = await datasources.get(datasourceId, { enriched: true }) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." From fcedb02ed190c8d55517d54991cc279f3dbcc4c5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 18 Jul 2023 12:18:48 +0200 Subject: [PATCH 29/30] Update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 1929d93660..9c564edb37 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 1929d93660e95f4b190d1936d0b3c65c4ad6a1f5 +Subproject commit 9c564edb37cb619cb5971e10c4317fa6e7c5bb00 From fc592342c3cdec5950b31f840064da5c0d8e61fb Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 18 Jul 2023 10:24:51 +0000 Subject: [PATCH 30/30] Bump version to 2.8.12-alpha.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index dd403724b2..992ee8a5e2 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.8.12-alpha.2", + "version": "2.8.12-alpha.3", "npmClient": "yarn", "packages": [ "packages/*"