From 3de8c531667ab04725e3de114100fefde66bc368 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Oct 2023 11:57:11 +0100 Subject: [PATCH 1/8] Adding a mapping layer to search queries so that we can map search inputs based on the table schema if desired - primarily for the user column. --- .../controls/FilterEditor/FilterDrawer.svelte | 2 +- .../src/sdk/app/rows/search/external.ts | 5 +- .../src/sdk/app/rows/search/internal.ts | 6 +-- .../sdk/app/rows/search/tests/utils.spec.ts | 49 +++++++++++++++++ .../server/src/sdk/app/rows/search/utils.ts | 53 +++++++++++++++++++ packages/shared-core/src/filters.ts | 1 - 6 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 packages/server/src/sdk/app/rows/search/tests/utils.spec.ts create mode 100644 packages/server/src/sdk/app/rows/search/utils.ts diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index 822442fcea..c03085e66f 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -191,7 +191,7 @@ } const getValidOperatorsForType = filter => { - if (!filter) { + if (!filter.field) { return [] } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index f908be0b3c..8dd141f8ef 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -16,6 +16,7 @@ import { cleanExportRows } from "../utils" import { utils } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "../search" import { HTTPError, db } from "@budibase/backend-core" +import { searchInputMapping } from "./utils" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" @@ -50,7 +51,10 @@ export async function search(options: SearchParams) { [params.sort]: { direction }, } } + try { + const table = await sdk.tables.getTable(tableId) + options = searchInputMapping(table, options) let rows = (await handleRequest(Operation.READ, tableId, { filters: query, sort, @@ -76,7 +80,6 @@ export async function search(options: SearchParams) { rows = rows.map((r: any) => pick(r, fields)) } - const table = await sdk.tables.getTable(tableId) rows = await outputProcessing(table, rows, { preserveLinks: true }) // need wrapper object for bookmarks etc when paginating diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 4cdeca87f6..d78c0213b3 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -29,6 +29,7 @@ import { } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" import { ExportRowsParams, ExportRowsResult } from "../search" +import { searchInputMapping } from "./utils" import pick from "lodash/pick" export async function search(options: SearchParams) { @@ -47,9 +48,9 @@ export async function search(options: SearchParams) { disableEscaping: options.disableEscaping, } - let table + let table = await sdk.tables.getTable(tableId) + options = searchInputMapping(table, options) if (params.sort && !params.sortType) { - table = await sdk.tables.getTable(tableId) const schema = table.schema const sortField = schema[params.sort] params.sortType = sortField.type === "number" ? "number" : "string" @@ -68,7 +69,6 @@ export async function search(options: SearchParams) { if (tableId === InternalTables.USER_METADATA) { response.rows = await getGlobalUsersFromMetadata(response.rows) } - table = table || (await sdk.tables.getTable(tableId)) if (options.fields) { const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS] diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts new file mode 100644 index 0000000000..01db0d421b --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -0,0 +1,49 @@ +import { searchInputMapping } from "../utils" +import { db as dbCore } from "@budibase/backend-core" +import { + FieldType, + FieldTypeSubtypes, + Table, + SearchParams, +} from "@budibase/types" + +const tableId = "ta_a" +const tableWithUserCol: Table = { + _id: tableId, + name: "table", + schema: { + user: { + name: "user", + type: FieldType.BB_REFERENCE, + subtype: FieldTypeSubtypes.BB_REFERENCE.USER, + }, + }, +} + +describe("searchInputMapping", () => { + it("should be able to map ro_ to global user IDs", () => { + const globalUserId = dbCore.generateGlobalUserID() + const userMedataId = dbCore.generateUserMetadataID(globalUserId) + const params: SearchParams = { + tableId, + query: { + equal: { + "1:user": userMedataId, + }, + }, + } + const output = searchInputMapping(tableWithUserCol, params) + expect(output.query.equal!["1:user"]).toBe(globalUserId) + }) + + it("shouldn't change any other input", () => { + const params: SearchParams = { + tableId, + query: { + equal: { + "1:user": "test@test.com", + }, + }, + } + }) +}) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts new file mode 100644 index 0000000000..c86df9bc0d --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -0,0 +1,53 @@ +import { + FieldType, + FieldTypeSubtypes, + SearchParams, + Table, + DocumentType, + SEPARATOR, +} from "@budibase/types" +import { db as dbCore } from "@budibase/backend-core" + +function findColumnInQueries( + column: string, + options: SearchParams, + callback: (filter: T) => T +) { + for (let filterBlock of Object.values(options.query)) { + if (typeof filterBlock !== "object") { + continue + } + for (let [key, filter] of Object.entries(filterBlock)) { + if (key.endsWith(column)) { + filterBlock[key] = callback(filter) + } + } + } +} + +function userColumnMapping(column: string, options: SearchParams) { + findColumnInQueries(column, options, (filterValue: any): string => { + if (typeof filterValue !== "string") { + return filterValue + } + const rowPrefix = DocumentType.ROW + SEPARATOR + // TODO: at some point in future might want to handle mapping emails to IDs + if (filterValue.startsWith(rowPrefix)) { + return dbCore.getGlobalIDFromUserMetadataID(filterValue) + } + return filterValue + }) +} + +export function searchInputMapping(table: Table, options: SearchParams) { + for (let [key, column] of Object.entries(table.schema)) { + switch (column.type) { + case FieldType.BB_REFERENCE: + if (column.subtype === FieldTypeSubtypes.BB_REFERENCE.USER) { + userColumnMapping(key, options) + } + break + } + } + return options +} diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 6ab8ce623e..e443f35dbe 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -14,7 +14,6 @@ const HBS_REGEX = /{{([^{].*?)}}/g /** * Returns the valid operator options for a certain data type - * @param type the data type */ export const getValidOperatorsForType = ( type: FieldType, From dc50515bcc16ed0935fd8bd4a1c04feb729758d9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Oct 2023 12:16:37 +0100 Subject: [PATCH 2/8] Adding negative test case and fixing build issue. --- packages/server/src/sdk/app/rows/search/tests/utils.spec.ts | 5 ++++- packages/server/src/sdk/app/rows/search/utils.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index 01db0d421b..a1cb4fe48d 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -37,13 +37,16 @@ describe("searchInputMapping", () => { }) it("shouldn't change any other input", () => { + const email = "test@test.com" const params: SearchParams = { tableId, query: { equal: { - "1:user": "test@test.com", + "1:user": email, }, }, } + const output = searchInputMapping(tableWithUserCol, params) + expect(output.query.equal!["1:user"]).toBe(email) }) }) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index c86df9bc0d..5afa13c53d 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -11,7 +11,7 @@ import { db as dbCore } from "@budibase/backend-core" function findColumnInQueries( column: string, options: SearchParams, - callback: (filter: T) => T + callback: (filter: any) => any ) { for (let filterBlock of Object.values(options.query)) { if (typeof filterBlock !== "object") { From 6e6c5bc776f0352ab5e81939ba57be64228dd6ae Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Oct 2023 12:31:52 +0100 Subject: [PATCH 3/8] Handle arrays and fix issue brought up by REST testcase. --- .../sdk/app/rows/search/tests/utils.spec.ts | 29 ++++++++++++++++-- .../server/src/sdk/app/rows/search/utils.ts | 30 +++++++++++++++---- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index a1cb4fe48d..08d1f1b1cb 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -21,9 +21,10 @@ const tableWithUserCol: Table = { } describe("searchInputMapping", () => { + const globalUserId = dbCore.generateGlobalUserID() + const userMedataId = dbCore.generateUserMetadataID(globalUserId) + it("should be able to map ro_ to global user IDs", () => { - const globalUserId = dbCore.generateGlobalUserID() - const userMedataId = dbCore.generateUserMetadataID(globalUserId) const params: SearchParams = { tableId, query: { @@ -36,6 +37,22 @@ describe("searchInputMapping", () => { expect(output.query.equal!["1:user"]).toBe(globalUserId) }) + it("should handle array of user IDs", () => { + const params: SearchParams = { + tableId, + query: { + oneOf: { + "1:user": [userMedataId, globalUserId], + }, + }, + } + const output = searchInputMapping(tableWithUserCol, params) + expect(output.query.oneOf!["1:user"]).toStrictEqual([ + globalUserId, + globalUserId, + ]) + }) + it("shouldn't change any other input", () => { const email = "test@test.com" const params: SearchParams = { @@ -49,4 +66,12 @@ describe("searchInputMapping", () => { const output = searchInputMapping(tableWithUserCol, params) expect(output.query.equal!["1:user"]).toBe(email) }) + + it("shouldn't error if no query supplied", () => { + const params: any = { + tableId, + } + const output = searchInputMapping(tableWithUserCol, params) + expect(output.query).toBeUndefined() + }) }) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 5afa13c53d..f475296333 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -13,6 +13,9 @@ function findColumnInQueries( options: SearchParams, callback: (filter: any) => any ) { + if (!options.query) { + return + } for (let filterBlock of Object.values(options.query)) { if (typeof filterBlock !== "object") { continue @@ -27,15 +30,30 @@ function findColumnInQueries( function userColumnMapping(column: string, options: SearchParams) { findColumnInQueries(column, options, (filterValue: any): string => { - if (typeof filterValue !== "string") { + const isArray = Array.isArray(filterValue), + isString = typeof filterValue === "string" + if (!isString && !isArray) { return filterValue } - const rowPrefix = DocumentType.ROW + SEPARATOR - // TODO: at some point in future might want to handle mapping emails to IDs - if (filterValue.startsWith(rowPrefix)) { - return dbCore.getGlobalIDFromUserMetadataID(filterValue) + const processString = (input: string) => { + const rowPrefix = DocumentType.ROW + SEPARATOR + if (input.startsWith(rowPrefix)) { + return dbCore.getGlobalIDFromUserMetadataID(input) + } else { + return input + } + } + if (isArray) { + return filterValue.map(el => { + if (typeof el === "string") { + return processString(el) + } else { + return el + } + }) + } else { + return processString(filterValue) } - return filterValue }) } From 18cca671d49fcff4f84afe8b8d77c80e1178b7d5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Oct 2023 12:56:07 +0100 Subject: [PATCH 4/8] PR comments. --- .../design/settings/controls/FilterEditor/FilterDrawer.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index c03085e66f..0fd30b82bb 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -191,7 +191,7 @@ } const getValidOperatorsForType = filter => { - if (!filter.field) { + if (!filter?.field) { return [] } From 017b522a3f6ac014c9188498dfcbcb8f9e0537b9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Oct 2023 12:56:55 +0100 Subject: [PATCH 5/8] Comment to explain function. --- packages/server/src/sdk/app/rows/search/utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index f475296333..27964d2ad2 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -57,6 +57,8 @@ function userColumnMapping(column: string, options: SearchParams) { }) } +// maps through the search parameters to check if any of the inputs are invalid +// based on the table schema, converts them to something that is valid. export function searchInputMapping(table: Table, options: SearchParams) { for (let [key, column] of Object.entries(table.schema)) { switch (column.type) { From 7acc164e12a98c29d46df6d40126c33bdd7272e3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Oct 2023 13:28:27 +0100 Subject: [PATCH 6/8] Fix build. --- packages/server/src/sdk/app/rows/search/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 27964d2ad2..1b4f60f7da 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -29,7 +29,7 @@ function findColumnInQueries( } function userColumnMapping(column: string, options: SearchParams) { - findColumnInQueries(column, options, (filterValue: any): string => { + findColumnInQueries(column, options, (filterValue: any): any => { const isArray = Array.isArray(filterValue), isString = typeof filterValue === "string" if (!isString && !isArray) { From bc17bc43fa7fb8716a8c7ee936dd6bfb2f42847d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Oct 2023 13:32:12 +0100 Subject: [PATCH 7/8] Fixing issue brought up by some unit tests. --- packages/server/src/sdk/app/rows/search/utils.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 1b4f60f7da..14f7907e4f 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -60,6 +60,9 @@ function userColumnMapping(column: string, options: SearchParams) { // maps through the search parameters to check if any of the inputs are invalid // based on the table schema, converts them to something that is valid. export function searchInputMapping(table: Table, options: SearchParams) { + if (!table?.schema) { + return options + } for (let [key, column] of Object.entries(table.schema)) { switch (column.type) { case FieldType.BB_REFERENCE: From 575693d69a1f3f404bafa991b5e5bd194037818d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Oct 2023 14:07:12 +0100 Subject: [PATCH 8/8] Updating pro. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 7040ae5282..044bec6447 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 7040ae5282cc23d7ae56ac1be8a369d1c32aab2f +Subproject commit 044bec6447066b215932d6726c437e7ec5a9e42e