From 74287370dff0508c3fcee59455cbfc5a587b1f11 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 7 Aug 2023 11:18:36 +0300 Subject: [PATCH 1/5] Move filter types to types --- packages/shared-core/src/filters.ts | 78 +++++----------------- packages/types/src/api/web/index.ts | 1 + packages/types/src/api/web/searchFilter.ts | 49 ++++++++++++++ 3 files changed, 66 insertions(+), 62 deletions(-) create mode 100644 packages/types/src/api/web/searchFilter.ts diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 54e0df92df..6602762160 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -1,4 +1,12 @@ -import { Datasource, FieldType, SortDirection, SortType } from "@budibase/types" +import { + Datasource, + FieldType, + SearchFilter, + SearchQuery, + SearchQueryFields, + SortDirection, + SortType, +} from "@budibase/types" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" import { deepGet } from "./helpers" @@ -73,13 +81,13 @@ export const NoEmptyFilterStrings = [ OperatorOptions.NotEquals.value, OperatorOptions.Contains.value, OperatorOptions.NotContains.value, -] as (keyof QueryFields)[] +] as (keyof SearchQueryFields)[] /** * Removes any fields that contain empty strings that would cause inconsistent * behaviour with how backend tables are filtered (no value means no filter). */ -const cleanupQuery = (query: Query) => { +const cleanupQuery = (query: SearchQuery) => { if (!query) { return query } @@ -110,66 +118,12 @@ const removeKeyNumbering = (key: string) => { } } -type Filter = { - operator: keyof Query - field: string - type: any - value: any - externalType: keyof typeof SqlNumberTypeRangeMap -} - -type Query = QueryFields & QueryConfig -type QueryFields = { - string?: { - [key: string]: string - } - fuzzy?: { - [key: string]: string - } - range?: { - [key: string]: { - high: number | string - low: number | string - } - } - equal?: { - [key: string]: any - } - notEqual?: { - [key: string]: any - } - empty?: { - [key: string]: any - } - notEmpty?: { - [key: string]: any - } - oneOf?: { - [key: string]: any[] - } - contains?: { - [key: string]: any[] - } - notContains?: { - [key: string]: any[] - } - containsAny?: { - [key: string]: any[] - } -} - -type QueryConfig = { - allOr?: boolean -} - -type QueryFieldsType = keyof QueryFields - /** * Builds a lucene JSON query from the filter structure generated in the builder * @param filter the builder filter structure */ -export const buildLuceneQuery = (filter: Filter[]) => { - let query: Query = { +export const buildLuceneQuery = (filter: SearchFilter[]) => { + let query: SearchQuery = { string: {}, fuzzy: {}, range: {}, @@ -275,7 +229,7 @@ export const buildLuceneQuery = (filter: Filter[]) => { * @param docs the data * @param query the JSON lucene query */ -export const runLuceneQuery = (docs: any[], query?: Query) => { +export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { if (!docs || !Array.isArray(docs)) { return [] } @@ -289,7 +243,7 @@ export const runLuceneQuery = (docs: any[], query?: Query) => { // Iterates over a set of filters and evaluates a fail function against a doc const match = ( - type: QueryFieldsType, + type: keyof SearchQueryFields, failFn: (docValue: any, testValue: any) => boolean ) => (doc: any) => { @@ -456,7 +410,7 @@ export const luceneLimit = (docs: any[], limit: string) => { return docs.slice(0, numLimit) } -export const hasFilters = (query?: Query) => { +export const hasFilters = (query?: SearchQuery) => { if (!query) { return false } diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 0e0527eb7f..cba1e04f9a 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -8,3 +8,4 @@ export * from "./system" export * from "./app" export * from "./global" export * from "./pagination" +export * from "./searchFilter" diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts new file mode 100644 index 0000000000..4dc00453d3 --- /dev/null +++ b/packages/types/src/api/web/searchFilter.ts @@ -0,0 +1,49 @@ +export type SearchFilter = { + operator: keyof SearchQuery + field: string + type: any + value: any + externalType: "integer" | "int" | "smallint" | "mediumint" +} + +export type SearchQuery = { + allOr?: boolean + string?: { + [key: string]: string + } + fuzzy?: { + [key: string]: string + } + range?: { + [key: string]: { + high: number | string + low: number | string + } + } + equal?: { + [key: string]: any + } + notEqual?: { + [key: string]: any + } + empty?: { + [key: string]: any + } + notEmpty?: { + [key: string]: any + } + oneOf?: { + [key: string]: any[] + } + contains?: { + [key: string]: any[] + } + notContains?: { + [key: string]: any[] + } + containsAny?: { + [key: string]: any[] + } +} + +export type SearchQueryFields = Omit From 77568af7048cfdd569b9afe30dc83195d43fd21b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 7 Aug 2023 13:14:18 +0300 Subject: [PATCH 2/5] Change filter query to store the expected shape --- packages/server/src/api/controllers/row/views.ts | 4 +++- packages/server/src/api/routes/tests/row.spec.ts | 9 ++++++++- packages/shared-core/src/filters.ts | 8 ++++++-- packages/types/src/api/web/searchFilter.ts | 6 ++++-- packages/types/src/documents/app/view.ts | 7 +++---- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index c38b7fe56e..9d6e38b1ec 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -6,6 +6,7 @@ import { SortType, ViewV2, } from "@budibase/types" +import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" export async function searchView(ctx: UserCtx) { @@ -29,11 +30,12 @@ export async function searchView(ctx: UserCtx) { undefined ctx.status = 200 + const query = dataFilters.buildLuceneQuery(view.query || []) const result = await quotas.addQuery( () => sdk.rows.search({ tableId: view.tableId, - query: view.query || {}, + query, fields: viewFields, ...getSortOptions(ctx, view), }), diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index b986ffb945..230aaa54cf 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -869,7 +869,14 @@ describe("/rows", () => { ) const createViewResponse = await config.api.viewV2.create({ - query: { equal: { age: 40 } }, + query: [ + { + operator: "equal", + field: "age", + value: "40", + type: FieldType.NUMBER, + }, + ], }) const response = await config.api.viewV2.search(createViewResponse.id) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 6602762160..8739db1b40 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -181,9 +181,13 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { } if (operator.startsWith("range") && query.range) { const minint = - SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER + SqlNumberTypeRangeMap[ + externalType as keyof typeof SqlNumberTypeRangeMap + ]?.min || Number.MIN_SAFE_INTEGER const maxint = - SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER + SqlNumberTypeRangeMap[ + externalType as keyof typeof SqlNumberTypeRangeMap + ]?.max || Number.MAX_SAFE_INTEGER if (!query.range[field]) { query.range[field] = { low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 4dc00453d3..1b5948e50c 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,9 +1,11 @@ +import { FieldType } from "../../documents" + export type SearchFilter = { operator: keyof SearchQuery field: string - type: any + type?: FieldType value: any - externalType: "integer" | "int" | "smallint" | "mediumint" + externalType?: string } export type SearchQuery = { diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 3fe8b4a500..204b995337 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,6 +1,5 @@ -import { SortOrder, SortType } from "../../api" -import { SearchFilters } from "../../sdk" -import { TableSchema, UIFieldMetadata } from "./table" +import { SearchFilter, SortOrder, SortType } from "../../api" +import { UIFieldMetadata } from "./table" export interface View { name: string @@ -20,7 +19,7 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilters + query?: SearchFilter[] sort?: { field: string order?: SortOrder From c6fba3a82787483229b78f661478f2f2383595ae Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 7 Aug 2023 14:16:23 +0300 Subject: [PATCH 3/5] Fix types --- .../server/src/api/routes/tests/viewV2.spec.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index fc97e657c5..e30bc2c0b1 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -62,7 +62,7 @@ describe("/v2/views", () => { name: generator.name(), tableId: config.table!._id!, primaryDisplay: generator.word(), - query: { allOr: false, equal: { field: "value" } }, + query: [{ operator: "equal", field: "field", value: "value" }], sort: { field: "fieldToSort", order: SortOrder.DESCENDING, @@ -190,7 +190,7 @@ describe("/v2/views", () => { const tableId = config.table!._id! await config.api.viewV2.update({ ...view, - query: { equal: { newField: "thatValue" } }, + query: [{ operator: "equal", field: "newField", value: "thatValue" }], }) expect(await config.api.table.get(tableId)).toEqual({ @@ -198,7 +198,9 @@ describe("/v2/views", () => { views: { [view.name]: { ...view, - query: { equal: { newField: "thatValue" } }, + query: [ + { operator: "equal", field: "newField", value: "thatValue" }, + ], schema: expect.anything(), }, }, @@ -216,7 +218,13 @@ describe("/v2/views", () => { tableId, name: view.name, primaryDisplay: generator.word(), - query: { equal: { [generator.word()]: generator.word() } }, + query: [ + { + operator: "equal", + field: generator.word(), + value: generator.word(), + }, + ], sort: { field: generator.word(), order: SortOrder.DESCENDING, @@ -285,7 +293,7 @@ describe("/v2/views", () => { { ...view, tableId: generator.guid(), - query: { equal: { newField: "thatValue" } }, + query: [{ operator: "equal", field: "newField", value: "thatValue" }], }, { expectStatus: 404 } ) From 4d2fef6b199aa0b4ecdb9cfc70d26deb1c6ebc6d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 7 Aug 2023 15:33:50 +0300 Subject: [PATCH 4/5] Fix types after merge --- packages/server/src/api/routes/tests/row.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index b9f584706e..3284d25de8 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1072,7 +1072,7 @@ describe("/rows", () => { ) const createViewResponse = await config.api.viewV2.create({ - query: { equal: { age: 40 } }, + query: [{ operator: "equal", field: "age", value: 40 }], }) const response = await config.api.viewV2.search(createViewResponse.id) From 206be7e8a1daf7f3174ce9711e3427c6e1d29f18 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 7 Aug 2023 15:36:56 +0300 Subject: [PATCH 5/5] Update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index cf3bef2aad..ecee8071eb 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit cf3bef2aad9c739111b306fd0712397adc363f81 +Subproject commit ecee8071ebe0f98a5bb19646954e373264be210d