From 6e660151bdef6cd5c9130e09a695bc47b80d4b0c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Sep 2024 18:06:47 +0100 Subject: [PATCH 01/10] backport of V3 backend changes for search filters on view, giving this the correct type to support conditionals. --- .../server/src/api/controllers/row/views.ts | 48 ++-- .../src/api/controllers/view/viewsV2.ts | 4 +- packages/shared-core/src/filters.ts | 208 +++++++++++++++++- packages/shared-core/src/utils.ts | 110 ++++++++- packages/types/src/api/web/app/view.ts | 5 +- packages/types/src/api/web/searchFilter.ts | 16 +- packages/types/src/documents/app/view.ts | 8 +- packages/types/src/sdk/search.ts | 5 + 8 files changed, 372 insertions(+), 32 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 68958da8e7..398121f49b 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -5,6 +5,9 @@ import { SearchViewRowRequest, SearchFilterKey, LogicalOperator, + RequiredKeys, + RowSearchParams, + LegacyFilter, } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" @@ -17,7 +20,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view = await sdk.views.get(viewId) + const view: ViewV2 = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } @@ -25,23 +28,35 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } + const viewFields = Object.entries(view.schema || {}) + .filter(([_, value]) => value.visible) + .map(([key]) => key) const { body } = ctx.request + const sqsEnabled = await features.flags.isEnabled("SQS") + const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled + // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query = dataFilters.buildQuery(view.query || []) + let query = dataFilters.buildQueryLegacy(view.query) + + delete query?.onEmptyFilter + if (body.query) { // Delete extraneous search params that cannot be overridden delete body.query.onEmptyFilter - if ( - !isExternalTableID(view.tableId) && - !(await features.flags.isEnabled("SQS")) - ) { + if (!supportsLogicalOperators) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored entirely + let queryFilters: LegacyFilter[] = Array.isArray(view.query) + ? view.query + : [] + // Extract existing fields const existingFields = - view.query + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] @@ -49,15 +64,16 @@ export async function searchView( Object.keys(body.query).forEach(key => { const operator = key as Exclude Object.keys(body.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { + if (query && !existingFields.includes(db.removeKeyNumbering(field))) { query[operator]![field] = body.query[operator]![field] } }) }) } else { + const conditions = query ? [query] : [] query = { $and: { - conditions: [query, body.query], + conditions: [...conditions, body.query], }, } } @@ -65,25 +81,29 @@ export async function searchView( await context.ensureSnippetContext(true) - const enrichedQuery = await enrichSearchContext(query, { + const enrichedQuery = await enrichSearchContext(query || {}, { user: sdk.users.getUserContextBindings(ctx.user), }) - const result = await sdk.rows.search({ - viewId: view.id, + const searchOptions: RequiredKeys & + RequiredKeys< + Pick + > = { tableId: view.tableId, + viewId: view.id, query: enrichedQuery, + fields: viewFields, ...getSortOptions(body, view), limit: body.limit, bookmark: body.bookmark, paginate: body.paginate, countRows: body.countRows, - }) + } + const result = await sdk.rows.search(searchOptions) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } - function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { if (request.sort) { return { diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 7f6f638541..3df7172de2 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,7 +99,7 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version"> = { + const parsedView: Omit, "id" | "version" | "queryUI"> = { name: view.name, tableId: view.tableId, query: view.query, @@ -132,7 +132,7 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys = { + const parsedView: RequiredKeys> = { id: view.id, name: view.name, version: view.version, diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index ef0500b01a..18ce4b6ed7 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -3,7 +3,7 @@ import { BBReferenceFieldSubType, FieldType, FormulaType, - SearchFilter, + LegacyFilter, SearchFilters, SearchQueryFields, ArrayOperator, @@ -19,9 +19,12 @@ import { RangeOperator, LogicalOperator, isLogicalSearchOperator, + SearchFilterGroup, + FilterGroupLogicalOperator, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" +import { processSearchFilters } from "./utils" import { deepGet, schema } from "./helpers" import { isPlainObject, isEmpty } from "lodash" import { decodeNonAscii } from "./helpers/schema" @@ -124,7 +127,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters?.[logical]) { + if (filters[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) @@ -304,10 +307,143 @@ export class ColumnSplitter { } /** - * Builds a JSON query from the filter structure generated in the builder + * Builds a JSON query from the filter a SearchFilter definition * @param filter the builder filter structure */ -export const buildQuery = (filter: SearchFilter[]) => { + +const buildCondition = (expression: LegacyFilter) => { + // Filter body + let query: SearchFilters = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + oneOf: {}, + containsAny: {}, + } + let { operator, field, type, value, externalType, onEmptyFilter } = expression + + if (!operator || !field) { + return + } + + const queryOperator = operator as SearchFilterOperator + const isHbs = + typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 + // Parse all values into correct types + if (operator === "allOr") { + query.allOr = true + return + } + if (onEmptyFilter) { + query.onEmptyFilter = onEmptyFilter + return + } + + // Default the value for noValue fields to ensure they are correctly added + // to the final query + if (queryOperator === "empty" || queryOperator === "notEmpty") { + value = null + } + + if ( + type === "datetime" && + !isHbs && + queryOperator !== "empty" && + queryOperator !== "notEmpty" + ) { + // Ensure date value is a valid date and parse into correct format + if (!value) { + return + } + try { + value = new Date(value).toISOString() + } catch (error) { + return + } + } + if (type === "number" && typeof value === "string" && !isHbs) { + if (queryOperator === "oneOf") { + value = value.split(",").map(item => parseFloat(item)) + } else { + value = parseFloat(value) + } + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if ( + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && + type === "array" && + typeof value === "string" + ) { + value = value.split(",") + } + if (operator.toLocaleString().startsWith("range") && query.range) { + const minint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.min || Number.MIN_SAFE_INTEGER + const maxint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.max || Number.MAX_SAFE_INTEGER + if (!query.range[field]) { + query.range[field] = { + low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", + high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } + } + } else if (isLogicalSearchOperator(queryOperator)) { + // TODO + } else if (query[queryOperator] && operator !== "onEmptyFilter") { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (queryOperator === "equal" && value === false) { + query.notEqual = query.notEqual || {} + query.notEqual[field] = true + } else if (queryOperator === "notEqual" && value === false) { + query.equal = query.equal || {} + query.equal[field] = true + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } + + return query +} + +export const buildQueryLegacy = ( + filter?: LegacyFilter[] | SearchFilters +): SearchFilters | undefined => { + // this is of type SearchFilters or is undefined + if (!Array.isArray(filter)) { + return filter + } + let query: SearchFilters = { string: {}, fuzzy: {}, @@ -368,13 +504,15 @@ export const buildQuery = (filter: SearchFilter[]) => { value = `${value}`?.toLowerCase() === "true" } if ( - ["contains", "notContains", "containsAny"].includes(operator) && + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && type === "array" && typeof value === "string" ) { value = value.split(",") } - if (operator.startsWith("range") && query.range) { + if (operator.toLocaleString().startsWith("range") && query.range) { const minint = SqlNumberTypeRangeMap[ externalType as keyof typeof SqlNumberTypeRangeMap @@ -401,7 +539,7 @@ export const buildQuery = (filter: SearchFilter[]) => { } } } else if (isLogicalSearchOperator(queryOperator)) { - // TODO + // ignore } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { // Transform boolean filters to cope with null. @@ -423,14 +561,68 @@ export const buildQuery = (filter: SearchFilter[]) => { } } }) - return query } +/** + * Converts a **SearchFilterGroup** filter definition into a grouped + * search query of type **SearchFilters** + * + * Legacy support remains for the old **SearchFilter[]** format. + * These will be migrated to an appropriate **SearchFilters** object, if encountered + * + * @param filter + * + * @returns {SearchFilters} + */ + +export const buildQuery = ( + filter?: SearchFilterGroup | LegacyFilter[] +): SearchFilters | undefined => { + const parsedFilter: SearchFilterGroup | undefined = + processSearchFilters(filter) + + if (!parsedFilter) { + return + } + + const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } = + { + [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, + [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, + } + + const globalOnEmpty = parsedFilter.onEmptyFilter + ? parsedFilter.onEmptyFilter + : null + + const globalOperator: LogicalOperator = + operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] + + const coreRequest: SearchFilters = { + ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), + [globalOperator]: { + conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { + return { + [operatorMap[group.logicalOperator]]: { + conditions: group.filters + ?.map(x => buildCondition(x)) + .filter(filter => filter), + }, + } + }), + }, + } + return coreRequest +} + // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { + if (!filters) { + return filters + } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 81fab659c6..b441791751 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,5 +1,20 @@ -import { ArrayOperator, BasicOperator, SearchFilters } from "@budibase/types" +import { + LegacyFilter, + SearchFilterGroup, + FilterGroupLogicalOperator, + SearchFilters, + BasicOperator, + ArrayOperator, +} from "@budibase/types" import * as Constants from "./constants" +import { removeKeyNumbering } from "./filters" + +// an array of keys from filter type to properties that are in the type +// this can then be converted using .fromEntries to an object +type WhitelistedFilters = [ + keyof LegacyFilter, + LegacyFilter[keyof LegacyFilter] +][] export function unreachable( value: never, @@ -104,3 +119,96 @@ export function isSupportedUserSearch(query: SearchFilters) { } return true } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {LegacyFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: LegacyFilter[] | SearchFilterGroup | undefined +): SearchFilterGroup | undefined => { + if (!filters) { + return + } + + // Base search config. + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + + const filterWhitelistKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => { + // Sort the properties for easier debugging + const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[]) + .sort((a, b) => { + return a.localeCompare(b) + }) + .filter(key => key in filter) + + if (filterPropertyKeys.length == 1) { + const key = filterPropertyKeys[0], + value = filter[key] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const whiteListedFilterSettings: WhitelistedFilters = + filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => { + const value = filter[key] + if (filterWhitelistKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, []) + + const migratedFilter: LegacyFilter = Object.fromEntries( + whiteListedFilterSettings + ) as LegacyFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, defaultCfg) + } else if (!filters?.groups) { + return + } + return filters +} diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a6be5e2986..a99f2938ab 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,6 +9,7 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest extends Omit {} +export interface CreateViewRequest + extends Omit {} -export interface UpdateViewRequest extends ViewV2 {} +export interface UpdateViewRequest extends Omit {} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 5223204a7f..23c599027e 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,7 +1,11 @@ import { FieldType } from "../../documents" -import { EmptyFilterOption, SearchFilters } from "../../sdk" +import { + EmptyFilterOption, + FilterGroupLogicalOperator, + SearchFilters, +} from "../../sdk" -export type SearchFilter = { +export type LegacyFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" onEmptyFilter?: EmptyFilterOption field: string @@ -9,3 +13,11 @@ export type SearchFilter = { value: any externalType?: string } + +// this is a type purely used by the UI +export type SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator + onEmptyFilter?: EmptyFilterOption + groups?: SearchFilterGroup[] + filters?: LegacyFilter[] +} diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index a957564039..87667a71e0 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,7 +1,7 @@ -import { SearchFilter, SortOrder, SortType } from "../../api" +import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" -import { DBView } from "../../sdk" +import { DBView, SearchFilters } from "../../sdk" export type ViewTemplateOpts = { field: string @@ -65,7 +65,9 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilter[] + query?: LegacyFilter[] | SearchFilters + // duplicate to store UI information about filters + queryUI?: SearchFilterGroup sort?: { field: string order?: SortOrder diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 647a9e7d00..d41bb0fb99 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -191,6 +191,11 @@ export enum EmptyFilterOption { RETURN_NONE = "none", } +export enum FilterGroupLogicalOperator { + ALL = "all", + ANY = "any", +} + export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", From 975e348de5b9bed62030ed73b7db00cef700df0d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 10:25:15 +0100 Subject: [PATCH 02/10] Check options.fields are in the table. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 809bd73d1f..eb04e9fe62 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -89,7 +89,7 @@ export async function search( if (options.query) { const visibleFields = ( options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible !== false) + ).filter(field => table.schema[field]?.visible !== false) const queryableFields = await getQueryableFields(table, visibleFields) options.query = removeInvalidFilters(options.query, queryableFields) From 119767a30e970220a64f7aa8a43c25cc349c56b0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 12:20:18 +0200 Subject: [PATCH 03/10] Cleanup --- packages/server/src/api/controllers/row/views.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index b8d01424f2..622688deb6 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -14,7 +14,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view: ViewV2 = await sdk.views.get(viewId) + const view = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } From 522941abf004cae1e04f650a0128df73aca2814a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:31:41 +0100 Subject: [PATCH 04/10] PR comments. --- packages/shared-core/src/filters.ts | 9 +-------- packages/shared-core/src/utils.ts | 19 +++++++++---------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 18ce4b6ed7..b10375acb0 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -163,9 +163,6 @@ export function recurseSearchFilters( * https://github.com/Budibase/budibase/issues/10118 */ export const cleanupQuery = (query: SearchFilters) => { - if (!query) { - return query - } for (let filterField of NoEmptyFilterStrings) { if (!query[filterField]) { continue @@ -599,7 +596,7 @@ export const buildQuery = ( const globalOperator: LogicalOperator = operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] - const coreRequest: SearchFilters = { + return { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), [globalOperator]: { conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { @@ -613,16 +610,12 @@ export const buildQuery = ( }), }, } - return coreRequest } // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { - if (!filters) { - return filters - } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index b441791751..14b3c84425 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -11,10 +11,7 @@ import { removeKeyNumbering } from "./filters" // an array of keys from filter type to properties that are in the type // this can then be converted using .fromEntries to an object -type WhitelistedFilters = [ - keyof LegacyFilter, - LegacyFilter[keyof LegacyFilter] -][] +type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][] export function unreachable( value: never, @@ -141,7 +138,7 @@ export const processSearchFilters = ( groups: [], } - const filterWhitelistKeys = [ + const filterAllowedKeys = [ "field", "operator", "value", @@ -181,10 +178,10 @@ export const processSearchFilters = ( return acc } - const whiteListedFilterSettings: WhitelistedFilters = - filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => { + const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce( + (acc: AllowedFilters, key) => { const value = filter[key] - if (filterWhitelistKeys.includes(key)) { + if (filterAllowedKeys.includes(key)) { if (key === "field") { acc.push([key, removeKeyNumbering(value)]) } else { @@ -192,10 +189,12 @@ export const processSearchFilters = ( } } return acc - }, []) + }, + [] + ) const migratedFilter: LegacyFilter = Object.fromEntries( - whiteListedFilterSettings + allowedFilterSettings ) as LegacyFilter baseGroup.filters!.push(migratedFilter) From 19407d5e37bc4a2e937c4e153a8bdf74fd46a83f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:38:02 +0100 Subject: [PATCH 05/10] Check filters have been provided. --- packages/server/src/sdk/app/rows/search/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 1dba420a28..90303a6ca7 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { } return dataFilters.recurseLogicalOperators(filters, checkFilters) } - options.query = checkFilters(options.query) + if (options.query) { + options.query = checkFilters(options.query) + } return options } From d7873c5c6e5cd6908f4fc75bc44a53b9af6436aa Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:42:16 +0100 Subject: [PATCH 06/10] Test fix. --- .../server/src/api/routes/tests/search.spec.ts | 14 +++++++------- packages/server/src/sdk/app/rows/search.ts | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ec5ca792a..092a851e14 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - ["in-memory", undefined], - ["lucene", undefined], + // ["in-memory", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 87129fdbc8..7e73a51889 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -81,12 +81,13 @@ export async function search( options.query = {} } + // need to make sure filters in correct shape before checking for view + options = searchInputMapping(table, options) + if (options.viewId) { // Delete extraneous search params that cannot be overridden delete options.query.onEmptyFilter - options = searchInputMapping(table, options) - const view = source as ViewV2 // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as From 4d33106b450781573083fd6359f6c1a3297e374e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:42:44 +0100 Subject: [PATCH 07/10] Undo commenting out other DBs. --- .../server/src/api/routes/tests/search.spec.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 092a851e14..1ec5ca792a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - // ["in-memory", undefined], - // ["lucene", undefined], + ["in-memory", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" From 9e7ed0471905eaa5eb5ba3fdf7444c733e6c7858 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:43:07 +0100 Subject: [PATCH 08/10] Support saving queryUI. --- packages/server/src/api/controllers/view/viewsV2.ts | 6 ++++-- packages/types/src/api/web/app/view.ts | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 3df7172de2..0257c86ded 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,10 +99,11 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version" | "queryUI"> = { + const parsedView: Omit, "id" | "version"> = { name: view.name, tableId: view.tableId, query: view.query, + queryUI: view.queryUI, sort: view.sort, schema, primaryDisplay: view.primaryDisplay, @@ -132,12 +133,13 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys> = { + const parsedView: RequiredKeys = { id: view.id, name: view.name, version: view.version, tableId: view.tableId, query: view.query, + queryUI: view.queryUI, sort: view.sort, schema, primaryDisplay: view.primaryDisplay, diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a99f2938ab..a6be5e2986 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,7 +9,6 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest - extends Omit {} +export interface CreateViewRequest extends Omit {} -export interface UpdateViewRequest extends Omit {} +export interface UpdateViewRequest extends ViewV2 {} From 98ebd8f18cf45a026aad06a5b31bc0b538440005 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:59:20 +0100 Subject: [PATCH 09/10] Omitting in tests. --- packages/server/src/api/routes/tests/viewV2.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 1d6c1d50cd..da2354017a 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -154,7 +154,7 @@ describe.each([ }) it("can persist views with all fields", async () => { - const newView: Required = { + const newView: Required> = { name: generator.name(), tableId: table._id!, primaryDisplay: "id", @@ -584,7 +584,7 @@ describe.each([ it("can update all fields", async () => { const tableId = table._id! - const updatedData: Required = { + const updatedData: Required> = { version: view.version, id: view.id, tableId, From 9c70ed92ba58094abba9b53d28a991f3e89db559 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Oct 2024 14:47:31 +0100 Subject: [PATCH 10/10] Small build fix. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 7e73a51889..1a15eb5839 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -92,7 +92,7 @@ export async function search( // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let viewQuery = dataFilters.buildQueryLegacy(view.query || []) + let viewQuery = dataFilters.buildQueryLegacy(view.query) || {} delete viewQuery?.onEmptyFilter const sqsEnabled = await features.flags.isEnabled("SQS")