backport of V3 backend changes for search filters on view, giving this the correct type to support conditionals.

packages/server/src/api/controllers/row/views.ts

@@ -5,6 +5,9 @@ import {
   SearchViewRowRequest,
   SearchFilterKey,
   LogicalOperator,
+  RequiredKeys,
+  RowSearchParams,
+  LegacyFilter,
 } from "@budibase/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
@@ -17,7 +20,7 @@ export async function searchView(
 ) {
   const { viewId } = ctx.params
 
-  const view = await sdk.views.get(viewId)
+  const view: ViewV2 = await sdk.views.get(viewId)
   if (!view) {
     ctx.throw(404, `View ${viewId} not found`)
   }
@@ -25,23 +28,35 @@ export async function searchView(
     ctx.throw(400, `This method only supports viewsV2`)
   }
 
+  const viewFields = Object.entries(view.schema || {})
+    .filter(([_, value]) => value.visible)
+    .map(([key]) => key)
   const { body } = ctx.request
 
+  const sqsEnabled = await features.flags.isEnabled("SQS")
+  const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled
+
   // Enrich saved query with ephemeral query params.
   // We prevent searching on any fields that are saved as part of the query, as
   // that could let users find rows they should not be allowed to access.
-  let query = dataFilters.buildQuery(view.query || [])
+  let query = dataFilters.buildQueryLegacy(view.query)
+
+  delete query?.onEmptyFilter
+
   if (body.query) {
     // Delete extraneous search params that cannot be overridden
     delete body.query.onEmptyFilter
 
-    if (
+    if (!supportsLogicalOperators) {
-      !isExternalTableID(view.tableId) &&
+      // In the unlikely event that a Grouped Filter is in a non-SQS environment
-      !(await features.flags.isEnabled("SQS"))
+      // It needs to be ignored entirely
-    ) {
+      let queryFilters: LegacyFilter[] = Array.isArray(view.query)
+        ? view.query
+        : []
+
       // Extract existing fields
       const existingFields =
-        view.query
+        queryFilters
           ?.filter(filter => filter.field)
           .map(filter => db.removeKeyNumbering(filter.field)) || []
 
@@ -49,15 +64,16 @@ export async function searchView(
       Object.keys(body.query).forEach(key => {
         const operator = key as Exclude<SearchFilterKey, LogicalOperator>
         Object.keys(body.query[operator] || {}).forEach(field => {
-          if (!existingFields.includes(db.removeKeyNumbering(field))) {
+          if (query && !existingFields.includes(db.removeKeyNumbering(field))) {
             query[operator]![field] = body.query[operator]![field]
           }
         })
       })
     } else {
+      const conditions = query ? [query] : []
       query = {
         $and: {
-          conditions: [query, body.query],
+          conditions: [...conditions, body.query],
         },
       }
     }
@@ -65,25 +81,29 @@ export async function searchView(
 
   await context.ensureSnippetContext(true)
 
-  const enrichedQuery = await enrichSearchContext(query, {
+  const enrichedQuery = await enrichSearchContext(query || {}, {
     user: sdk.users.getUserContextBindings(ctx.user),
   })
 
-  const result = await sdk.rows.search({
+  const searchOptions: RequiredKeys<SearchViewRowRequest> &
-    viewId: view.id,
+    RequiredKeys<
+      Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
+    > = {
     tableId: view.tableId,
+    viewId: view.id,
     query: enrichedQuery,
+    fields: viewFields,
     ...getSortOptions(body, view),
     limit: body.limit,
     bookmark: body.bookmark,
     paginate: body.paginate,
     countRows: body.countRows,
-  })
+  }
 
+  const result = await sdk.rows.search(searchOptions)
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }
-
 function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
   if (request.sort) {
     return {

packages/server/src/api/controllers/view/viewsV2.ts

@@ -99,7 +99,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
 
   const schema = await parseSchema(view)
 
-  const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
+  const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version" | "queryUI"> = {
     name: view.name,
     tableId: view.tableId,
     query: view.query,
@@ -132,7 +132,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
   const { tableId } = view
 
   const schema = await parseSchema(view)
-  const parsedView: RequiredKeys<ViewV2> = {
+  const parsedView: RequiredKeys<Omit<ViewV2, "queryUI">> = {
     id: view.id,
     name: view.name,
     version: view.version,

packages/shared-core/src/filters.ts

@@ -3,7 +3,7 @@ import {
   BBReferenceFieldSubType,
   FieldType,
   FormulaType,
-  SearchFilter,
+  LegacyFilter,
   SearchFilters,
   SearchQueryFields,
   ArrayOperator,
@@ -19,9 +19,12 @@ import {
   RangeOperator,
   LogicalOperator,
   isLogicalSearchOperator,
+  SearchFilterGroup,
+  FilterGroupLogicalOperator,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
+import { processSearchFilters } from "./utils"
 import { deepGet, schema } from "./helpers"
 import { isPlainObject, isEmpty } from "lodash"
 import { decodeNonAscii } from "./helpers/schema"
@@ -124,7 +127,7 @@ export function recurseLogicalOperators(
   fn: (f: SearchFilters) => SearchFilters
 ) {
   for (const logical of LOGICAL_OPERATORS) {
-    if (filters?.[logical]) {
+    if (filters[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => fn(condition)
       )
@@ -304,10 +307,143 @@ export class ColumnSplitter {
 }
 
 /**
- * Builds a JSON query from the filter structure generated in the builder
+ * Builds a JSON query from the filter a SearchFilter definition
  * @param filter the builder filter structure
  */
-export const buildQuery = (filter: SearchFilter[]) => {
+
+const buildCondition = (expression: LegacyFilter) => {
+  // Filter body
+  let query: SearchFilters = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+    contains: {},
+    notContains: {},
+    oneOf: {},
+    containsAny: {},
+  }
+  let { operator, field, type, value, externalType, onEmptyFilter } = expression
+
+  if (!operator || !field) {
+    return
+  }
+
+  const queryOperator = operator as SearchFilterOperator
+  const isHbs =
+    typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+  // Parse all values into correct types
+  if (operator === "allOr") {
+    query.allOr = true
+    return
+  }
+  if (onEmptyFilter) {
+    query.onEmptyFilter = onEmptyFilter
+    return
+  }
+
+  // Default the value for noValue fields to ensure they are correctly added
+  // to the final query
+  if (queryOperator === "empty" || queryOperator === "notEmpty") {
+    value = null
+  }
+
+  if (
+    type === "datetime" &&
+    !isHbs &&
+    queryOperator !== "empty" &&
+    queryOperator !== "notEmpty"
+  ) {
+    // Ensure date value is a valid date and parse into correct format
+    if (!value) {
+      return
+    }
+    try {
+      value = new Date(value).toISOString()
+    } catch (error) {
+      return
+    }
+  }
+  if (type === "number" && typeof value === "string" && !isHbs) {
+    if (queryOperator === "oneOf") {
+      value = value.split(",").map(item => parseFloat(item))
+    } else {
+      value = parseFloat(value)
+    }
+  }
+  if (type === "boolean") {
+    value = `${value}`?.toLowerCase() === "true"
+  }
+  if (
+    ["contains", "notContains", "containsAny"].includes(
+      operator.toLocaleString()
+    ) &&
+    type === "array" &&
+    typeof value === "string"
+  ) {
+    value = value.split(",")
+  }
+  if (operator.toLocaleString().startsWith("range") && query.range) {
+    const minint =
+      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
+        ?.min || Number.MIN_SAFE_INTEGER
+    const maxint =
+      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
+        ?.max || Number.MAX_SAFE_INTEGER
+    if (!query.range[field]) {
+      query.range[field] = {
+        low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+        high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+      }
+    }
+    if (operator === "rangeLow" && value != null && value !== "") {
+      query.range[field] = {
+        ...query.range[field],
+        low: value,
+      }
+    } else if (operator === "rangeHigh" && value != null && value !== "") {
+      query.range[field] = {
+        ...query.range[field],
+        high: value,
+      }
+    }
+  } else if (isLogicalSearchOperator(queryOperator)) {
+    // TODO
+  } else if (query[queryOperator] && operator !== "onEmptyFilter") {
+    if (type === "boolean") {
+      // Transform boolean filters to cope with null.
+      // "equals false" needs to be "not equals true"
+      // "not equals false" needs to be "equals true"
+      if (queryOperator === "equal" && value === false) {
+        query.notEqual = query.notEqual || {}
+        query.notEqual[field] = true
+      } else if (queryOperator === "notEqual" && value === false) {
+        query.equal = query.equal || {}
+        query.equal[field] = true
+      } else {
+        query[queryOperator] ??= {}
+        query[queryOperator]![field] = value
+      }
+    } else {
+      query[queryOperator] ??= {}
+      query[queryOperator]![field] = value
+    }
+  }
+
+  return query
+}
+
+export const buildQueryLegacy = (
+  filter?: LegacyFilter[] | SearchFilters
+): SearchFilters | undefined => {
+  // this is of type SearchFilters or is undefined
+  if (!Array.isArray(filter)) {
+    return filter
+  }
+
   let query: SearchFilters = {
     string: {},
     fuzzy: {},
@@ -368,13 +504,15 @@ export const buildQuery = (filter: SearchFilter[]) => {
       value = `${value}`?.toLowerCase() === "true"
     }
     if (
-      ["contains", "notContains", "containsAny"].includes(operator) &&
+      ["contains", "notContains", "containsAny"].includes(
+        operator.toLocaleString()
+      ) &&
       type === "array" &&
       typeof value === "string"
     ) {
       value = value.split(",")
     }
-    if (operator.startsWith("range") && query.range) {
+    if (operator.toLocaleString().startsWith("range") && query.range) {
       const minint =
         SqlNumberTypeRangeMap[
           externalType as keyof typeof SqlNumberTypeRangeMap
@@ -401,7 +539,7 @@ export const buildQuery = (filter: SearchFilter[]) => {
         }
       }
     } else if (isLogicalSearchOperator(queryOperator)) {
-      // TODO
+      // ignore
     } else if (query[queryOperator] && operator !== "onEmptyFilter") {
       if (type === "boolean") {
         // Transform boolean filters to cope with null.
@@ -423,14 +561,68 @@ export const buildQuery = (filter: SearchFilter[]) => {
       }
     }
   })
-
   return query
 }
 
+/**
+ * Converts a **SearchFilterGroup** filter definition into a grouped
+ * search query of type **SearchFilters**
+ *
+ * Legacy support remains for the old **SearchFilter[]** format.
+ * These will be migrated to an appropriate **SearchFilters** object, if encountered
+ *
+ * @param filter
+ *
+ * @returns {SearchFilters}
+ */
+
+export const buildQuery = (
+  filter?: SearchFilterGroup | LegacyFilter[]
+): SearchFilters | undefined => {
+  const parsedFilter: SearchFilterGroup | undefined =
+    processSearchFilters(filter)
+
+  if (!parsedFilter) {
+    return
+  }
+
+  const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } =
+    {
+      [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
+      [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
+    }
+
+  const globalOnEmpty = parsedFilter.onEmptyFilter
+    ? parsedFilter.onEmptyFilter
+    : null
+
+  const globalOperator: LogicalOperator =
+    operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
+
+  const coreRequest: SearchFilters = {
+    ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
+    [globalOperator]: {
+      conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
+        return {
+          [operatorMap[group.logicalOperator]]: {
+            conditions: group.filters
+              ?.map(x => buildCondition(x))
+              .filter(filter => filter),
+          },
+        }
+      }),
+    },
+  }
+  return coreRequest
+}
+
 // The frontend can send single values for array fields sometimes, so to handle
 // this we convert them to arrays at the controller level so that nothing below
 // this has to worry about the non-array values.
 export function fixupFilterArrays(filters: SearchFilters) {
+  if (!filters) {
+    return filters
+  }
   for (const searchField of Object.values(ArrayOperator)) {
     const field = filters[searchField]
     if (field == null || !isPlainObject(field)) {

packages/shared-core/src/utils.ts

@@ -1,5 +1,20 @@
-import { ArrayOperator, BasicOperator, SearchFilters } from "@budibase/types"
+import {
+  LegacyFilter,
+  SearchFilterGroup,
+  FilterGroupLogicalOperator,
+  SearchFilters,
+  BasicOperator,
+  ArrayOperator,
+} from "@budibase/types"
 import * as Constants from "./constants"
+import { removeKeyNumbering } from "./filters"
+
+// an array of keys from filter type to properties that are in the type
+// this can then be converted using .fromEntries to an object
+type WhitelistedFilters = [
+  keyof LegacyFilter,
+  LegacyFilter[keyof LegacyFilter]
+][]
 
 export function unreachable(
   value: never,
@@ -104,3 +119,96 @@ export function isSupportedUserSearch(query: SearchFilters) {
   }
   return true
 }
+
+/**
+ * Processes the filter config. Filters are migrated from
+ * SearchFilter[] to SearchFilterGroup
+ *
+ * If filters is not an array, the migration is skipped
+ *
+ * @param {LegacyFilter[] | SearchFilterGroup} filters
+ */
+export const processSearchFilters = (
+  filters: LegacyFilter[] | SearchFilterGroup | undefined
+): SearchFilterGroup | undefined => {
+  if (!filters) {
+    return
+  }
+
+  // Base search config.
+  const defaultCfg: SearchFilterGroup = {
+    logicalOperator: FilterGroupLogicalOperator.ALL,
+    groups: [],
+  }
+
+  const filterWhitelistKeys = [
+    "field",
+    "operator",
+    "value",
+    "type",
+    "externalType",
+    "valueType",
+    "noValue",
+    "formulaType",
+  ]
+
+  if (Array.isArray(filters)) {
+    let baseGroup: SearchFilterGroup = {
+      filters: [],
+      logicalOperator: FilterGroupLogicalOperator.ALL,
+    }
+
+    return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
+      // Sort the properties for easier debugging
+      const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
+        .sort((a, b) => {
+          return a.localeCompare(b)
+        })
+        .filter(key => key in filter)
+
+      if (filterPropertyKeys.length == 1) {
+        const key = filterPropertyKeys[0],
+          value = filter[key]
+        // Global
+        if (key === "onEmptyFilter") {
+          // unset otherwise
+          acc.onEmptyFilter = value
+        } else if (key === "operator" && value === "allOr") {
+          // Group 1 logical operator
+          baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
+        }
+
+        return acc
+      }
+
+      const whiteListedFilterSettings: WhitelistedFilters =
+        filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => {
+          const value = filter[key]
+          if (filterWhitelistKeys.includes(key)) {
+            if (key === "field") {
+              acc.push([key, removeKeyNumbering(value)])
+            } else {
+              acc.push([key, value])
+            }
+          }
+          return acc
+        }, [])
+
+      const migratedFilter: LegacyFilter = Object.fromEntries(
+        whiteListedFilterSettings
+      ) as LegacyFilter
+
+      baseGroup.filters!.push(migratedFilter)
+
+      if (!acc.groups || !acc.groups.length) {
+        // init the base group
+        acc.groups = [baseGroup]
+      }
+
+      return acc
+    }, defaultCfg)
+  } else if (!filters?.groups) {
+    return
+  }
+  return filters
+}

packages/types/src/api/web/app/view.ts

@@ -9,6 +9,7 @@ export interface ViewResponseEnriched {
   data: ViewV2Enriched
 }
 
-export interface CreateViewRequest extends Omit<ViewV2, "version" | "id"> {}
+export interface CreateViewRequest
+  extends Omit<ViewV2, "version" | "id" | "queryUI"> {}
 
-export interface UpdateViewRequest extends ViewV2 {}
+export interface UpdateViewRequest extends Omit<ViewV2, "queryUI"> {}

packages/types/src/api/web/searchFilter.ts

@@ -1,7 +1,11 @@
 import { FieldType } from "../../documents"
-import { EmptyFilterOption, SearchFilters } from "../../sdk"
+import {
+  EmptyFilterOption,
+  FilterGroupLogicalOperator,
+  SearchFilters,
+} from "../../sdk"
 
-export type SearchFilter = {
+export type LegacyFilter = {
   operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
   onEmptyFilter?: EmptyFilterOption
   field: string
@@ -9,3 +13,11 @@ export type SearchFilter = {
   value: any
   externalType?: string
 }
+
+// this is a type purely used by the UI
+export type SearchFilterGroup = {
+  logicalOperator: FilterGroupLogicalOperator
+  onEmptyFilter?: EmptyFilterOption
+  groups?: SearchFilterGroup[]
+  filters?: LegacyFilter[]
+}

packages/types/src/documents/app/view.ts

@@ -1,7 +1,7 @@
-import { SearchFilter, SortOrder, SortType } from "../../api"
+import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
 import { Document } from "../document"
-import { DBView } from "../../sdk"
+import { DBView, SearchFilters } from "../../sdk"
 
 export type ViewTemplateOpts = {
   field: string
@@ -65,7 +65,9 @@ export interface ViewV2 {
   name: string
   primaryDisplay?: string
   tableId: string
-  query?: SearchFilter[]
+  query?: LegacyFilter[] | SearchFilters
+  // duplicate to store UI information about filters
+  queryUI?: SearchFilterGroup
   sort?: {
     field: string
     order?: SortOrder

packages/types/src/sdk/search.ts

@@ -191,6 +191,11 @@ export enum EmptyFilterOption {
   RETURN_NONE = "none",
 }
 
+export enum FilterGroupLogicalOperator {
+  ALL = "all",
+  ANY = "any",
+}
+
 export enum SqlClient {
   MS_SQL = "mssql",
   POSTGRES = "pg",