1
0
Fork 0
mirror of synced 2024-09-10 22:46:09 +12:00
budibase/packages/shared-core/src/filters.ts

462 lines
12 KiB
TypeScript

import { Datasource, FieldType, SortDirection, SortType } from "@budibase/types"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { deepGet } from "./helpers"
const HBS_REGEX = /{{([^{].*?)}}/g
/**
* Returns the valid operator options for a certain data type
* @param type the data type
*/
export const getValidOperatorsForType = (
type: FieldType,
field: string,
datasource: Datasource & { tableId: any } // TODO: is this table id ever populated?
) => {
const Op = OperatorOptions
const stringOps = [
Op.Equals,
Op.NotEquals,
Op.StartsWith,
Op.Like,
Op.Empty,
Op.NotEmpty,
Op.In,
]
const numOps = [
Op.Equals,
Op.NotEquals,
Op.MoreThan,
Op.LessThan,
Op.Empty,
Op.NotEmpty,
Op.In,
]
let ops: {
value: string
label: string
}[] = []
if (type === "string") {
ops = stringOps
} else if (type === "number") {
ops = numOps
} else if (type === "options") {
ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
} else if (type === "array") {
ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
} else if (type === "boolean") {
ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
} else if (type === "longform") {
ops = stringOps
} else if (type === "datetime") {
ops = numOps
} else if (type === "formula") {
ops = stringOps.concat([Op.MoreThan, Op.LessThan])
}
// Filter out "like" for internal tables
const externalTable = datasource?.tableId?.includes("datasource_plus")
if (datasource?.type === "table" && !externalTable) {
ops = ops.filter(x => x !== Op.Like)
}
// Only allow equal/not equal for _id in SQL tables
if (field === "_id" && externalTable) {
ops = [Op.Equals, Op.NotEquals, Op.In]
}
return ops
}
/**
* Operators which do not support empty strings as values
*/
export const NoEmptyFilterStrings = [
OperatorOptions.StartsWith.value,
OperatorOptions.Like.value,
OperatorOptions.Equals.value,
OperatorOptions.NotEquals.value,
OperatorOptions.Contains.value,
OperatorOptions.NotContains.value,
] as (keyof QueryFields)[]
/**
* Removes any fields that contain empty strings that would cause inconsistent
* behaviour with how backend tables are filtered (no value means no filter).
*/
const cleanupQuery = (query: Query) => {
if (!query) {
return query
}
for (let filterField of NoEmptyFilterStrings) {
if (!query[filterField]) {
continue
}
for (let [key, value] of Object.entries(query[filterField]!)) {
if (value == null || value === "") {
delete query[filterField]![key]
}
}
}
return query
}
/**
* Removes a numeric prefix on field names designed to give fields uniqueness
*/
const removeKeyNumbering = (key: string) => {
if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) {
const parts = key.split(":")
parts.shift()
return parts.join(":")
} else {
return key
}
}
type Filter = {
operator: keyof Query
field: string
type: any
value: any
externalType: keyof typeof SqlNumberTypeRangeMap
}
type Query = QueryFields & QueryConfig
type QueryFields = {
string?: {
[key: string]: string
}
fuzzy?: {
[key: string]: string
}
range?: {
[key: string]: {
high: number | string
low: number | string
}
}
equal?: {
[key: string]: any
}
notEqual?: {
[key: string]: any
}
empty?: {
[key: string]: any
}
notEmpty?: {
[key: string]: any
}
oneOf?: {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
}
notContains?: {
[key: string]: any[]
}
containsAny?: {
[key: string]: any[]
}
}
type QueryConfig = {
allOr?: boolean
}
type QueryFieldsType = keyof QueryFields
/**
* Builds a lucene JSON query from the filter structure generated in the builder
* @param filter the builder filter structure
*/
export const buildLuceneQuery = (filter: Filter[]) => {
let query: Query = {
string: {},
fuzzy: {},
range: {},
equal: {},
notEqual: {},
empty: {},
notEmpty: {},
contains: {},
notContains: {},
oneOf: {},
containsAny: {},
}
if (Array.isArray(filter)) {
filter.forEach(expression => {
let { operator, field, type, value, externalType } = expression
const isHbs =
typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
// Parse all values into correct types
if (operator === "allOr") {
query.allOr = true
return
}
if (
type === "datetime" &&
!isHbs &&
operator !== "empty" &&
operator !== "notEmpty"
) {
// Ensure date value is a valid date and parse into correct format
if (!value) {
return
}
try {
value = new Date(value).toISOString()
} catch (error) {
return
}
}
if (type === "number" && typeof value === "string") {
if (operator === "oneOf") {
value = value.split(",").map(item => parseFloat(item))
} else if (!isHbs) {
value = parseFloat(value)
}
}
if (type === "boolean") {
value = `${value}`?.toLowerCase() === "true"
}
if (
["contains", "notContains", "containsAny"].includes(operator) &&
type === "array" &&
typeof value === "string"
) {
value = value.split(",")
}
if (operator.startsWith("range") && query.range) {
const minint =
SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
const maxint =
SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
if (!query.range[field]) {
query.range[field] = {
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
}
}
if ((operator as any) === "rangeLow" && value != null && value !== "") {
query.range[field].low = value
} else if (
(operator as any) === "rangeHigh" &&
value != null &&
value !== ""
) {
query.range[field].high = value
}
} else if (query[operator]) {
if (type === "boolean") {
// Transform boolean filters to cope with null.
// "equals false" needs to be "not equals true"
// "not equals false" needs to be "equals true"
if (operator === "equal" && value === false) {
query.notEqual = query.notEqual || {}
query.notEqual[field] = true
} else if (operator === "notEqual" && value === false) {
query.equal = query.equal || {}
query.equal[field] = true
} else {
query[operator] = query[operator] || {}
query[operator]![field] = value
}
} else {
query[operator] = query[operator] || {}
query[operator]![field] = value
}
}
})
}
return query
}
/**
* Performs a client-side lucene search on an array of data
* @param docs the data
* @param query the JSON lucene query
*/
export const runLuceneQuery = (docs: any[], query?: Query) => {
if (!docs || !Array.isArray(docs)) {
return []
}
if (!query) {
return docs
}
// Make query consistent first
query = cleanupQuery(query)
// Iterates over a set of filters and evaluates a fail function against a doc
const match =
(
type: QueryFieldsType,
failFn: (docValue: any, testValue: any) => boolean
) =>
(doc: any) => {
const filters = Object.entries(query![type] || {})
for (let i = 0; i < filters.length; i++) {
const [key, testValue] = filters[i]
const docValue = deepGet(doc, removeKeyNumbering(key))
if (failFn(docValue, testValue)) {
return false
}
}
return true
}
// Process a string match (fails if the value does not start with the string)
const stringMatch = match("string", (docValue: string, testValue: string) => {
return (
!docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
)
})
// Process a fuzzy match (treat the same as starts with when running locally)
const fuzzyMatch = match("fuzzy", (docValue: string, testValue: string) => {
return (
!docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
)
})
// Process a range match
const rangeMatch = match(
"range",
(
docValue: string | number | null,
testValue: { low: number; high: number }
) => {
return (
docValue == null ||
docValue === "" ||
docValue < testValue.low ||
docValue > testValue.high
)
}
)
// Process an equal match (fails if the value is different)
const equalMatch = match(
"equal",
(docValue: any, testValue: string | null) => {
return testValue != null && testValue !== "" && docValue !== testValue
}
)
// Process a not-equal match (fails if the value is the same)
const notEqualMatch = match(
"notEqual",
(docValue: any, testValue: string | null) => {
return testValue != null && testValue !== "" && docValue === testValue
}
)
// Process an empty match (fails if the value is not empty)
const emptyMatch = match("empty", (docValue: string | null) => {
return docValue != null && docValue !== ""
})
// Process a not-empty match (fails is the value is empty)
const notEmptyMatch = match("notEmpty", (docValue: string | null) => {
return docValue == null || docValue === ""
})
// Process an includes match (fails if the value is not included)
const oneOf = match("oneOf", (docValue: any, testValue: any) => {
if (typeof testValue === "string") {
testValue = testValue.split(",")
if (typeof docValue === "number") {
testValue = testValue.map((item: string) => parseFloat(item))
}
}
return !testValue?.includes(docValue)
})
const containsAny = match("containsAny", (docValue: any, testValue: any) => {
return !docValue?.includes(...testValue)
})
const contains = match(
"contains",
(docValue: string | any[], testValue: any[]) => {
return !testValue?.every((item: any) => docValue?.includes(item))
}
)
const notContains = match(
"notContains",
(docValue: string | any[], testValue: any[]) => {
return testValue?.every((item: any) => docValue?.includes(item))
}
)
// Match a document against all criteria
const docMatch = (doc: any) => {
return (
stringMatch(doc) &&
fuzzyMatch(doc) &&
rangeMatch(doc) &&
equalMatch(doc) &&
notEqualMatch(doc) &&
emptyMatch(doc) &&
notEmptyMatch(doc) &&
oneOf(doc) &&
contains(doc) &&
containsAny(doc) &&
notContains(doc)
)
}
// Process all docs
return docs.filter(docMatch)
}
/**
* Performs a client-side sort from the equivalent server-side lucene sort
* parameters.
* @param docs the data
* @param sort the sort column
* @param sortOrder the sort order ("ascending" or "descending")
* @param sortType the type of sort ("string" or "number")
*/
export const luceneSort = (
docs: any[],
sort: string,
sortOrder: SortDirection,
sortType = SortType.STRING
) => {
if (!sort || !sortOrder || !sortType) {
return docs
}
const parse =
sortType === "string" ? (x: any) => `${x}` : (x: string) => parseFloat(x)
return docs
.slice()
.sort((a: { [x: string]: any }, b: { [x: string]: any }) => {
const colA = parse(a[sort])
const colB = parse(b[sort])
if (sortOrder.toLowerCase() === "descending") {
return colA > colB ? -1 : 1
} else {
return colA > colB ? 1 : -1
}
})
}
/**
* Limits the specified docs to the specified number of rows from the equivalent
* server-side lucene limit parameters.
* @param docs the data
* @param limit the number of docs to limit to
*/
export const luceneLimit = (docs: any[], limit: string) => {
const numLimit = parseFloat(limit)
if (isNaN(numLimit)) {
return docs
}
return docs.slice(0, numLimit)
}