From d10d332b9fdf373399e5df9d85c6422d8505ca98 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 6 Dec 2021 12:04:22 +0000 Subject: [PATCH] Update client side lucene searching to work with nested JSON fields and fix bug with flattening nested JSON schema --- .../builder/src/builderStore/dataBinding.js | 1 + .../builder/src/builderStore/jsonUtils.js | 7 ++- .../FilterEditor/FilterDrawer.svelte | 2 +- packages/builder/src/helpers/lucene.js | 51 +++++++++++++------ .../app/dynamic-filter/FilterModal.svelte | 2 +- 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index f5b8fcbb5f..f96ff6037e 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -410,6 +410,7 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => { jsonSchema = jsonSchema[keysToSchema[i]].schema } schema = convertJSONSchemaToTableSchema(jsonSchema, true) + console.log(schema) } // Otherwise we assume we're targeting an internal table or a plus diff --git a/packages/builder/src/builderStore/jsonUtils.js b/packages/builder/src/builderStore/jsonUtils.js index bc865893e3..4b03a67234 100644 --- a/packages/builder/src/builderStore/jsonUtils.js +++ b/packages/builder/src/builderStore/jsonUtils.js @@ -1,11 +1,14 @@ -export const convertJSONSchemaToTableSchema = jsonSchema => { +export const convertJSONSchemaToTableSchema = ( + jsonSchema, + squashObjects = false +) => { if (!jsonSchema) { return null } if (jsonSchema.schema) { jsonSchema = jsonSchema.schema } - const keys = extractJSONSchemaKeys(jsonSchema) + const keys = extractJSONSchemaKeys(jsonSchema, squashObjects) let schema = {} keys.forEach(({ key, type }) => { schema[key] = { type, name: key } diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index e30fd6d491..277ab85586 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -21,7 +21,7 @@ export let panel = ClientBindingPanel export let allowBindings = true - const BannedTypes = ["link", "attachment", "formula"] + const BannedTypes = ["link", "attachment", "formula", "json"] $: fieldOptions = (schemaFields ?? []) .filter(field => !BannedTypes.includes(field.type)) diff --git a/packages/builder/src/helpers/lucene.js b/packages/builder/src/helpers/lucene.js index d344b462d1..fff0c5e4e8 100644 --- a/packages/builder/src/helpers/lucene.js +++ b/packages/builder/src/helpers/lucene.js @@ -99,6 +99,7 @@ export const luceneQuery = (docs, query) => { if (!query) { return docs } + // make query consistent first query = cleanupQuery(query) @@ -106,7 +107,9 @@ export const luceneQuery = (docs, query) => { const match = (type, failFn) => doc => { const filters = Object.entries(query[type] || {}) for (let i = 0; i < filters.length; i++) { - if (failFn(filters[i][0], filters[i][1], doc)) { + const [key, testValue] = filters[i] + const docValue = deepGet(doc, key) + if (failFn(docValue, testValue)) { return false } } @@ -114,38 +117,38 @@ export const luceneQuery = (docs, query) => { } // Process a string match (fails if the value does not start with the string) - const stringMatch = match("string", (key, value, doc) => { - return !doc[key] || !doc[key].startsWith(value) + const stringMatch = match("string", (docValue, testValue) => { + return !docValue || !docValue.startsWith(testValue) }) // Process a fuzzy match (treat the same as starts with when running locally) - const fuzzyMatch = match("fuzzy", (key, value, doc) => { - return !doc[key] || !doc[key].startsWith(value) + const fuzzyMatch = match("fuzzy", (docValue, testValue) => { + return !docValue || !docValue.startsWith(testValue) }) // Process a range match - const rangeMatch = match("range", (key, value, doc) => { - return !doc[key] || doc[key] < value.low || doc[key] > value.high + const rangeMatch = match("range", (docValue, testValue) => { + return !docValue || docValue < testValue.low || docValue > testValue.high }) // Process an equal match (fails if the value is different) - const equalMatch = match("equal", (key, value, doc) => { - return value != null && value !== "" && doc[key] !== value + const equalMatch = match("equal", (docValue, testValue) => { + return testValue != null && testValue !== "" && docValue !== testValue }) // Process a not-equal match (fails if the value is the same) - const notEqualMatch = match("notEqual", (key, value, doc) => { - return value != null && value !== "" && doc[key] === value + const notEqualMatch = match("notEqual", (docValue, testValue) => { + return testValue != null && testValue !== "" && docValue === testValue }) // Process an empty match (fails if the value is not empty) - const emptyMatch = match("empty", (key, value, doc) => { - return doc[key] != null && doc[key] !== "" + const emptyMatch = match("empty", docValue => { + return docValue != null && docValue !== "" }) // Process a not-empty match (fails is the value is empty) - const notEmptyMatch = match("notEmpty", (key, value, doc) => { - return doc[key] == null || doc[key] === "" + const notEmptyMatch = match("notEmpty", docValue => { + return docValue == null || docValue === "" }) // Match a document against all criteria @@ -202,3 +205,21 @@ export const luceneLimit = (docs, limit) => { } return docs.slice(0, numLimit) } + +/** + * Gets a key within an object. The key supports dot syntax for retriving deep + * fields - e.g. "a.b.c". + * @param obj the object + * @param key the key + */ +const deepGet = (obj, key) => { + if (!obj || !key) { + return null + } + const split = key.split(".") + let value = obj + for (let i = 0; i < split.length; i++) { + value = value?.[split[i]] + } + return value +} diff --git a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte index 99b9a1a6f6..f303c69aaf 100644 --- a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte +++ b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte @@ -19,7 +19,7 @@ export let schemaFields export let filters = [] - const BannedTypes = ["link", "attachment", "formula"] + const BannedTypes = ["link", "attachment", "formula", "json"] $: fieldOptions = (schemaFields ?? []) .filter(field => !BannedTypes.includes(field.type))