From bc8791e91eaa96a32b632d9e594bd142086bbdd5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 14:44:20 +0100 Subject: [PATCH 1/4] Remove the word 'lucene' from runLuceneQuery and buildLuceneQuery. --- .../SetupPanel/AutomationBlockSetup.svelte | 2 +- .../DataTable/modals/ExportModal.svelte | 2 +- .../src/components/app/DataProvider.svelte | 2 +- .../app/dynamic-filter/DynamicFilter.svelte | 2 +- packages/client/src/utils/conditions.js | 4 +- packages/frontend-core/src/fetch/DataFetch.js | 7 ++- .../server/src/api/controllers/row/views.ts | 2 +- .../server/src/integrations/googlesheets.ts | 2 +- packages/shared-core/src/filters.ts | 10 ++-- .../shared-core/src/tests/filters.test.ts | 46 +++++++++---------- 10 files changed, 39 insertions(+), 40 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 5d2e4ee28d..cf3cd95581 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -343,7 +343,7 @@ } function saveFilters(key) { - const filters = LuceneUtils.buildLuceneQuery(tempFilters) + const filters = LuceneUtils.buildQuery(tempFilters) const defKey = `${key}-def` onChange({ detail: filters }, key) // need to store the builder definition in the automation diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte index c2a7ec33fe..b26cfa65ad 100644 --- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte @@ -49,7 +49,7 @@ exportFormat = Array.isArray(options) ? options[0]?.key : [] } - $: luceneFilter = LuceneUtils.buildLuceneQuery(appliedFilters) + $: luceneFilter = LuceneUtils.buildQuery(appliedFilters) $: exportOpDisplay = buildExportOpDisplay( sorting, filterDisplay, diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index dc0c40f55c..983e7324dd 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -19,7 +19,7 @@ // We need to manage our lucene query manually as we want to allow components // to extend it - $: defaultQuery = LuceneUtils.buildLuceneQuery(filter) + $: defaultQuery = LuceneUtils.buildQuery(filter) $: query = extendQuery(defaultQuery, queryExtensions) $: fetch = createFetch(dataSource) $: fetch.update({ diff --git a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte index 2f7af56744..1c206545d9 100644 --- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte +++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte @@ -36,7 +36,7 @@ // Add query extension to data provider $: { if (filters?.length) { - const queryExtension = LuceneUtils.buildLuceneQuery(filters) + const queryExtension = LuceneUtils.buildQuery(filters) addExtension?.($component.id, queryExtension) } else { removeExtension?.($component.id) diff --git a/packages/client/src/utils/conditions.js b/packages/client/src/utils/conditions.js index 1914e942ad..12152fbeed 100644 --- a/packages/client/src/utils/conditions.js +++ b/packages/client/src/utils/conditions.js @@ -33,8 +33,8 @@ export const getActiveConditions = conditions => { value: condition.referenceValue, } - const query = LuceneUtils.buildLuceneQuery([luceneCondition]) - const result = LuceneUtils.runLuceneQuery([luceneCondition], query) + const query = LuceneUtils.buildQuery([luceneCondition]) + const result = LuceneUtils.runQuery([luceneCondition], query) return result.length > 0 }) } diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js index 3a45543c60..16ab4e9ffb 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.js @@ -3,8 +3,7 @@ import { cloneDeep } from "lodash/fp" import { LuceneUtils } from "../utils" import { convertJSONSchemaToTableSchema } from "../utils/json" -const { buildLuceneQuery, luceneLimit, runLuceneQuery, luceneSort } = - LuceneUtils +const { buildQuery, luceneLimit, runQuery, luceneSort } = LuceneUtils /** * Parent class which handles the implementation of fetching data from an @@ -180,7 +179,7 @@ export default class DataFetch { // Build the lucene query let query = this.options.query if (!query) { - query = buildLuceneQuery(filter) + query = buildQuery(filter) } // Update store @@ -229,7 +228,7 @@ export default class DataFetch { // If we don't support searching, do a client search if (!this.features.supportsSearch && clientSideSearching) { - rows = runLuceneQuery(rows, query) + rows = runQuery(rows, query) } // If we don't support sorting, do a client-side sort diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 13f6ce8eb0..80aa97d8c0 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -31,7 +31,7 @@ export async function searchView( // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query = dataFilters.buildLuceneQuery(view.query || []) + let query = dataFilters.buildQuery(view.query || []) if (body.query) { // Extract existing fields const existingFields = diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 2da0c2c201..593f3ef891 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -566,7 +566,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { query.filters.equal[`_${GOOGLE_SHEETS_PRIMARY_KEY}`] = id } } - let filtered = dataFilters.runLuceneQuery(rows, query.filters) + let filtered = dataFilters.runQuery(rows, query.filters) if (hasFilters && query.paginate) { filtered = filtered.slice(offset, offset + limit) } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 1401b75e93..9b4cf86ed4 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -138,10 +138,10 @@ export const removeKeyNumbering = (key: string): string => { } /** - * Builds a lucene JSON query from the filter structure generated in the builder + * Builds a JSON query from the filter structure generated in the builder * @param filter the builder filter structure */ -export const buildLuceneQuery = (filter: SearchFilter[]) => { +export const buildQuery = (filter: SearchFilter[]) => { let query: SearchFilters = { string: {}, fuzzy: {}, @@ -260,11 +260,11 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { } /** - * Performs a client-side lucene search on an array of data + * Performs a client-side search on an array of data * @param docs the data - * @param query the JSON lucene query + * @param query the JSON query */ -export const runLuceneQuery = (docs: any[], query?: SearchFilters) => { +export const runQuery = (docs: any[], query?: SearchFilters) => { if (!docs || !Array.isArray(docs)) { return [] } diff --git a/packages/shared-core/src/tests/filters.test.ts b/packages/shared-core/src/tests/filters.test.ts index f188c5f951..f5c2c2f06c 100644 --- a/packages/shared-core/src/tests/filters.test.ts +++ b/packages/shared-core/src/tests/filters.test.ts @@ -4,9 +4,9 @@ import { FieldType, SearchFilter, } from "@budibase/types" -import { buildLuceneQuery, runLuceneQuery } from "../filters" +import { buildQuery, runQuery } from "../filters" -describe("runLuceneQuery", () => { +describe("runQuery", () => { const docs = [ { order_id: 1, @@ -70,14 +70,14 @@ describe("runLuceneQuery", () => { } it("should return input docs if no search query is provided", () => { - expect(runLuceneQuery(docs)).toBe(docs) + expect(runQuery(docs)).toBe(docs) }) it("should return matching rows for equal filter", () => { const query = buildQuery({ equal: { order_status: 4 }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2]) + expect(runQuery(docs, query).map(row => row.order_id)).toEqual([1, 2]) }) it("should return matching row for notEqual filter", () => { @@ -85,12 +85,12 @@ describe("runLuceneQuery", () => { notEqual: { order_status: 4 }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3]) + expect(runQuery(docs, query).map(row => row.order_id)).toEqual([3]) }) it("should return starts with matching rows for fuzzy and string filters", () => { expect( - runLuceneQuery( + runQuery( docs, buildQuery({ fuzzy: { description: "sm" }, @@ -98,7 +98,7 @@ describe("runLuceneQuery", () => { ).map(row => row.description) ).toEqual(["Small box"]) expect( - runLuceneQuery( + runQuery( docs, buildQuery({ string: { description: "SM" }, @@ -117,7 +117,7 @@ describe("runLuceneQuery", () => { }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3]) + expect(runQuery(docs, query).map(row => row.order_id)).toEqual([3]) }) it("should return rows with numeric strings within a range filter", () => { @@ -129,7 +129,7 @@ describe("runLuceneQuery", () => { }, }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3]) + expect(runQuery(docs, query).map(row => row.order_id)).toEqual([3]) }) it("should return rows with ISO date strings within a range filter", () => { @@ -142,7 +142,7 @@ describe("runLuceneQuery", () => { }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2]) + expect(runQuery(docs, query).map(row => row.order_id)).toEqual([2]) }) it("should return return all docs if an invalid doc value is passed into a range filter", async () => { @@ -170,7 +170,7 @@ describe("runLuceneQuery", () => { }, }) - expect(runLuceneQuery(docs, query)).toEqual(docs) + expect(runQuery(docs, query)).toEqual(docs) }) it("should return rows with matches on empty filter", () => { @@ -180,7 +180,7 @@ describe("runLuceneQuery", () => { }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1]) + expect(runQuery(docs, query).map(row => row.order_id)).toEqual([1]) }) it("should return rows with matches on notEmpty filter", () => { @@ -190,7 +190,7 @@ describe("runLuceneQuery", () => { }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2, 3]) + expect(runQuery(docs, query).map(row => row.order_id)).toEqual([2, 3]) }) it.each([[523, 259], "523,259"])( @@ -202,7 +202,7 @@ describe("runLuceneQuery", () => { }, }) - expect(runLuceneQuery(docs, query).map(row => row.customer_id)).toEqual([ + expect(runQuery(docs, query).map(row => row.customer_id)).toEqual([ 259, 523, ]) } @@ -218,7 +218,7 @@ describe("runLuceneQuery", () => { contains: { description: ["box"] }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual( + expect(runQuery(docs, query).map(row => row.order_id)).toEqual( expectedResult ) }) @@ -230,7 +230,7 @@ describe("runLuceneQuery", () => { oneOf: { label: ["FRAGILE"] }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2]) + expect(runQuery(docs, query).map(row => row.order_id)).toEqual([1, 2]) }) it("should handle when a value is null or undefined", () => { @@ -240,14 +240,14 @@ describe("runLuceneQuery", () => { oneOf: { label: ["FRAGILE"] }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2]) + expect(runQuery(docs, query).map(row => row.order_id)).toEqual([2]) }) }) -describe("buildLuceneQuery", () => { +describe("buildQuery", () => { it("should return a basic search query template if the input is not an array", () => { const filter: any = "NOT_AN_ARRAY" - expect(buildLuceneQuery(filter)).toEqual({ + expect(buildQuery(filter)).toEqual({ string: {}, fuzzy: {}, range: {}, @@ -277,7 +277,7 @@ describe("buildLuceneQuery", () => { value: "1000,1212,3400", }, ] - expect(buildLuceneQuery(filter)).toEqual({ + expect(buildQuery(filter)).toEqual({ string: {}, fuzzy: {}, range: {}, @@ -311,7 +311,7 @@ describe("buildLuceneQuery", () => { value: "{{ list_of_customer_ids }}", }, ] - expect(buildLuceneQuery(filter)).toEqual({ + expect(buildQuery(filter)).toEqual({ string: {}, fuzzy: {}, range: {}, @@ -351,7 +351,7 @@ describe("buildLuceneQuery", () => { value: "true", }, ] - expect(buildLuceneQuery(filter)).toEqual({ + expect(buildQuery(filter)).toEqual({ string: {}, fuzzy: {}, range: {}, @@ -392,7 +392,7 @@ describe("buildLuceneQuery", () => { value: "Large box,Heavy box,Small box", }, ] - expect(buildLuceneQuery(filter)).toEqual({ + expect(buildQuery(filter)).toEqual({ string: {}, fuzzy: {}, range: {}, From 6363965a8b136aa52ebfdd119a750715dfd288f0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 15:04:56 +0100 Subject: [PATCH 2/4] Also rename LuceneUtils -> QueryUtils --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 4 ++-- .../components/backend/DataTable/modals/ExportModal.svelte | 4 ++-- .../_components/Component/ConditionalUIDrawer.svelte | 6 +++--- packages/client/src/components/app/DataProvider.svelte | 4 ++-- .../src/components/app/dynamic-filter/DynamicFilter.svelte | 4 ++-- packages/client/src/sdk.js | 4 ++-- packages/client/src/utils/conditions.js | 6 +++--- packages/frontend-core/src/components/FilterBuilder.svelte | 4 ++-- packages/frontend-core/src/fetch/DataFetch.js | 4 ++-- packages/frontend-core/src/fetch/UserFetch.js | 4 ++-- packages/frontend-core/src/utils/index.js | 2 +- 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index cf3cd95581..b8b7c5ae54 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -41,7 +41,7 @@ EditorModes, } from "components/common/CodeEditor" import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte" - import { LuceneUtils, Utils } from "@budibase/frontend-core" + import { QueryUtils, Utils } from "@budibase/frontend-core" import { getSchemaForDatasourcePlus, getEnvironmentBindings, @@ -343,7 +343,7 @@ } function saveFilters(key) { - const filters = LuceneUtils.buildQuery(tempFilters) + const filters = QueryUtils.buildQuery(tempFilters) const defKey = `${key}-def` onChange({ detail: filters }, key) // need to store the builder definition in the automation diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte index b26cfa65ad..52e23b9a37 100644 --- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte @@ -8,7 +8,7 @@ } from "@budibase/bbui" import download from "downloadjs" import { API } from "api" - import { LuceneUtils } from "@budibase/frontend-core" + import { QueryUtils } from "@budibase/frontend-core" import { utils } from "@budibase/shared-core" import { ROW_EXPORT_FORMATS } from "constants/backend" @@ -49,7 +49,7 @@ exportFormat = Array.isArray(options) ? options[0]?.key : [] } - $: luceneFilter = LuceneUtils.buildQuery(appliedFilters) + $: luceneFilter = QueryUtils.buildQuery(appliedFilters) $: exportOpDisplay = buildExportOpDisplay( sorting, filterDisplay, diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte index 6558ba9dc2..8c57e82998 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte @@ -12,7 +12,7 @@ import { dndzone } from "svelte-dnd-action" import { generate } from "shortid" import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" - import { LuceneUtils, Constants } from "@budibase/frontend-core" + import { QueryUtils, Constants } from "@budibase/frontend-core" import { selectedComponent, componentStore } from "stores/builder" import { getComponentForSetting } from "components/design/settings/componentSettings" import PropertyControl from "components/design/settings/controls/PropertyControl.svelte" @@ -119,7 +119,7 @@ } const getOperatorOptions = condition => { - return LuceneUtils.getValidOperatorsForType({ type: condition.valueType }) + return QueryUtils.getValidOperatorsForType({ type: condition.valueType }) } const onOperatorChange = (condition, newOperator) => { @@ -138,7 +138,7 @@ condition.referenceValue = null // Ensure a valid operator is set - const validOperators = LuceneUtils.getValidOperatorsForType({ + const validOperators = QueryUtils.getValidOperatorsForType({ type: newType, }).map(x => x.value) if (!validOperators.includes(condition.operator)) { diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index 983e7324dd..3e005b02cf 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -1,7 +1,7 @@