From 661fc361a05e2c1864e6dc1030f9e0cb483cdc98 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Aug 2024 15:01:33 +0100 Subject: [PATCH] Refactoring search SDK to make it obvious the different search methods, exports was using lucene always when doing internal export rows which shouldn't be the case, should go through the complete search SDK. --- packages/server/src/sdk/app/rows/search.ts | 5 +- .../src/sdk/app/rows/search/internal/index.ts | 3 + .../rows/search/{ => internal}/internal.ts | 93 +++---------------- .../sdk/app/rows/search/internal/lucene.ts | 66 +++++++++++++ .../sdk/app/rows/search/{ => internal}/sqs.ts | 18 ++-- 5 files changed, 95 insertions(+), 90 deletions(-) create mode 100644 packages/server/src/sdk/app/rows/search/internal/index.ts rename packages/server/src/sdk/app/rows/search/{ => internal}/internal.ts (70%) create mode 100644 packages/server/src/sdk/app/rows/search/internal/lucene.ts rename packages/server/src/sdk/app/rows/search/{ => internal}/sqs.ts (96%) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index d074e8e2ac..c008d43548 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -8,7 +8,6 @@ import { import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" -import * as sqs from "./search/sqs" import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" @@ -55,9 +54,9 @@ export async function search( if (isExternalTable) { return external.search(options, table) } else if (dbCore.isSqsEnabledForTenant()) { - return sqs.search(options, table) + return internal.sqs.search(options, table) } else { - return internal.search(options, table) + return internal.lucene.search(options, table) } } diff --git a/packages/server/src/sdk/app/rows/search/internal/index.ts b/packages/server/src/sdk/app/rows/search/internal/index.ts new file mode 100644 index 0000000000..f3db9169f4 --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/internal/index.ts @@ -0,0 +1,3 @@ +export * as sqs from "./sqs" +export * as lucene from "./lucene" +export * from "./internal" diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal/internal.ts similarity index 70% rename from packages/server/src/sdk/app/rows/search/internal.ts rename to packages/server/src/sdk/app/rows/search/internal/internal.ts index f86b041597..acc7033ce1 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal/internal.ts @@ -1,90 +1,30 @@ import { context, HTTPError } from "@budibase/backend-core" -import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" -import env from "../../../../environment" -import { fullSearch, paginatedSearch } from "./utils" -import { getRowParams, InternalTables } from "../../../../db/utils" +import env from "../../../../../environment" +import { getRowParams, InternalTables } from "../../../../../db/utils" import { Database, DocumentType, Row, - RowSearchParams, - SearchResponse, - SortType, Table, TableSchema, - User, } from "@budibase/types" -import { getGlobalUsersFromMetadata } from "../../../../utilities/global" -import { outputProcessing } from "../../../../utilities/rowProcessor" +import { outputProcessing } from "../../../../../utilities/rowProcessor" import { csv, Format, json, jsonWithSchema, -} from "../../../../api/controllers/view/exporters" -import * as inMemoryViews from "../../../../db/inMemoryView" +} from "../../../../../api/controllers/view/exporters" +import * as inMemoryViews from "../../../../../db/inMemoryView" import { getFromDesignDoc, getFromMemoryDoc, migrateToDesignView, migrateToInMemoryView, -} from "../../../../api/controllers/view/utils" -import sdk from "../../../../sdk" -import { ExportRowsParams, ExportRowsResult } from "./types" -import pick from "lodash/pick" -import { breakRowIdField } from "../../../../integrations/utils" - -export async function search( - options: RowSearchParams, - table: Table -): Promise> { - const { tableId } = options - - const { paginate, query } = options - - const params: RowSearchParams = { - tableId: options.tableId, - sort: options.sort, - sortOrder: options.sortOrder, - sortType: options.sortType, - limit: options.limit, - bookmark: options.bookmark, - version: options.version, - disableEscaping: options.disableEscaping, - query: {}, - } - - if (params.sort && !params.sortType) { - const schema = table.schema - const sortField = schema[params.sort] - params.sortType = - sortField.type === "number" ? SortType.NUMBER : SortType.STRING - } - - let response - if (paginate) { - response = await paginatedSearch(query, params) - } else { - response = await fullSearch(query, params) - } - - // Enrich search results with relationships - if (response.rows && response.rows.length) { - // enrich with global users if from users table - if (tableId === InternalTables.USER_METADATA) { - response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) - } - - if (options.fields) { - const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] - response.rows = response.rows.map((r: any) => pick(r, fields)) - } - - response.rows = await outputProcessing(table, response.rows) - } - - return response -} +} from "../../../../../api/controllers/view/utils" +import sdk from "../../../../../sdk" +import { ExportRowsParams, ExportRowsResult } from "../types" +import { breakRowIdField } from "../../../../../integrations/utils" export async function exportRows( options: ExportRowsParams @@ -123,15 +63,12 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { - let searchResponse = await search( - { - tableId, - query, - sort, - sortOrder, - }, - table - ) + let searchResponse = await sdk.rows.search({ + tableId, + query, + sort, + sortOrder, + }) result = searchResponse.rows } diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts new file mode 100644 index 0000000000..a25803804b --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -0,0 +1,66 @@ +import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" +import { fullSearch, paginatedSearch } from "../utils" +import { InternalTables } from "../../../../../db/utils" +import { + Row, + RowSearchParams, + SearchResponse, + SortType, + Table, + User, +} from "@budibase/types" +import { getGlobalUsersFromMetadata } from "../../../../../utilities/global" +import { outputProcessing } from "../../../../../utilities/rowProcessor" +import pick from "lodash/pick" + +export async function search( + options: RowSearchParams, + table: Table +): Promise> { + const { tableId } = options + + const { paginate, query } = options + + const params: RowSearchParams = { + tableId: options.tableId, + sort: options.sort, + sortOrder: options.sortOrder, + sortType: options.sortType, + limit: options.limit, + bookmark: options.bookmark, + version: options.version, + disableEscaping: options.disableEscaping, + query: {}, + } + + if (params.sort && !params.sortType) { + const schema = table.schema + const sortField = schema[params.sort] + params.sortType = + sortField.type === "number" ? SortType.NUMBER : SortType.STRING + } + + let response + if (paginate) { + response = await paginatedSearch(query, params) + } else { + response = await fullSearch(query, params) + } + + // Enrich search results with relationships + if (response.rows && response.rows.length) { + // enrich with global users if from users table + if (tableId === InternalTables.USER_METADATA) { + response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) + } + + if (options.fields) { + const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] + response.rows = response.rows.map((r: any) => pick(r, fields)) + } + + response.rows = await outputProcessing(table, response.rows) + } + + return response +} diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts similarity index 96% rename from packages/server/src/sdk/app/rows/search/sqs.ts rename to packages/server/src/sdk/app/rows/search/internal/sqs.ts index f6154cf70c..bc46e8d4c1 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -18,31 +18,31 @@ import { import { buildInternalRelationships, sqlOutputProcessing, -} from "../../../../api/controllers/row/utils" +} from "../../../../../api/controllers/row/utils" import { decodeNonAscii, mapToUserColumn, USER_COLUMN_PREFIX, -} from "../../tables/internal/sqs" -import sdk from "../../../index" +} from "../../../tables/internal/sqs" +import sdk from "../../../../index" import { context, sql, SQLITE_DESIGN_DOC_ID, SQS_DATASOURCE_INTERNAL, } from "@budibase/backend-core" -import { generateJunctionTableID } from "../../../../db/utils" -import AliasTables from "../sqlAlias" -import { outputProcessing } from "../../../../utilities/rowProcessor" +import { generateJunctionTableID } from "../../../../../db/utils" +import AliasTables from "../../sqlAlias" +import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" -import { processRowCountResponse } from "../utils" +import { processRowCountResponse } from "../../utils" import { updateFilterKeys, getRelationshipColumns, getTableIDList, -} from "./filters" +} from "../filters" import { dataFilters, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" -import { isSearchingByRowID } from "./utils" +import { isSearchingByRowID } from "../utils" import tracer from "dd-trace" const builder = new sql.Sql(SqlClient.SQL_LITE)