From 05ceee1cfee2e91e2a6110c8e6d768fbdfbeeb45 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 13 Feb 2023 17:13:59 +0000 Subject: [PATCH 1/2] Moving lucene handling to backend-core so that it can be used for other databases (outside row indexes). --- packages/backend-core/src/db/index.ts | 1 + packages/backend-core/src/db/lucene.ts | 563 ++++++++++++++++++ packages/backend-core/src/index.ts | 2 + .../src/api/controllers/row/internalSearch.ts | 527 +--------------- .../server/src/api/controllers/row/utils.ts | 3 +- packages/server/src/integrations/base/sql.ts | 4 +- .../server/src/integrations/base/utils.ts | 12 - packages/worker/src/api/routes/index.ts | 3 + 8 files changed, 578 insertions(+), 537 deletions(-) create mode 100644 packages/backend-core/src/db/lucene.ts delete mode 100644 packages/server/src/integrations/base/utils.ts diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 0d9f75fa18..a569b17b36 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -7,3 +7,4 @@ export { default as Replication } from "./Replication" // exports to support old export structure export * from "../constants/db" export { getGlobalDBName, baseGlobalDBName } from "../context" +export * from "./lucene" diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts new file mode 100644 index 0000000000..028750797f --- /dev/null +++ b/packages/backend-core/src/db/lucene.ts @@ -0,0 +1,563 @@ +import fetch from "node-fetch" +import { db as dbCore } from "../" +import { SearchFilters, Row } from "@budibase/types" + +const QUERY_START_REGEX = /\d[0-9]*:/g + +export type SearchParams = { + tableId: string + sort?: string + sortOrder?: string + sortType?: string + limit?: number + bookmark?: string + version?: string + rows?: Row[] +} + +export function removeKeyNumbering(key: any): string { + if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { + const parts = key.split(":") + // remove the number + parts.shift() + return parts.join(":") + } else { + return key + } +} + +/** + * Class to build lucene query URLs. + * Optionally takes a base lucene query object. + */ +export class QueryBuilder { + dbName: string + index: string + query: SearchFilters + limit: number + sort?: string + bookmark?: string + sortOrder: string + sortType: string + includeDocs: boolean + version?: string + + constructor(dbName: string, index: string, base?: SearchFilters) { + this.dbName = dbName + this.index = index + this.query = { + allOr: false, + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + oneOf: {}, + contains: {}, + notContains: {}, + containsAny: {}, + ...base, + } + this.limit = 50 + this.sortOrder = "ascending" + this.sortType = "string" + this.includeDocs = true + } + + setVersion(version?: string) { + if (version != null) { + this.version = version + } + return this + } + + setTable(tableId: string) { + this.query.equal!.tableId = tableId + return this + } + + setLimit(limit?: number) { + if (limit != null) { + this.limit = limit + } + return this + } + + setSort(sort?: string) { + if (sort != null) { + this.sort = sort + } + return this + } + + setSortOrder(sortOrder?: string) { + if (sortOrder != null) { + this.sortOrder = sortOrder + } + return this + } + + setSortType(sortType?: string) { + if (sortType != null) { + this.sortType = sortType + } + return this + } + + setBookmark(bookmark?: string) { + if (bookmark != null) { + this.bookmark = bookmark + } + return this + } + + excludeDocs() { + this.includeDocs = false + return this + } + + addString(key: string, partial: string) { + this.query.string![key] = partial + return this + } + + addFuzzy(key: string, fuzzy: string) { + this.query.fuzzy![key] = fuzzy + return this + } + + addRange(key: string, low: string | number, high: string | number) { + this.query.range![key] = { + low, + high, + } + return this + } + + addEqual(key: string, value: any) { + this.query.equal![key] = value + return this + } + + addNotEqual(key: string, value: any) { + this.query.notEqual![key] = value + return this + } + + addEmpty(key: string, value: any) { + this.query.empty![key] = value + return this + } + + addNotEmpty(key: string, value: any) { + this.query.notEmpty![key] = value + return this + } + + addOneOf(key: string, value: any) { + this.query.oneOf![key] = value + return this + } + + addContains(key: string, value: any) { + this.query.contains![key] = value + return this + } + + addNotContains(key: string, value: any) { + this.query.notContains![key] = value + return this + } + + addContainsAny(key: string, value: any) { + this.query.containsAny![key] = value + return this + } + + /** + * Preprocesses a value before going into a lucene search. + * Transforms strings to lowercase and wraps strings and bools in quotes. + * @param value The value to process + * @param options The preprocess options + * @returns {string|*} + */ + preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) { + const hasVersion = !!this.version + // Determine if type needs wrapped + const originalType = typeof value + // Convert to lowercase + if (value && lowercase) { + value = value.toLowerCase ? value.toLowerCase() : value + } + // Escape characters + if (escape && originalType === "string") { + value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&") + } + + // Wrap in quotes + if (originalType === "string" && !isNaN(value) && !type) { + value = `"${value}"` + } else if (hasVersion && wrap) { + value = originalType === "number" ? value : `"${value}"` + } + return value + } + + buildSearchQuery() { + const builder = this + let allOr = this.query && this.query.allOr + let query = allOr ? "" : "*:*" + const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } + let tableId + if (this.query.equal!.tableId) { + tableId = this.query.equal!.tableId + delete this.query.equal!.tableId + } + + const equal = (key: string, value: any) => { + // 0 evaluates to false, which means we would return all rows if we don't check it + if (!value && value !== 0) { + return null + } + return `${key}:${builder.preprocess(value, allPreProcessingOpts)}` + } + + const contains = (key: string, value: any, mode = "AND") => { + if (Array.isArray(value) && value.length === 0) { + return null + } + if (!Array.isArray(value)) { + return `${key}:${value}` + } + let statement = `${builder.preprocess(value[0], { escape: true })}` + for (let i = 1; i < value.length; i++) { + statement += ` ${mode} ${builder.preprocess(value[i], { + escape: true, + })}` + } + return `${key}:(${statement})` + } + + const notContains = (key: string, value: any) => { + // @ts-ignore + const allPrefix = allOr === "" ? "*:* AND" : "" + return allPrefix + "NOT " + contains(key, value) + } + + const containsAny = (key: string, value: any) => { + return contains(key, value, "OR") + } + + const oneOf = (key: string, value: any) => { + if (!Array.isArray(value)) { + if (typeof value === "string") { + value = value.split(",") + } else { + return "" + } + } + let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}` + for (let i = 1; i < value.length; i++) { + orStatement += ` OR ${builder.preprocess( + value[i], + allPreProcessingOpts + )}` + } + return `${key}:(${orStatement})` + } + + function build(structure: any, queryFn: any) { + for (let [key, value] of Object.entries(structure)) { + // check for new format - remove numbering if needed + key = removeKeyNumbering(key) + key = builder.preprocess(key.replace(/ /g, "_"), { + escape: true, + }) + const expression = queryFn(key, value) + if (expression == null) { + continue + } + if (query.length > 0) { + query += ` ${allOr ? "OR" : "AND"} ` + } + query += expression + } + } + + // Construct the actual lucene search query string from JSON structure + if (this.query.string) { + build(this.query.string, (key: string, value: any) => { + if (!value) { + return null + } + value = builder.preprocess(value, { + escape: true, + lowercase: true, + type: "string", + }) + return `${key}:${value}*` + }) + } + if (this.query.range) { + build(this.query.range, (key: string, value: any) => { + if (!value) { + return null + } + if (value.low == null || value.low === "") { + return null + } + if (value.high == null || value.high === "") { + return null + } + const low = builder.preprocess(value.low, allPreProcessingOpts) + const high = builder.preprocess(value.high, allPreProcessingOpts) + return `${key}:[${low} TO ${high}]` + }) + } + if (this.query.fuzzy) { + build(this.query.fuzzy, (key: string, value: any) => { + if (!value) { + return null + } + value = builder.preprocess(value, { + escape: true, + lowercase: true, + type: "fuzzy", + }) + return `${key}:${value}~` + }) + } + if (this.query.equal) { + build(this.query.equal, equal) + } + if (this.query.notEqual) { + build(this.query.notEqual, (key: string, value: any) => { + if (!value) { + return null + } + return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}` + }) + } + if (this.query.empty) { + build(this.query.empty, (key: string) => `!${key}:["" TO *]`) + } + if (this.query.notEmpty) { + build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`) + } + if (this.query.oneOf) { + build(this.query.oneOf, oneOf) + } + if (this.query.contains) { + build(this.query.contains, contains) + } + if (this.query.notContains) { + build(this.query.notContains, notContains) + } + if (this.query.containsAny) { + build(this.query.containsAny, containsAny) + } + // make sure table ID is always added as an AND + if (tableId) { + query = `(${query})` + allOr = false + build({ tableId }, equal) + } + return query + } + + buildSearchBody() { + let body: any = { + q: this.buildSearchQuery(), + limit: Math.min(this.limit, 200), + include_docs: this.includeDocs, + } + if (this.bookmark) { + body.bookmark = this.bookmark + } + if (this.sort) { + const order = this.sortOrder === "descending" ? "-" : "" + const type = `<${this.sortType}>` + body.sort = `${order}${this.sort.replace(/ /g, "_")}${type}` + } + return body + } + + async run() { + const { url, cookie } = dbCore.getCouchInfo() + const fullPath = `${url}/${this.dbName}/_design/database/_search/${this.index}` + const body = this.buildSearchBody() + return await runQuery(fullPath, body, cookie) + } +} + +/** + * Executes a lucene search query. + * @param url The query URL + * @param body The request body defining search criteria + * @param cookie The auth cookie for CouchDB + * @returns {Promise<{rows: []}>} + */ +const runQuery = async (url: string, body: any, cookie: string) => { + const response = await fetch(url, { + body: JSON.stringify(body), + method: "POST", + headers: { + Authorization: cookie, + }, + }) + const json = await response.json() + + let output: any = { + rows: [], + } + if (json.rows != null && json.rows.length > 0) { + output.rows = json.rows.map((row: any) => row.doc) + } + if (json.bookmark) { + output.bookmark = json.bookmark + } + return output +} + +/** + * Gets round the fixed limit of 200 results from a query by fetching as many + * pages as required and concatenating the results. This recursively operates + * until enough results have been found. + * @param dbName {string} Which database to run a lucene query on + * @param index {string} Which search index to utilise + * @param query {object} The JSON query structure + * @param params {object} The search params including: + * tableId {string} The table ID to search + * sort {string} The sort column + * sortOrder {string} The sort order ("ascending" or "descending") + * sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * limit {number} The number of results to fetch + * bookmark {string|null} Current bookmark in the recursive search + * rows {array|null} Current results in the recursive search + * @returns {Promise<*[]|*>} + */ +async function recursiveSearch( + dbName: string, + index: string, + query: any, + params: any +): Promise { + const bookmark = params.bookmark + const rows = params.rows || [] + if (rows.length >= params.limit) { + return rows + } + let pageSize = 200 + if (rows.length > params.limit - 200) { + pageSize = params.limit - rows.length + } + const page = await new QueryBuilder(dbName, index, query) + .setVersion(params.version) + .setTable(params.tableId) + .setBookmark(bookmark) + .setLimit(pageSize) + .setSort(params.sort) + .setSortOrder(params.sortOrder) + .setSortType(params.sortType) + .run() + if (!page.rows.length) { + return rows + } + if (page.rows.length < 200) { + return [...rows, ...page.rows] + } + const newParams = { + ...params, + bookmark: page.bookmark, + rows: [...rows, ...page.rows], + } + return await recursiveSearch(dbName, index, query, newParams) +} + +/** + * Performs a paginated search. A bookmark will be returned to allow the next + * page to be fetched. There is a max limit off 200 results per page in a + * paginated search. + * @param dbName {string} Which database to run a lucene query on + * @param index {string} Which search index to utilise + * @param query {object} The JSON query structure + * @param params {object} The search params including: + * tableId {string} The table ID to search + * sort {string} The sort column + * sortOrder {string} The sort order ("ascending" or "descending") + * sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * limit {number} The desired page size + * bookmark {string} The bookmark to resume from + * @returns {Promise<{hasNextPage: boolean, rows: *[]}>} + */ +export async function paginatedSearch( + dbName: string, + index: string, + query: SearchFilters, + params: SearchParams +) { + let limit = params.limit + if (limit == null || isNaN(limit) || limit < 0) { + limit = 50 + } + limit = Math.min(limit, 200) + const search = new QueryBuilder(dbName, index, query) + .setVersion(params.version) + .setTable(params.tableId) + .setSort(params.sort) + .setSortOrder(params.sortOrder) + .setSortType(params.sortType) + const searchResults = await search + .setBookmark(params.bookmark) + .setLimit(limit) + .run() + + // Try fetching 1 row in the next page to see if another page of results + // exists or not + const nextResults = await search + .setTable(params.tableId) + .setBookmark(searchResults.bookmark) + .setLimit(1) + .run() + + return { + ...searchResults, + hasNextPage: nextResults.rows && nextResults.rows.length > 0, + } +} + +/** + * Performs a full search, fetching multiple pages if required to return the + * desired amount of results. There is a limit of 1000 results to avoid + * heavy performance hits, and to avoid client components breaking from + * handling too much data. + * @param dbName {string} Which database to run a lucene query on + * @param index {string} Which search index to utilise + * @param query {object} The JSON query structure + * @param params {object} The search params including: + * tableId {string} The table ID to search + * sort {string} The sort column + * sortOrder {string} The sort order ("ascending" or "descending") + * sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * limit {number} The desired number of results + * @returns {Promise<{rows: *}>} + */ +export async function fullSearch( + dbName: string, + index: string, + query: SearchFilters, + params: SearchParams +) { + let limit = params.limit + if (limit == null || isNaN(limit) || limit < 0) { + limit = 1000 + } + params.limit = Math.min(limit, 1000) + const rows = await recursiveSearch(dbName, index, query, params) + return { rows } +} diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index b38a53e9e4..aa205e0317 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -25,6 +25,8 @@ export * as utils from "./utils" export * as errors from "./errors" export { default as env } from "./environment" +export { SearchParams } from "./db" + // expose error classes directly export * from "./errors" diff --git a/packages/server/src/api/controllers/row/internalSearch.ts b/packages/server/src/api/controllers/row/internalSearch.ts index 7068aabc5a..a250fe0429 100644 --- a/packages/server/src/api/controllers/row/internalSearch.ts +++ b/packages/server/src/api/controllers/row/internalSearch.ts @@ -1,531 +1,16 @@ import { SearchIndexes } from "../../../db/utils" -import { removeKeyNumbering } from "./utils" -import fetch from "node-fetch" -import { db as dbCore, context } from "@budibase/backend-core" -import { SearchFilters, Row } from "@budibase/types" +import { db as dbCore, context, SearchParams } from "@budibase/backend-core" +import { SearchFilters } from "@budibase/types" -type SearchParams = { - tableId: string - sort?: string - sortOrder?: string - sortType?: string - limit?: number - bookmark?: string - version?: string - rows?: Row[] -} - -/** - * Class to build lucene query URLs. - * Optionally takes a base lucene query object. - */ -export class QueryBuilder { - query: SearchFilters - limit: number - sort?: string - bookmark?: string - sortOrder: string - sortType: string - includeDocs: boolean - version?: string - - constructor(base?: SearchFilters) { - this.query = { - allOr: false, - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - oneOf: {}, - contains: {}, - notContains: {}, - containsAny: {}, - ...base, - } - this.limit = 50 - this.sortOrder = "ascending" - this.sortType = "string" - this.includeDocs = true - } - - setVersion(version?: string) { - if (version != null) { - this.version = version - } - return this - } - - setTable(tableId: string) { - this.query.equal!.tableId = tableId - return this - } - - setLimit(limit?: number) { - if (limit != null) { - this.limit = limit - } - return this - } - - setSort(sort?: string) { - if (sort != null) { - this.sort = sort - } - return this - } - - setSortOrder(sortOrder?: string) { - if (sortOrder != null) { - this.sortOrder = sortOrder - } - return this - } - - setSortType(sortType?: string) { - if (sortType != null) { - this.sortType = sortType - } - return this - } - - setBookmark(bookmark?: string) { - if (bookmark != null) { - this.bookmark = bookmark - } - return this - } - - excludeDocs() { - this.includeDocs = false - return this - } - - addString(key: string, partial: string) { - this.query.string![key] = partial - return this - } - - addFuzzy(key: string, fuzzy: string) { - this.query.fuzzy![key] = fuzzy - return this - } - - addRange(key: string, low: string | number, high: string | number) { - this.query.range![key] = { - low, - high, - } - return this - } - - addEqual(key: string, value: any) { - this.query.equal![key] = value - return this - } - - addNotEqual(key: string, value: any) { - this.query.notEqual![key] = value - return this - } - - addEmpty(key: string, value: any) { - this.query.empty![key] = value - return this - } - - addNotEmpty(key: string, value: any) { - this.query.notEmpty![key] = value - return this - } - - addOneOf(key: string, value: any) { - this.query.oneOf![key] = value - return this - } - - addContains(key: string, value: any) { - this.query.contains![key] = value - return this - } - - addNotContains(key: string, value: any) { - this.query.notContains![key] = value - return this - } - - addContainsAny(key: string, value: any) { - this.query.containsAny![key] = value - return this - } - - /** - * Preprocesses a value before going into a lucene search. - * Transforms strings to lowercase and wraps strings and bools in quotes. - * @param value The value to process - * @param options The preprocess options - * @returns {string|*} - */ - preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) { - const hasVersion = !!this.version - // Determine if type needs wrapped - const originalType = typeof value - // Convert to lowercase - if (value && lowercase) { - value = value.toLowerCase ? value.toLowerCase() : value - } - // Escape characters - if (escape && originalType === "string") { - value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&") - } - - // Wrap in quotes - if (originalType === "string" && !isNaN(value) && !type) { - value = `"${value}"` - } else if (hasVersion && wrap) { - value = originalType === "number" ? value : `"${value}"` - } - return value - } - - buildSearchQuery() { - const builder = this - let allOr = this.query && this.query.allOr - let query = allOr ? "" : "*:*" - const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } - let tableId - if (this.query.equal!.tableId) { - tableId = this.query.equal!.tableId - delete this.query.equal!.tableId - } - - const equal = (key: string, value: any) => { - // 0 evaluates to false, which means we would return all rows if we don't check it - if (!value && value !== 0) { - return null - } - return `${key}:${builder.preprocess(value, allPreProcessingOpts)}` - } - - const contains = (key: string, value: any, mode = "AND") => { - if (Array.isArray(value) && value.length === 0) { - return null - } - if (!Array.isArray(value)) { - return `${key}:${value}` - } - let statement = `${builder.preprocess(value[0], { escape: true })}` - for (let i = 1; i < value.length; i++) { - statement += ` ${mode} ${builder.preprocess(value[i], { - escape: true, - })}` - } - return `${key}:(${statement})` - } - - const notContains = (key: string, value: any) => { - // @ts-ignore - const allPrefix = allOr === "" ? "*:* AND" : "" - return allPrefix + "NOT " + contains(key, value) - } - - const containsAny = (key: string, value: any) => { - return contains(key, value, "OR") - } - - const oneOf = (key: string, value: any) => { - if (!Array.isArray(value)) { - if (typeof value === "string") { - value = value.split(",") - } else { - return "" - } - } - let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}` - for (let i = 1; i < value.length; i++) { - orStatement += ` OR ${builder.preprocess( - value[i], - allPreProcessingOpts - )}` - } - return `${key}:(${orStatement})` - } - - function build(structure: any, queryFn: any) { - for (let [key, value] of Object.entries(structure)) { - // check for new format - remove numbering if needed - key = removeKeyNumbering(key) - key = builder.preprocess(key.replace(/ /g, "_"), { - escape: true, - }) - const expression = queryFn(key, value) - if (expression == null) { - continue - } - if (query.length > 0) { - query += ` ${allOr ? "OR" : "AND"} ` - } - query += expression - } - } - - // Construct the actual lucene search query string from JSON structure - if (this.query.string) { - build(this.query.string, (key: string, value: any) => { - if (!value) { - return null - } - value = builder.preprocess(value, { - escape: true, - lowercase: true, - type: "string", - }) - return `${key}:${value}*` - }) - } - if (this.query.range) { - build(this.query.range, (key: string, value: any) => { - if (!value) { - return null - } - if (value.low == null || value.low === "") { - return null - } - if (value.high == null || value.high === "") { - return null - } - const low = builder.preprocess(value.low, allPreProcessingOpts) - const high = builder.preprocess(value.high, allPreProcessingOpts) - return `${key}:[${low} TO ${high}]` - }) - } - if (this.query.fuzzy) { - build(this.query.fuzzy, (key: string, value: any) => { - if (!value) { - return null - } - value = builder.preprocess(value, { - escape: true, - lowercase: true, - type: "fuzzy", - }) - return `${key}:${value}~` - }) - } - if (this.query.equal) { - build(this.query.equal, equal) - } - if (this.query.notEqual) { - build(this.query.notEqual, (key: string, value: any) => { - if (!value) { - return null - } - return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}` - }) - } - if (this.query.empty) { - build(this.query.empty, (key: string) => `!${key}:["" TO *]`) - } - if (this.query.notEmpty) { - build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`) - } - if (this.query.oneOf) { - build(this.query.oneOf, oneOf) - } - if (this.query.contains) { - build(this.query.contains, contains) - } - if (this.query.notContains) { - build(this.query.notContains, notContains) - } - if (this.query.containsAny) { - build(this.query.containsAny, containsAny) - } - // make sure table ID is always added as an AND - if (tableId) { - query = `(${query})` - allOr = false - build({ tableId }, equal) - } - return query - } - - buildSearchBody() { - let body: any = { - q: this.buildSearchQuery(), - limit: Math.min(this.limit, 200), - include_docs: this.includeDocs, - } - if (this.bookmark) { - body.bookmark = this.bookmark - } - if (this.sort) { - const order = this.sortOrder === "descending" ? "-" : "" - const type = `<${this.sortType}>` - body.sort = `${order}${this.sort.replace(/ /g, "_")}${type}` - } - return body - } - - async run() { - const appId = context.getAppId() - const { url, cookie } = dbCore.getCouchInfo() - const fullPath = `${url}/${appId}/_design/database/_search/${SearchIndexes.ROWS}` - const body = this.buildSearchBody() - return await runQuery(fullPath, body, cookie) - } -} - -/** - * Executes a lucene search query. - * @param url The query URL - * @param body The request body defining search criteria - * @param cookie The auth cookie for CouchDB - * @returns {Promise<{rows: []}>} - */ -const runQuery = async (url: string, body: any, cookie: string) => { - const response = await fetch(url, { - body: JSON.stringify(body), - method: "POST", - headers: { - Authorization: cookie, - }, - }) - const json = await response.json() - - let output: any = { - rows: [], - } - if (json.rows != null && json.rows.length > 0) { - output.rows = json.rows.map((row: any) => row.doc) - } - if (json.bookmark) { - output.bookmark = json.bookmark - } - return output -} - -/** - * Gets round the fixed limit of 200 results from a query by fetching as many - * pages as required and concatenating the results. This recursively operates - * until enough results have been found. - * @param query {object} The JSON query structure - * @param params {object} The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The number of results to fetch - * bookmark {string|null} Current bookmark in the recursive search - * rows {array|null} Current results in the recursive search - * @returns {Promise<*[]|*>} - */ -async function recursiveSearch(query: any, params: any): Promise { - const bookmark = params.bookmark - const rows = params.rows || [] - if (rows.length >= params.limit) { - return rows - } - let pageSize = 200 - if (rows.length > params.limit - 200) { - pageSize = params.limit - rows.length - } - const page = await new QueryBuilder(query) - .setVersion(params.version) - .setTable(params.tableId) - .setBookmark(bookmark) - .setLimit(pageSize) - .setSort(params.sort) - .setSortOrder(params.sortOrder) - .setSortType(params.sortType) - .run() - if (!page.rows.length) { - return rows - } - if (page.rows.length < 200) { - return [...rows, ...page.rows] - } - const newParams = { - ...params, - bookmark: page.bookmark, - rows: [...rows, ...page.rows], - } - return await recursiveSearch(query, newParams) -} - -/** - * Performs a paginated search. A bookmark will be returned to allow the next - * page to be fetched. There is a max limit off 200 results per page in a - * paginated search. - * @param query {object} The JSON query structure - * @param params {object} The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The desired page size - * bookmark {string} The bookmark to resume from - * @returns {Promise<{hasNextPage: boolean, rows: *[]}>} - */ export async function paginatedSearch( query: SearchFilters, params: SearchParams ) { - let limit = params.limit - if (limit == null || isNaN(limit) || limit < 0) { - limit = 50 - } - limit = Math.min(limit, 200) - const search = new QueryBuilder(query) - .setVersion(params.version) - .setTable(params.tableId) - .setSort(params.sort) - .setSortOrder(params.sortOrder) - .setSortType(params.sortType) - const searchResults = await search - .setBookmark(params.bookmark) - .setLimit(limit) - .run() - - // Try fetching 1 row in the next page to see if another page of results - // exists or not - const nextResults = await search - .setTable(params.tableId) - .setBookmark(searchResults.bookmark) - .setLimit(1) - .run() - - return { - ...searchResults, - hasNextPage: nextResults.rows && nextResults.rows.length > 0, - } + const appId = context.getAppId() + return dbCore.paginatedSearch(appId!, SearchIndexes.ROWS, query, params) } -/** - * Performs a full search, fetching multiple pages if required to return the - * desired amount of results. There is a limit of 1000 results to avoid - * heavy performance hits, and to avoid client components breaking from - * handling too much data. - * @param query {object} The JSON query structure - * @param params {object} The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The desired number of results - * @returns {Promise<{rows: *}>} - */ export async function fullSearch(query: SearchFilters, params: SearchParams) { - let limit = params.limit - if (limit == null || isNaN(limit) || limit < 0) { - limit = 1000 - } - params.limit = Math.min(limit, 1000) - const rows = await recursiveSearch(query, params) - return { rows } + const appId = context.getAppId() + return dbCore.fullSearch(appId!, SearchIndexes.ROWS, query, params) } diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index f0f1075205..82232b7f98 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -3,8 +3,7 @@ import * as userController from "../user" import { FieldTypes } from "../../../constants" import { context } from "@budibase/backend-core" import { makeExternalQuery } from "../../../integrations/base/query" -import { BBContext, Row, Table } from "@budibase/types" -export { removeKeyNumbering } from "../../../integrations/base/utils" +import { Row, Table } from "@budibase/types" const validateJs = require("validate.js") const { cloneDeep } = require("lodash/fp") import { Format } from "../view/exporters" diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index e42350091b..557fbbedfd 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -6,11 +6,11 @@ import { SearchFilters, SortDirection, } from "@budibase/types" +import { db as dbCore } from "@budibase/backend-core" import { QueryOptions } from "../../definitions/datasource" import { isIsoDateString, SqlClient } from "../utils" import SqlTableQueryBuilder from "./sqlTable" import environment from "../../environment" -import { removeKeyNumbering } from "./utils" const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) @@ -136,7 +136,7 @@ class InternalBuilder { fn: (key: string, value: any) => void ) { for (let [key, value] of Object.entries(structure)) { - const updatedKey = removeKeyNumbering(key) + const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { fn(`${opts.tableName}.${updatedKey}`, value) diff --git a/packages/server/src/integrations/base/utils.ts b/packages/server/src/integrations/base/utils.ts deleted file mode 100644 index 54efdb91a0..0000000000 --- a/packages/server/src/integrations/base/utils.ts +++ /dev/null @@ -1,12 +0,0 @@ -const QUERY_START_REGEX = /\d[0-9]*:/g - -export function removeKeyNumbering(key: any): string { - if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { - const parts = key.split(":") - // remove the number - parts.shift() - return parts.join(":") - } else { - return key - } -} diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index 3aa9422238..c64ad44423 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -18,6 +18,8 @@ import accountRoutes from "./system/accounts" import restoreRoutes from "./system/restore" let userGroupRoutes = api.groups +let auditLogRoutes = api.auditLogs + export const routes: Router[] = [ configRoutes, userRoutes, @@ -32,6 +34,7 @@ export const routes: Router[] = [ selfRoutes, licenseRoutes, userGroupRoutes, + auditLogRoutes, migrationRoutes, accountRoutes, restoreRoutes, From 46e9bf144393b62163984113cb23b9ed87a5aab0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 13 Feb 2023 18:16:13 +0000 Subject: [PATCH 2/2] Some updates to add in the audit log DB. --- packages/backend-core/src/constants/db.ts | 4 ++++ packages/backend-core/src/context/mainContext.ts | 14 ++++++++++++++ packages/types/src/api/web/global/auditLogs.ts | 6 ++++-- packages/types/src/documents/global/auditLogs.ts | 4 ++-- packages/types/src/documents/global/index.ts | 1 + 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index f7d15b3880..d41098c405 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -68,6 +68,7 @@ export enum DocumentType { MEM_VIEW = "view", USER_FLAG = "flag", AUTOMATION_METADATA = "meta_au", + AUDIT_LOG = "al", } export const StaticDatabases = { @@ -88,6 +89,9 @@ export const StaticDatabases = { install: "install", }, }, + AUDIT_LOGS: { + name: "audit-logs", + }, } export const APP_PREFIX = DocumentType.APP + SEPARATOR diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 9884d25d5a..1f14a20778 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -30,6 +30,13 @@ export function getGlobalDBName(tenantId?: string) { return baseGlobalDBName(tenantId) } +export function getAuditLogDBName(tenantId?: string) { + if (!tenantId) { + tenantId = getTenantId() + } + return `${tenantId}${SEPARATOR}${StaticDatabases.AUDIT_LOGS.name}` +} + export function baseGlobalDBName(tenantId: string | undefined | null) { let dbName if (!tenantId || tenantId === DEFAULT_TENANT_ID) { @@ -228,6 +235,13 @@ export function getGlobalDB(): Database { return getDB(baseGlobalDBName(context?.tenantId)) } +export function getAuditLogsDB(): Database { + if (!getTenantId()) { + throw new Error("Audit log DB not found") + } + return getDB(getAuditLogDBName()) +} + /** * Gets the app database based on whatever the request * contained, dev or prod. diff --git a/packages/types/src/api/web/global/auditLogs.ts b/packages/types/src/api/web/global/auditLogs.ts index 7281d0ca36..443cd79aa1 100644 --- a/packages/types/src/api/web/global/auditLogs.ts +++ b/packages/types/src/api/web/global/auditLogs.ts @@ -1,7 +1,7 @@ import { Event, AuditedEventFriendlyName } from "../../../sdk" import { PaginationResponse, PaginationRequest } from "../" -export interface DownloadAuditLogsRequest { +export interface AuditLogSearchParams { userId?: string[] appId?: string[] event?: Event[] @@ -10,9 +10,11 @@ export interface DownloadAuditLogsRequest { metadataSearch?: string } +export interface DownloadAuditLogsRequest extends AuditLogSearchParams {} + export interface SearchAuditLogsRequest extends PaginationRequest, - DownloadAuditLogsRequest {} + AuditLogSearchParams {} export interface SearchAuditLogsResponse extends PaginationResponse { data: { diff --git a/packages/types/src/documents/global/auditLogs.ts b/packages/types/src/documents/global/auditLogs.ts index bc07bc88d4..5b23650eec 100644 --- a/packages/types/src/documents/global/auditLogs.ts +++ b/packages/types/src/documents/global/auditLogs.ts @@ -1,8 +1,8 @@ import { Document } from "../document" import { Event } from "../../sdk" -export interface AuditLogDocument extends Document { - appId: string +export interface AuditLogDoc extends Document { + appId?: string event: Event userId: string timestamp: string diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts index 11ce7513f2..b728439dd6 100644 --- a/packages/types/src/documents/global/index.ts +++ b/packages/types/src/documents/global/index.ts @@ -6,3 +6,4 @@ export * from "./quotas" export * from "./schedule" export * from "./templates" export * from "./environmentVariables" +export * from "./auditLogs"