From a5fd8d0e335e50cb4289d9cc6ee78ba8a352f962 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 25 Mar 2021 23:42:50 +0000 Subject: [PATCH] Rewriting search to use the new couchdb 3.0 search functionality. --- packages/client/src/api/tables.js | 6 +- .../server/src/api/controllers/application.js | 4 +- packages/server/src/api/controllers/row.js | 40 ++--- packages/server/src/api/controllers/search.js | 138 ++++++++++++++++++ .../src/api/controllers/static/index.js | 6 +- packages/server/src/api/routes/search.js | 8 + packages/server/src/db/utils.js | 5 + packages/server/src/db/views/staticViews.js | 58 +++++--- packages/server/src/utilities/index.js | 4 + .../standard-components/src/Search.svelte | 32 ++-- 10 files changed, 240 insertions(+), 61 deletions(-) create mode 100644 packages/server/src/api/controllers/search.js create mode 100644 packages/server/src/api/routes/search.js diff --git a/packages/client/src/api/tables.js b/packages/client/src/api/tables.js index ce06019b54..248e1516c2 100644 --- a/packages/client/src/api/tables.js +++ b/packages/client/src/api/tables.js @@ -21,14 +21,16 @@ export const fetchTableData = async tableId => { * Perform a mango query against an internal table * @param {String} tableId - id of the table to search * @param {Object} search - Mango Compliant search object + * @param {Object} pagination - the pagination controls */ export const searchTableData = async ({ tableId, search, pagination }) => { - const rows = await API.post({ + const output = await API.post({ url: `/api/${tableId}/rows/search`, body: { query: search, pagination, }, }) - return await enrichRows(rows, tableId) + output.rows = await enrichRows(output.rows, tableId) + return output } diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 59cdb58904..00678b85a0 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -5,7 +5,7 @@ const packageJson = require("../../../package.json") const { createLinkView, createRoutingView, - createFulltextSearchIndex, + createAllSearchIndex, } = require("../../db/views/staticViews") const { getTemplateStream, @@ -95,7 +95,7 @@ async function createInstance(template) { // add view for linked rows await createLinkView(appId) await createRoutingView(appId) - await createFulltextSearchIndex(appId) + await createAllSearchIndex(appId) // replicate the template data to the instance DB // this is currently very hard to test, downloading and importing template files diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index bf985fe55d..1f5c410aaf 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -17,6 +17,7 @@ const { const { FieldTypes } = require("../../constants") const { isEqual } = require("lodash") const { cloneDeep } = require("lodash/fp") +const searchController = require("./search") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -259,39 +260,40 @@ exports.search = async function(ctx) { const db = new CouchDB(appId) const { query, - pagination: { pageSize = 10, page }, + pagination: { pageSize = 10, bookmark }, } = ctx.request.body + const tableId = ctx.params.tableId + + const queryBuilder = new searchController.QueryBuilder(appId) + .setLimit(pageSize) + .addTable(tableId) + if (bookmark) { + queryBuilder.setBookmark(bookmark) + } // make all strings a starts with operation rather than pure equality for (const [key, queryVal] of Object.entries(query)) { if (typeof queryVal === "string") { - query[key] = { - $gt: queryVal, - $lt: `${queryVal}\uffff`, - } + queryBuilder.addString(key, queryVal) + } else { + queryBuilder.addEqual(key, queryVal) } } - // pure equality for table - query.tableId = ctx.params.tableId - const response = await db.find({ - selector: query, - limit: pageSize, - skip: pageSize * page, - }) - - const rows = response.docs + const response = await searchController.search(queryBuilder.complete()) // delete passwords from users - if (query.tableId === ViewNames.USERS) { - for (let row of rows) { + if (tableId === ViewNames.USERS) { + for (let row of response.rows) { delete row.password } } - const table = await db.get(ctx.params.tableId) - - ctx.body = await outputProcessing(appId, table, rows) + const table = await db.get(tableId) + ctx.body = { + rows: await outputProcessing(appId, table, response.rows), + bookmark: response.bookmark, + } } exports.fetchTableRows = async function(ctx) { diff --git a/packages/server/src/api/controllers/search.js b/packages/server/src/api/controllers/search.js new file mode 100644 index 0000000000..0c60ebe5e6 --- /dev/null +++ b/packages/server/src/api/controllers/search.js @@ -0,0 +1,138 @@ +const fetch = require("node-fetch") +const { SearchIndexes } = require("../../db/utils") +const { checkSlashesInUrl } = require("../../utilities") +const env = require("../../environment") + +function buildSearchUrl( + appId, + query, + bookmark = null, + limit = 50, + includeDocs = true +) { + let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search` + url += `/${SearchIndexes.ROWS}?q=${query}` + if (includeDocs) { + url += "&include_docs=true" + } + if (limit) { + url += `&limit=${limit}` + } + if (bookmark) { + url += `&bookmark=${bookmark}` + } + return checkSlashesInUrl(url) +} + +class QueryBuilder { + constructor(appId, base) { + this.appId = appId + this.query = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + meta: {}, + ...base, + } + this.limit = 50 + this.bookmark = null + } + + setLimit(limit) { + this.limit = limit + return this + } + + setBookmark(bookmark) { + this.bookmark = bookmark + return this + } + + addString(key, partial) { + this.query.string[key] = partial + return this + } + + addFuzzy(key, fuzzy) { + this.query.fuzzy[key] = fuzzy + return this + } + + addRange(key, low, high) { + this.query.range = { + low, + high, + } + return this + } + + addEqual(key, value) { + this.query.equal[key] = value + return this + } + + addTable(tableId) { + this.query.equal.tableId = tableId + return this + } + + complete() { + let output = "" + function build(structure, queryFn) { + for (let [key, value] of Object.entries(structure)) { + if (output.length !== 0) { + output += " AND " + } + output += queryFn(key, value) + } + } + + if (this.query.string) { + build(this.query.string, (key, value) => `${key}:${value}*`) + } + if (this.query.number) { + build(this.query.number, (key, value) => + value.length == null + ? `${key}:${value}` + : `${key}:[${value[0]} TO ${value[1]}]` + ) + } + if (this.query.fuzzy) { + build(this.query.fuzzy, (key, value) => `${key}:${value}~`) + } + return buildSearchUrl(this.appId, output, this.bookmark, this.limit) + } +} + +exports.QueryBuilder = QueryBuilder + +exports.search = async query => { + const response = await fetch(query, { + method: "GET", + }) + const json = await response.json() + let output = { + rows: [], + } + if (json.rows != null && json.rows.length > 0) { + output.rows = json.rows.map(row => row.doc) + } + if (json.bookmark) { + output.bookmark = json.bookmark + } + return output +} + +exports.rowSearch = async ctx => { + // this can't be done through pouch, have to reach for trusty node-fetch + const appId = ctx.user.appId + const bookmark = ctx.params.bookmark + let url + if (ctx.params.query) { + url = new QueryBuilder(appId, ctx.params.query, bookmark).complete() + } else if (ctx.params.raw) { + url = buildSearchUrl(appId, ctx.params.raw, bookmark) + } + ctx.body = await exports.search(url) +} diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index c866db3561..7caf6d0f7f 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -2,6 +2,7 @@ require("svelte/register") const send = require("koa-send") const { resolve, join } = require("../../../utilities/centralPath") +const { checkSlashesInUrl } = require("../../../utilities") const fetch = require("node-fetch") const uuid = require("uuid") const { prepareUpload } = require("../deploy/utils") @@ -28,10 +29,7 @@ function objectStoreUrl() { function internalObjectStoreUrl() { if (env.SELF_HOSTED) { - return (env.MINIO_URL + OBJ_STORE_DIRECTORY).replace( - /(https?:\/\/)|(\/)+/g, - "$1$2" - ) + return checkSlashesInUrl(env.MINIO_URL + OBJ_STORE_DIRECTORY) } else { return BB_CDN } diff --git a/packages/server/src/api/routes/search.js b/packages/server/src/api/routes/search.js new file mode 100644 index 0000000000..8858a72d6e --- /dev/null +++ b/packages/server/src/api/routes/search.js @@ -0,0 +1,8 @@ +const Router = require("@koa/router") +const controller = require("../controllers/search") + +const router = Router() + +router.get("/api/search/rows", controller.rowSearch) + +module.exports = router diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index e480d4f554..4c31f0398e 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -37,11 +37,16 @@ const ViewNames = { USERS: "ta_users", } +const SearchIndexes = { + ROWS: "rows", +} + exports.StaticDatabases = StaticDatabases exports.ViewNames = ViewNames exports.DocumentTypes = DocumentTypes exports.SEPARATOR = SEPARATOR exports.UNICODE_MAX = UNICODE_MAX +exports.SearchIndexes = SearchIndexes exports.getQueryIndex = viewName => { return `database/${viewName}` diff --git a/packages/server/src/db/views/staticViews.js b/packages/server/src/db/views/staticViews.js index ae6496bb4a..305d042217 100644 --- a/packages/server/src/db/views/staticViews.js +++ b/packages/server/src/db/views/staticViews.js @@ -1,5 +1,10 @@ const CouchDB = require("../index") -const { DocumentTypes, SEPARATOR, ViewNames } = require("../utils") +const { + DocumentTypes, + SEPARATOR, + ViewNames, + SearchIndexes, +} = require("../utils") const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR /************************************************** @@ -73,30 +78,41 @@ exports.createRoutingView = async appId => { await db.put(designDoc) } -exports.createFulltextSearchIndex = async appId => { +async function searchIndex(appId, indexName, fnString) { const db = new CouchDB(appId) const designDoc = await db.get("_design/database") designDoc.indexes = { - rows: { - index: function(doc) { - // eslint-disable-next-line no-undef - index("id", doc._id) - function idx(obj, prev = "") { - for (let key of Object.keys(obj)) { - let prevKey = prev !== "" ? `${prev}.${key}` : key - if (typeof obj[key] !== "object") { - // eslint-disable-next-line no-undef - index(prevKey, obj[key], { store: true }) - } else { - idx(obj[key], prevKey) - } - } - } - if (doc._id.startsWith("ro_")) { - idx(doc) - } - }.toString(), + [indexName]: { + index: fnString, }, } await db.put(designDoc) } + +exports.createAllSearchIndex = async appId => { + await searchIndex( + appId, + SearchIndexes.ROWS, + function(doc) { + function idx(input, prev) { + for (let key of Object.keys(input)) { + const idxKey = prev != null ? `${prev}.${key}` : key + if (key === "_id" || key === "_rev") { + continue + } + if (typeof input[key] !== "object") { + // eslint-disable-next-line no-undef + index(idxKey, input[key], { store: true }) + } else { + idx(input[key], idxKey) + } + } + } + if (doc._id.startsWith("ro_")) { + // eslint-disable-next-line no-undef + index("default", doc._id) + idx(doc) + } + }.toString() + ) +} diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index 7d6794b1b3..ad92987434 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -106,3 +106,7 @@ exports.getAllApps = async () => { .map(({ value }) => value) } } + +exports.checkSlashesInUrl = url => { + return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2") +} diff --git a/packages/standard-components/src/Search.svelte b/packages/standard-components/src/Search.svelte index 509205f8f1..fc09070a87 100644 --- a/packages/standard-components/src/Search.svelte +++ b/packages/standard-components/src/Search.svelte @@ -25,10 +25,11 @@ let tableDefinition let schema - // pagination - let page = 0 + let nextBookmark = null + let bookmark = null + let lastBookmark = null - $: fetchData(table, page) + $: fetchData(table, bookmark) // omit empty strings $: parsedSearch = Object.keys(search).reduce( (acc, next) => @@ -38,33 +39,38 @@ $: actions = [ { type: ActionTypes.RefreshDatasource, - callback: () => fetchData(table, page), + callback: () => fetchData(table, bookmark), metadata: { datasource: { type: "table", tableId: table } }, }, ] - async function fetchData(table, page) { + async function fetchData(table, mark) { if (table) { const tableDef = await API.fetchTableDefinition(table) schema = tableDef.schema - rows = await API.searchTableData({ + lastBookmark = mark + const output = await API.searchTableData({ tableId: table, search: parsedSearch, pagination: { pageSize, - page, + bookmark: mark, }, }) + rows = output.rows + nextBookmark = output.bookmark } loaded = true } function nextPage() { - page += 1 + lastBookmark = bookmark + bookmark = nextBookmark } function previousPage() { - page -= 1 + nextBookmark = bookmark + bookmark = lastBookmark } @@ -99,15 +105,15 @@ secondary on:click={() => { search = {} - page = 0 + bookmark = null }}> Reset @@ -129,7 +135,7 @@ {/if} {/if}