1
0
Fork 0
mirror of synced 2024-06-16 09:25:12 +12:00

Rewriting search to use the new couchdb 3.0 search functionality.

This commit is contained in:
mike12345567 2021-03-25 23:42:50 +00:00
parent 8ae0e6bf15
commit a5fd8d0e33
10 changed files with 240 additions and 61 deletions

View file

@ -21,14 +21,16 @@ export const fetchTableData = async tableId => {
* Perform a mango query against an internal table
* @param {String} tableId - id of the table to search
* @param {Object} search - Mango Compliant search object
* @param {Object} pagination - the pagination controls
*/
export const searchTableData = async ({ tableId, search, pagination }) => {
const rows = await API.post({
const output = await API.post({
url: `/api/${tableId}/rows/search`,
body: {
query: search,
pagination,
},
})
return await enrichRows(rows, tableId)
output.rows = await enrichRows(output.rows, tableId)
return output
}

View file

@ -5,7 +5,7 @@ const packageJson = require("../../../package.json")
const {
createLinkView,
createRoutingView,
createFulltextSearchIndex,
createAllSearchIndex,
} = require("../../db/views/staticViews")
const {
getTemplateStream,
@ -95,7 +95,7 @@ async function createInstance(template) {
// add view for linked rows
await createLinkView(appId)
await createRoutingView(appId)
await createFulltextSearchIndex(appId)
await createAllSearchIndex(appId)
// replicate the template data to the instance DB
// this is currently very hard to test, downloading and importing template files

View file

@ -17,6 +17,7 @@ const {
const { FieldTypes } = require("../../constants")
const { isEqual } = require("lodash")
const { cloneDeep } = require("lodash/fp")
const searchController = require("./search")
const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}`
@ -259,39 +260,40 @@ exports.search = async function(ctx) {
const db = new CouchDB(appId)
const {
query,
pagination: { pageSize = 10, page },
pagination: { pageSize = 10, bookmark },
} = ctx.request.body
const tableId = ctx.params.tableId
const queryBuilder = new searchController.QueryBuilder(appId)
.setLimit(pageSize)
.addTable(tableId)
if (bookmark) {
queryBuilder.setBookmark(bookmark)
}
// make all strings a starts with operation rather than pure equality
for (const [key, queryVal] of Object.entries(query)) {
if (typeof queryVal === "string") {
query[key] = {
$gt: queryVal,
$lt: `${queryVal}\uffff`,
}
queryBuilder.addString(key, queryVal)
} else {
queryBuilder.addEqual(key, queryVal)
}
}
// pure equality for table
query.tableId = ctx.params.tableId
const response = await db.find({
selector: query,
limit: pageSize,
skip: pageSize * page,
})
const rows = response.docs
const response = await searchController.search(queryBuilder.complete())
// delete passwords from users
if (query.tableId === ViewNames.USERS) {
for (let row of rows) {
if (tableId === ViewNames.USERS) {
for (let row of response.rows) {
delete row.password
}
}
const table = await db.get(ctx.params.tableId)
ctx.body = await outputProcessing(appId, table, rows)
const table = await db.get(tableId)
ctx.body = {
rows: await outputProcessing(appId, table, response.rows),
bookmark: response.bookmark,
}
}
exports.fetchTableRows = async function(ctx) {

View file

@ -0,0 +1,138 @@
const fetch = require("node-fetch")
const { SearchIndexes } = require("../../db/utils")
const { checkSlashesInUrl } = require("../../utilities")
const env = require("../../environment")
function buildSearchUrl(
appId,
query,
bookmark = null,
limit = 50,
includeDocs = true
) {
let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search`
url += `/${SearchIndexes.ROWS}?q=${query}`
if (includeDocs) {
url += "&include_docs=true"
}
if (limit) {
url += `&limit=${limit}`
}
if (bookmark) {
url += `&bookmark=${bookmark}`
}
return checkSlashesInUrl(url)
}
class QueryBuilder {
constructor(appId, base) {
this.appId = appId
this.query = {
string: {},
fuzzy: {},
range: {},
equal: {},
meta: {},
...base,
}
this.limit = 50
this.bookmark = null
}
setLimit(limit) {
this.limit = limit
return this
}
setBookmark(bookmark) {
this.bookmark = bookmark
return this
}
addString(key, partial) {
this.query.string[key] = partial
return this
}
addFuzzy(key, fuzzy) {
this.query.fuzzy[key] = fuzzy
return this
}
addRange(key, low, high) {
this.query.range = {
low,
high,
}
return this
}
addEqual(key, value) {
this.query.equal[key] = value
return this
}
addTable(tableId) {
this.query.equal.tableId = tableId
return this
}
complete() {
let output = ""
function build(structure, queryFn) {
for (let [key, value] of Object.entries(structure)) {
if (output.length !== 0) {
output += " AND "
}
output += queryFn(key, value)
}
}
if (this.query.string) {
build(this.query.string, (key, value) => `${key}:${value}*`)
}
if (this.query.number) {
build(this.query.number, (key, value) =>
value.length == null
? `${key}:${value}`
: `${key}:[${value[0]} TO ${value[1]}]`
)
}
if (this.query.fuzzy) {
build(this.query.fuzzy, (key, value) => `${key}:${value}~`)
}
return buildSearchUrl(this.appId, output, this.bookmark, this.limit)
}
}
exports.QueryBuilder = QueryBuilder
exports.search = async query => {
const response = await fetch(query, {
method: "GET",
})
const json = await response.json()
let output = {
rows: [],
}
if (json.rows != null && json.rows.length > 0) {
output.rows = json.rows.map(row => row.doc)
}
if (json.bookmark) {
output.bookmark = json.bookmark
}
return output
}
exports.rowSearch = async ctx => {
// this can't be done through pouch, have to reach for trusty node-fetch
const appId = ctx.user.appId
const bookmark = ctx.params.bookmark
let url
if (ctx.params.query) {
url = new QueryBuilder(appId, ctx.params.query, bookmark).complete()
} else if (ctx.params.raw) {
url = buildSearchUrl(appId, ctx.params.raw, bookmark)
}
ctx.body = await exports.search(url)
}

View file

@ -2,6 +2,7 @@ require("svelte/register")
const send = require("koa-send")
const { resolve, join } = require("../../../utilities/centralPath")
const { checkSlashesInUrl } = require("../../../utilities")
const fetch = require("node-fetch")
const uuid = require("uuid")
const { prepareUpload } = require("../deploy/utils")
@ -28,10 +29,7 @@ function objectStoreUrl() {
function internalObjectStoreUrl() {
if (env.SELF_HOSTED) {
return (env.MINIO_URL + OBJ_STORE_DIRECTORY).replace(
/(https?:\/\/)|(\/)+/g,
"$1$2"
)
return checkSlashesInUrl(env.MINIO_URL + OBJ_STORE_DIRECTORY)
} else {
return BB_CDN
}

View file

@ -0,0 +1,8 @@
const Router = require("@koa/router")
const controller = require("../controllers/search")
const router = Router()
router.get("/api/search/rows", controller.rowSearch)
module.exports = router

View file

@ -37,11 +37,16 @@ const ViewNames = {
USERS: "ta_users",
}
const SearchIndexes = {
ROWS: "rows",
}
exports.StaticDatabases = StaticDatabases
exports.ViewNames = ViewNames
exports.DocumentTypes = DocumentTypes
exports.SEPARATOR = SEPARATOR
exports.UNICODE_MAX = UNICODE_MAX
exports.SearchIndexes = SearchIndexes
exports.getQueryIndex = viewName => {
return `database/${viewName}`

View file

@ -1,5 +1,10 @@
const CouchDB = require("../index")
const { DocumentTypes, SEPARATOR, ViewNames } = require("../utils")
const {
DocumentTypes,
SEPARATOR,
ViewNames,
SearchIndexes,
} = require("../utils")
const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
/**************************************************
@ -73,30 +78,41 @@ exports.createRoutingView = async appId => {
await db.put(designDoc)
}
exports.createFulltextSearchIndex = async appId => {
async function searchIndex(appId, indexName, fnString) {
const db = new CouchDB(appId)
const designDoc = await db.get("_design/database")
designDoc.indexes = {
rows: {
index: function(doc) {
// eslint-disable-next-line no-undef
index("id", doc._id)
function idx(obj, prev = "") {
for (let key of Object.keys(obj)) {
let prevKey = prev !== "" ? `${prev}.${key}` : key
if (typeof obj[key] !== "object") {
// eslint-disable-next-line no-undef
index(prevKey, obj[key], { store: true })
} else {
idx(obj[key], prevKey)
}
}
}
if (doc._id.startsWith("ro_")) {
idx(doc)
}
}.toString(),
[indexName]: {
index: fnString,
},
}
await db.put(designDoc)
}
exports.createAllSearchIndex = async appId => {
await searchIndex(
appId,
SearchIndexes.ROWS,
function(doc) {
function idx(input, prev) {
for (let key of Object.keys(input)) {
const idxKey = prev != null ? `${prev}.${key}` : key
if (key === "_id" || key === "_rev") {
continue
}
if (typeof input[key] !== "object") {
// eslint-disable-next-line no-undef
index(idxKey, input[key], { store: true })
} else {
idx(input[key], idxKey)
}
}
}
if (doc._id.startsWith("ro_")) {
// eslint-disable-next-line no-undef
index("default", doc._id)
idx(doc)
}
}.toString()
)
}

View file

@ -106,3 +106,7 @@ exports.getAllApps = async () => {
.map(({ value }) => value)
}
}
exports.checkSlashesInUrl = url => {
return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
}

View file

@ -25,10 +25,11 @@
let tableDefinition
let schema
// pagination
let page = 0
let nextBookmark = null
let bookmark = null
let lastBookmark = null
$: fetchData(table, page)
$: fetchData(table, bookmark)
// omit empty strings
$: parsedSearch = Object.keys(search).reduce(
(acc, next) =>
@ -38,33 +39,38 @@
$: actions = [
{
type: ActionTypes.RefreshDatasource,
callback: () => fetchData(table, page),
callback: () => fetchData(table, bookmark),
metadata: { datasource: { type: "table", tableId: table } },
},
]
async function fetchData(table, page) {
async function fetchData(table, mark) {
if (table) {
const tableDef = await API.fetchTableDefinition(table)
schema = tableDef.schema
rows = await API.searchTableData({
lastBookmark = mark
const output = await API.searchTableData({
tableId: table,
search: parsedSearch,
pagination: {
pageSize,
page,
bookmark: mark,
},
})
rows = output.rows
nextBookmark = output.bookmark
}
loaded = true
}
function nextPage() {
page += 1
lastBookmark = bookmark
bookmark = nextBookmark
}
function previousPage() {
page -= 1
nextBookmark = bookmark
bookmark = lastBookmark
}
</script>
@ -99,15 +105,15 @@
secondary
on:click={() => {
search = {}
page = 0
bookmark = null
}}>
Reset
</Button>
<Button
primary
on:click={() => {
page = 0
fetchData(table, page)
bookmark = null
fetchData(table, bookmark)
}}>
Search
</Button>
@ -129,7 +135,7 @@
{/if}
{/if}
<div class="pagination">
{#if page > 0}
{#if bookmark != null}
<Button primary on:click={previousPage}>Back</Button>
{/if}
{#if rows.length === pageSize}