CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-06-02 02:25:17 +12:00
Code Issues Projects Releases Wiki Activity

Allow multipage searches and implement optional pagination to data providers

Browse source
This commit is contained in:
Andrew Kingston 2021-05-13 12:26:18 +01:00
parent b52ccb5558
commit da40086c0b
8 changed files with 241 additions and 418 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
packages/client/src/api/tables.js
View file

@ -17,24 +17,6 @@ export const fetchTableData = async tableId => {
return await enrichRows(rows, tableId)
}
/**
* Perform a mango query against an internal table
* @param {String} tableId - id of the table to search
* @param {Object} search - Mango Compliant search object
* @param {Object} pagination - the pagination controls
*/
export const searchTableData = async ({ tableId, search, pagination }) => {
const output = await API.post({
url: `/api/${tableId}/rows/search`,
body: {
query: search,
pagination,
},
})
output.rows = await enrichRows(output.rows, tableId)
return output
}
/**
* Searches a table using Lucene.
*/
@ -47,6 +29,7 @@ export const searchTable = async ({
sort,
sortOrder,
sortType,
paginate,
}) => {
if (!tableId || (!query && !raw)) {
return
@ -61,10 +44,11 @@ export const searchTable = async ({
sort,
sortOrder,
sortType,
paginate,
},
})
return {
...res,
rows: await enrichRows(res?.rows, tableId),
bookmark: res.bookmark,
}
}

40
packages/server/src/api/controllers/row.js
View file

@ -16,7 +16,6 @@ const {
const { FieldTypes } = require("../../constants")
const { isEqual } = require("lodash")
const { cloneDeep } = require("lodash/fp")
const { QueryBuilder, search } = require("./search/utils")
const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}`
@ -248,45 +247,6 @@ exports.fetchView = async function (ctx) {
}
}
exports.search = async function (ctx) {
const appId = ctx.appId
const db = new CouchDB(appId)
const {
query,
pagination: { pageSize = 10, bookmark },
} = ctx.request.body
const tableId = ctx.params.tableId
const queryBuilder = new QueryBuilder(appId)
.setLimit(pageSize)
.addTable(tableId)
if (bookmark) {
queryBuilder.setBookmark(bookmark)
}
let searchString
if (ctx.query && ctx.query.raw && ctx.query.raw !== "") {
searchString = queryBuilder.complete(query["RAW"])
} else {
// make all strings a starts with operation rather than pure equality
for (const [key, queryVal] of Object.entries(query)) {
if (typeof queryVal === "string") {
queryBuilder.addString(key, queryVal)
} else {
queryBuilder.addEqual(key, queryVal)
}
}
searchString = queryBuilder.complete()
}
const response = await search(searchString)
const table = await db.get(tableId)
ctx.body = {
rows: await outputProcessing(appId, table, response.rows),
bookmark: response.bookmark,
}
}
exports.fetchTableRows = async function (ctx) {
const appId = ctx.appId
const db = new CouchDB(appId)

47
packages/server/src/api/controllers/search/index.js
View file

@ -1,4 +1,4 @@
const { QueryBuilder, buildSearchUrl, search } = require("./utils")
const { fullSearch, paginatedSearch } = require("./utils")
const CouchDB = require("../../../db")
const { outputProcessing } = require("../../../utilities/rowProcessor")
@ -8,38 +8,45 @@ exports.rowSearch = async ctx => {
const {
bookmark,
query,
raw,
limit,
sort,
sortOrder,
sortType,
paginate,
} = ctx.request.body
const db = new CouchDB(appId)
let url
if (query) {
url = new QueryBuilder(
let response
const start = Date.now()
if (paginate) {
response = await paginatedSearch(
appId,
query,
bookmark,
limit,
tableId,
sort,
sortOrder,
sortType
sortType,
limit,
bookmark
)
.addTable(tableId)
.complete()
} else if (raw) {
url = buildSearchUrl({
} else {
response = await fullSearch(
appId,
query: raw,
bookmark,
})
query,
tableId,
sort,
sortOrder,
sortType,
limit
)
}
const response = await search(url)
const table = await db.get(tableId)
ctx.body = {
rows: await outputProcessing(appId, table, response.rows),
bookmark: response.bookmark,
const end = Date.now()
console.log("Time: " + (end - start) / 1000 + " ms")
if (response.rows && response.rows.length) {
const table = await db.get(tableId)
response.rows = await outputProcessing(appId, table, response.rows)
}
ctx.body = response
}

206
packages/server/src/api/controllers/search/utils.js
View file

@ -3,51 +3,12 @@ const { checkSlashesInUrl } = require("../../../utilities")
const env = require("../../../environment")
const fetch = require("node-fetch")
/**
* Given a set of inputs this will generate the URL which is to be sent to the search proxy in CouchDB.
* @param {string} appId The ID of the app which we will be searching within.
* @param {string} query The lucene query string which is to be used for searching.
* @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was
* returned with query for next set of search results.
* @param {number} limit The number of entries to return per query.
* @param {string} sort The column to sort by.
* @param {string} sortOrder The order to sort by. "ascending" or "descending".
* @param {string} sortType The type of sort to perform. "string" or "number".
* @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled.
* @return {string} The URL which a GET can be performed on to receive results.
*/
function buildSearchUrl({
appId,
query,
bookmark,
sort,
sortOrder,
sortType,
excludeDocs,
limit = 50,
}) {
let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search`
url += `/${SearchIndexes.ROWS}?q=${query}`
url += `&limit=${Math.min(limit, 200)}`
if (!excludeDocs) {
url += "&include_docs=true"
}
if (sort) {
const orderChar = sortOrder === "descending" ? "-" : ""
url += `&sort="${orderChar}${sort.replace(/ /, "_")}<${sortType}>"`
}
if (bookmark) {
url += `&bookmark=${bookmark}`
}
return checkSlashesInUrl(url)
}
const luceneEscape = value => {
return `${value}`.replace(/[ #+\-&|!(){}\[\]^"~*?:\\]/g, "\\$&")
}
class QueryBuilder {
constructor(appId, base, bookmark, limit, sort, sortOrder, sortType) {
constructor(appId, base) {
this.appId = appId
this.query = {
string: {},
@ -59,11 +20,14 @@ class QueryBuilder {
notEmpty: {},
...base,
}
this.bookmark = bookmark
this.limit = limit || 50
this.sort = sort
this.sortOrder = sortOrder || "ascending"
this.sortType = sortType || "string"
this.limit = 50
this.sortOrder = "ascending"
this.sortType = "string"
}
setTable(tableId) {
this.query.equal.tableId = tableId
return this
}
setLimit(limit) {
@ -71,6 +35,21 @@ class QueryBuilder {
return this
}
setSort(sort) {
this.sort = sort
return this
}
setSortOrder(sortOrder) {
this.sortOrder = sortOrder
return this
}
setSortType(sortType) {
this.sortType = sortType
return this
}
setBookmark(bookmark) {
this.bookmark = bookmark
return this
@ -114,12 +93,7 @@ class QueryBuilder {
return this
}
addTable(tableId) {
this.query.equal.tableId = tableId
return this
}
complete(rawQuery = null) {
buildSearchURL(excludeDocs = false) {
let output = "*:*"
function build(structure, queryFn) {
for (let [key, value] of Object.entries(structure)) {
@ -171,22 +145,28 @@ class QueryBuilder {
if (this.query.notEmpty) {
build(this.query.notEmpty, key => `${key}:["" TO *]`)
}
if (rawQuery) {
output = output.length === 0 ? rawQuery : `&${rawQuery}`
let url = `${env.COUCH_DB_URL}/${this.appId}/_design/database/_search`
url += `/${SearchIndexes.ROWS}?q=${output}`
url += `&limit=${Math.min(this.limit, 200)}`
if (!excludeDocs) {
url += "&include_docs=true"
}
return buildSearchUrl({
appId: this.appId,
query: output,
bookmark: this.bookmark,
limit: this.limit,
sort: this.sort,
sortOrder: this.sortOrder,
sortType: this.sortType,
})
if (this.sort) {
const orderChar = this.sortOrder === "descending" ? "-" : ""
url += `&sort="${orderChar}${this.sort.replace(/ /, "_")}<${
this.sortType
}>"`
}
if (this.bookmark) {
url += `&bookmark=${this.bookmark}`
}
console.log(url)
return checkSlashesInUrl(url)
}
}
exports.search = async query => {
const runQuery = async query => {
const response = await fetch(query, {
method: "GET",
})
@ -203,5 +183,101 @@ exports.search = async query => {
return output
}
exports.QueryBuilder = QueryBuilder
exports.buildSearchUrl = buildSearchUrl
const recursiveSearch = async (
appId,
query,
tableId,
sort,
sortOrder,
sortType,
limit,
bookmark,
rows
) => {
if (rows.length >= limit) {
return rows
}
const pageSize = rows.length > limit - 200 ? limit - rows.length : 200
const url = new QueryBuilder(appId, query)
.setTable(tableId)
.setBookmark(bookmark)
.setLimit(pageSize)
.setSort(sort)
.setSortOrder(sortOrder)
.setSortType(sortType)
.buildSearchURL()
const page = await runQuery(url)
if (!page.rows.length) {
return rows
}
if (page.rows.length < 200) {
return [...rows, ...page.rows]
}
return await recursiveSearch(
appId,
query,
tableId,
sort,
sortOrder,
sortType,
limit,
page.bookmark,
[...rows, ...page.rows]
)
}
exports.paginatedSearch = async (
appId,
query,
tableId,
sort,
sortOrder,
sortType,
limit,
bookmark
) => {
if (limit == null || isNaN(limit) || limit < 0) {
limit = 50
}
const builder = new QueryBuilder(appId, query)
.setTable(tableId)
.setSort(sort)
.setSortOrder(sortOrder)
.setSortType(sortType)
.setBookmark(bookmark)
.setLimit(limit)
const searchUrl = builder.buildSearchURL()
const nextUrl = builder.setLimit(1).buildSearchURL()
const searchResults = await runQuery(searchUrl)
const nextResults = await runQuery(nextUrl)
return {
...searchResults,
hasNextPage: nextResults.rows && nextResults.rows.length > 0,
}
}
exports.fullSearch = async (
appId,
query,
tableId,
sort,
sortOrder,
sortType,
limit
) => {
if (limit == null || isNaN(limit) || limit < 0) {
limit = 1000
}
const rows = await recursiveSearch(
appId,
query,
tableId,
sort,
sortOrder,
sortType,
Math.min(limit, 1000),
null,
[]
)
return { rows }
}

6
packages/server/src/api/routes/row.js
View file

@ -39,12 +39,6 @@ router
usage,
rowController.save
)
.post(
"/api/:tableId/rows/search",
paramResource("tableId"),
authorized(PermissionTypes.TABLE, PermissionLevels.READ),
rowController.search
)
.patch(
"/api/:tableId/rows/:rowId",
paramSubResource("tableId", "rowId"),

52
packages/standard-components/manifest.json
View file

@ -65,41 +65,6 @@
"type": "schema"
}
},
"search": {
"name": "Search",
"description": "A searchable list of items.",
"icon": "Search",
"styleable": true,
"hasChildren": true,
"settings": [
{
"type": "table",
"label": "Table",
"key": "table"
},
{
"type": "multifield",
"label": "Columns",
"key": "columns",
"dependsOn": "table"
},
{
"type": "number",
"label": "Rows/Page",
"defaultValue": 25,
"key": "pageSize"
},
{
"type": "text",
"label": "Empty Text",
"key": "noRowsMessage",
"defaultValue": "No rows found."
}
],
"context": {
"type": "schema"
}
},
"stackedlist": {
"name": "Stacked List",
"icon": "TaskList",
@ -1446,7 +1411,14 @@
{
"type": "number",
"label": "Limit",
"key": "limit"
"key": "limit",
"defaultValue": 50
},
{
"type": "boolean",
"label": "Paginate",
"key": "paginate",
"defaultValue": true
}
],
"context": {
@ -1464,14 +1436,6 @@
"label": "Schema",
"key": "schema"
},
{
"label": "Loading",
"key": "loading"
},
{
"label": "Loaded",
"key": "loaded"
},
{
"label": "Page Number",
"key": "pageNumber"

91
packages/standard-components/src/DataProvider.svelte
View file

@ -1,11 +1,13 @@
<script>
import { getContext } from "svelte"
import { ProgressCircle, Pagination } from "@budibase/bbui"
export let dataSource
export let filter
export let sortColumn
export let sortOrder
export let limit = 50
export let limit
export let paginate
const { API, styleable, Provider, ActionTypes } = getContext("sdk")
const component = getContext("component")
@ -29,7 +31,15 @@
$: hasPrevPage = pageNumber > 0
$: getSchema(dataSource)
$: sortType = getSortType(schema, sortColumn)
$: fetchData(dataSource, query, limit, sortColumn, sortOrder, sortType)
$: fetchData(
dataSource,
query,
limit,
sortColumn,
sortOrder,
sortType,
paginate
)
$: {
if (internalTable) {
rows = allRows
@ -110,8 +120,10 @@
limit,
sortColumn,
sortOrder,
sortType
sortType,
paginate
) => {
console.log("FETCH")
loading = true
if (dataSource?.type === "table") {
const res = await API.searchTable({
@ -121,21 +133,11 @@
sort: sortColumn,
sortOrder: sortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate,
})
pageNumber = 0
allRows = res.rows
// Check we have next data
const next = await API.searchTable({
tableId: dataSource.tableId,
query,
limit: 1,
bookmark: res.bookmark,
sort: sortColumn,
sortOrder: sortOrder?.toLowerCase() ?? "ascending",
sortType,
})
if (next.rows?.length) {
if (res.hasNextPage) {
bookmarks = [null, res.bookmark]
} else {
bookmarks = [null]
@ -224,21 +226,11 @@
sort: sortColumn,
sortOrder: sortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate: true,
})
pageNumber++
allRows = res.rows
// Check we have next data
const next = await API.searchTable({
tableId: dataSource.tableId,
query,
limit: 1,
bookmark: res.bookmark,
sort: sortColumn,
sortOrder: sortOrder?.toLowerCase() ?? "ascending",
sortType,
})
if (next.rows?.length) {
if (res.hasNextPage) {
bookmarks[pageNumber + 1] = res.bookmark
}
}
@ -255,14 +247,55 @@
sort: sortColumn,
sortOrder: sortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate: true,
})
pageNumber--
allRows = res.rows
}
</script>
<div use:styleable={$component.styles}>
<div use:styleable={$component.styles} class="container">
<Provider {actions} data={dataContext}>
<slot />
{#if !loaded && loading}
<div class="loading">
<ProgressCircle />
</div>
{:else}
<slot />
{#if paginate}
<div class="pagination">
<Pagination
page={pageNumber + 1}
{hasPrevPage}
{hasNextPage}
goToPrevPage={prevPage}
goToNextPage={nextPage}
/>
</div>
{/if}
{/if}
</Provider>
</div>
<style>
.container {
display: flex;
flex-direction: column;
justify-content: flex-start;
align-items: stretch;
}
.loading {
display: flex;
flex-direction: row;
justify-content: center;
align-items: center;
height: 100px;
}
.pagination {
display: flex;
flex-direction: row;
justify-content: flex-end;
align-items: center;
margin-top: var(--spacing-xl);
}
</style>

195
packages/standard-components/src/Search.svelte
View file

@ -1,195 +0,0 @@
<script>
import { getContext } from "svelte"
import {
Button,
DatePicker,
Label,
Select,
Toggle,
Input,
} from "@budibase/bbui"
const { API, styleable, Provider, builderStore, ActionTypes } = getContext(
"sdk"
)
const component = getContext("component")
export let table
export let columns = []
export let pageSize
export let noRowsMessage
let rows = []
let loaded = false
let search = {}
let tableDefinition
let schema
let nextBookmark = null
let bookmark = null
let lastBookmark = null
$: fetchData(table, bookmark)
// omit empty strings
$: parsedSearch = Object.keys(search).reduce(
(acc, next) =>
search[next] === "" ? acc : { ...acc, [next]: search[next] },
{}
)
$: actions = [
{
type: ActionTypes.RefreshDatasource,
callback: () => fetchData(table, bookmark),
metadata: { datasource: { type: "table", tableId: table } },
},
]
async function fetchData(table, mark) {
if (table) {
const tableDef = await API.fetchTableDefinition(table)
schema = tableDef.schema
const output = await API.searchTableData({
tableId: table,
search: parsedSearch,
pagination: {
pageSize,
bookmark: mark,
},
})
rows = output.rows
nextBookmark = output.bookmark
}
loaded = true
}
function nextPage() {
lastBookmark = bookmark
bookmark = nextBookmark
}
function previousPage() {
nextBookmark = bookmark
if (lastBookmark !== bookmark) {
bookmark = lastBookmark
} else {
// special case for going back to beginning
bookmark = null
lastBookmark = null
}
}
</script>
<Provider {actions}>
<div use:styleable={$component.styles}>
<div class="query-builder">
{#if schema}
{#each columns as field}
<div class="form-field">
<Label extraSmall grey>{schema[field].name}</Label>
{#if schema[field].type === "options"}
<Select secondary bind:value={search[field]}>
<option value="">Choose an option</option>
{#each schema[field].constraints.inclusion as opt}
<option>{opt}</option>
{/each}
</Select>
{:else if schema[field].type === "datetime"}
<DatePicker bind:value={search[field]} />
{:else if schema[field].type === "boolean"}
<Toggle text={schema[field].name} bind:checked={search[field]} />
{:else if schema[field].type === "number"}
<Input type="number" bind:value={search[field]} />
{:else if schema[field].type === "string"}
<Input bind:value={search[field]} />
{/if}
</div>
{/each}
{/if}
<div class="actions">
<Button
secondary
on:click={() => {
search = {}
bookmark = null
}}
>
Reset
</Button>
<Button
primary
on:click={() => {
bookmark = null
fetchData(table, bookmark)
}}
>
Search
</Button>
</div>
</div>
{#if loaded}
{#if rows.length > 0}
{#if $component.children === 0 && $builderStore.inBuilder}
<p><i class="ri-image-line" />Add some components to display.</p>
{:else}
{#each rows as row}
<Provider data={row}>
<slot />
</Provider>
{/each}
{/if}
{:else if noRowsMessage}
<p><i class="ri-search-2-line" />{noRowsMessage}</p>
{/if}
{/if}
<div class="pagination">
{#if lastBookmark != null || bookmark != null}
<Button primary on:click={previousPage}>Back</Button>
{/if}
{#if nextBookmark != null && rows.length !== 0}
<Button primary on:click={nextPage}>Next</Button>
{/if}
</div>
</div>
</Provider>
<style>
p {
margin: 0 var(--spacing-m);
background-color: var(--grey-2);
color: var(--grey-6);
font-size: var(--font-size-s);
padding: var(--spacing-l);
border-radius: var(--border-radius-s);
display: grid;
place-items: center;
}
p i {
margin-bottom: var(--spacing-m);
font-size: 1.5rem;
color: var(--grey-5);
}
.query-builder {
padding: var(--spacing-m);
border-radius: var(--border-radius-s);
}
.actions {
display: grid;
grid-gap: var(--spacing-s);
justify-content: flex-end;
grid-auto-flow: column;
}
.form-field {
margin-bottom: var(--spacing-m);
}
.pagination {
display: grid;
grid-gap: var(--spacing-s);
justify-content: flex-end;
margin-top: var(--spacing-m);
grid-auto-flow: column;
}
</style>