From 717b0b0af6019cb737dc5df1c196e3a2cdf2dc4e Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 29 Apr 2021 14:55:33 +0100 Subject: [PATCH 01/37] Expose search API endpoint --- .../PropertyControls/FilterEditor/FilterBuilder.svelte | 0 .../PropertyControls/{ => FilterEditor}/FilterEditor.svelte | 0 packages/server/src/api/routes/index.js | 2 ++ 3 files changed, 2 insertions(+) create mode 100644 packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte rename packages/builder/src/components/design/PropertiesPanel/PropertyControls/{ => FilterEditor}/FilterEditor.svelte (100%) diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte similarity index 100% rename from packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor.svelte rename to packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte diff --git a/packages/server/src/api/routes/index.js b/packages/server/src/api/routes/index.js index 0b09a78bb8..5ea3ddacef 100644 --- a/packages/server/src/api/routes/index.js +++ b/packages/server/src/api/routes/index.js @@ -23,6 +23,7 @@ const queryRoutes = require("./query") const hostingRoutes = require("./hosting") const backupRoutes = require("./backup") const devRoutes = require("./dev") +const searchRoutes = require("./search") exports.mainRoutes = [ authRoutes, @@ -51,6 +52,7 @@ exports.mainRoutes = [ // this could be breaking as koa may recognise other routes as this tableRoutes, rowRoutes, + searchRoutes, ] exports.staticRoutes = staticRoutes From bd8269619c2501477b40c37a787a50882f7cc958 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 29 Apr 2021 14:55:51 +0100 Subject: [PATCH 02/37] Support NOT lucene queries and escape whitespace --- packages/server/src/api/controllers/search/utils.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js index d3ffb26be7..e8e26c813b 100644 --- a/packages/server/src/api/controllers/search/utils.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -34,6 +34,7 @@ class QueryBuilder { fuzzy: {}, range: {}, equal: {}, + notEqual: {}, ...base, } this.limit = 50 @@ -73,6 +74,11 @@ class QueryBuilder { return this } + addNotEqual(key, value) { + this.query.notEqual[key] = value + return this + } + addTable(tableId) { this.query.equal.tableId = tableId return this @@ -85,7 +91,7 @@ class QueryBuilder { if (output.length !== 0) { output += " AND " } - output += queryFn(key, value) + output += queryFn(key, value).replace(/ /, "\\ ") } } @@ -104,6 +110,9 @@ class QueryBuilder { if (this.query.equal) { build(this.query.equal, (key, value) => `${key}:${value}`) } + if (this.query.notEqual) { + build(this.query.notEqual, (key, value) => `!${key}:${value}`) + } if (rawQuery) { output = output.length === 0 ? rawQuery : `&${rawQuery}` } From b5ee768cb1f304d4bb5b9700388788288c46fd51 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 29 Apr 2021 15:24:59 +0100 Subject: [PATCH 03/37] Fix combobox disabled state and remove unnecessary props --- packages/bbui/src/Form/Checkbox.svelte | 2 +- packages/bbui/src/Form/Combobox.svelte | 4 ++-- packages/bbui/src/Form/Core/Combobox.svelte | 21 ++++++++++++++++----- packages/bbui/src/Form/DatePicker.svelte | 2 +- packages/bbui/src/Form/Dropzone.svelte | 2 +- packages/bbui/src/Form/Field.svelte | 1 - packages/bbui/src/Form/Input.svelte | 2 +- packages/bbui/src/Form/Multiselect.svelte | 2 +- packages/bbui/src/Form/RadioGroup.svelte | 2 +- packages/bbui/src/Form/Search.svelte | 2 +- packages/bbui/src/Form/Select.svelte | 2 +- packages/bbui/src/Form/TextArea.svelte | 2 +- packages/bbui/src/Form/Toggle.svelte | 2 +- 13 files changed, 28 insertions(+), 18 deletions(-) diff --git a/packages/bbui/src/Form/Checkbox.svelte b/packages/bbui/src/Form/Checkbox.svelte index 1f3e439c2a..90a2cddda5 100644 --- a/packages/bbui/src/Form/Checkbox.svelte +++ b/packages/bbui/src/Form/Checkbox.svelte @@ -17,6 +17,6 @@ } - + diff --git a/packages/bbui/src/Form/Combobox.svelte b/packages/bbui/src/Form/Combobox.svelte index 6b1e67a299..380465792b 100644 --- a/packages/bbui/src/Form/Combobox.svelte +++ b/packages/bbui/src/Form/Combobox.svelte @@ -8,7 +8,7 @@ export let disabled = false export let labelPosition = "above" export let error = null - export let placeholder = "Choose an option" + export let placeholder = "Choose an option or type" export let options = [] export let getOptionLabel = option => extractProperty(option, "label") export let getOptionValue = option => extractProperty(option, "value") @@ -26,7 +26,7 @@ } - + { // Always use placeholder if no value if (value == null || value === "") { - return placeholder || "Choose an option" + return placeholder || "Choose an option or type" } // Wait for options to load if there is a value but no options @@ -43,12 +43,19 @@ const onChange = e => { selectOption(e.target.value) } + + $: console.log(disabled) -
+
(focus = false)} on:change={onChange} {value} + {disabled} {placeholder} class="spectrum-Textfield-input spectrum-InputGroup-input" />
@@ -63,7 +71,7 @@ class="spectrum-Picker spectrum-Picker--sizeM spectrum-InputGroup-button" tabindex="-1" aria-haspopup="true" - disabled={!!error} + {disabled} on:click={() => (open = true)}> - + - + diff --git a/packages/bbui/src/Form/Input.svelte b/packages/bbui/src/Form/Input.svelte index e4a155d342..5059eb2543 100644 --- a/packages/bbui/src/Form/Input.svelte +++ b/packages/bbui/src/Form/Input.svelte @@ -19,7 +19,7 @@ } - + - + - + - + - +
onChange(event.detail)} {placeholder} /> -
- -
+ {#if !disabled} +
+ +
+ {/if}
diff --git a/packages/builder/src/components/common/ValuesList.svelte b/packages/builder/src/components/common/ValuesList.svelte index ff8cdb5479..eba81dac7c 100644 --- a/packages/builder/src/components/common/ValuesList.svelte +++ b/packages/builder/src/components/common/ValuesList.svelte @@ -7,7 +7,7 @@ const inputChanged = ev => { try { - values = ev.target.value.split("\n") + values = ev.detail.split("\n") } catch (_) { values = [] } diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/NextPage.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/NextPage.svelte new file mode 100644 index 0000000000..a96e1bd3d5 --- /dev/null +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/NextPage.svelte @@ -0,0 +1,38 @@ + + +
+ + x._instanceName} + getOptionValue={(x) => x._id} + /> +
+ + diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js index 4700ea5c8f..95c92a3f6d 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js @@ -6,6 +6,8 @@ import TriggerAutomation from "./TriggerAutomation.svelte" import ValidateForm from "./ValidateForm.svelte" import LogIn from "./LogIn.svelte" import LogOut from "./LogOut.svelte" +import NextPage from "./NextPage.svelte" +import PrevPage from "./PrevPage.svelte" // defines what actions are available, when adding a new one // the component is the setup panel for the action @@ -45,4 +47,12 @@ export default [ name: "Log Out", component: LogOut, }, + { + name: "Next Page", + component: NextPage, + }, + { + name: "Previous Page", + component: PrevPage, + }, ] diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte index e69de29bb2..a5c192cace 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte @@ -0,0 +1,217 @@ + + +{#if value?.length} +
+ {#each value as expression, idx} + onOperatorChange(expression, e.detail)} + placeholder={null} /> + {#if ['string', 'longform', 'number'].includes(expression.type)} + (expression.value = event.detail)} /> + {:else if expression.type === 'options'} + + {:else if expression.type === 'boolean'} + + {:else if expression.type === 'datetime'} + + {:else} + + {/if} + + removeField(expression.id)} /> + {/each} +
+{/if} +
+ +
+ + diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte index 06b4930d53..9336417b08 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte @@ -1,12 +1,19 @@ @@ -48,24 +51,7 @@ constaints. {/if} -
- -
+ - - diff --git a/packages/builder/src/components/design/PropertiesPanel/SettingsView.svelte b/packages/builder/src/components/design/PropertiesPanel/SettingsView.svelte index af81ea3d24..59fb549d09 100644 --- a/packages/builder/src/components/design/PropertiesPanel/SettingsView.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/SettingsView.svelte @@ -16,7 +16,7 @@ import MultiFieldSelect from "./PropertyControls/MultiFieldSelect.svelte" import SchemaSelect from "./PropertyControls/SchemaSelect.svelte" import EventsEditor from "./PropertyControls/EventsEditor" - import FilterEditor from "./PropertyControls/FilterEditor.svelte" + import FilterEditor from "./PropertyControls/FilterEditor/FilterEditor.svelte" import { IconSelect } from "./PropertyControls/IconSelect" import ColorPicker from "./PropertyControls/ColorPicker.svelte" import StringFieldSelect from "./PropertyControls/StringFieldSelect.svelte" diff --git a/packages/client/src/api/tables.js b/packages/client/src/api/tables.js index 248e1516c2..632f24e4a4 100644 --- a/packages/client/src/api/tables.js +++ b/packages/client/src/api/tables.js @@ -5,14 +5,14 @@ import { enrichRows } from "./rows" * Fetches a table definition. * Since definitions cannot change at runtime, the result is cached. */ -export const fetchTableDefinition = async tableId => { +export const fetchTableDefinition = async (tableId) => { return await API.get({ url: `/api/tables/${tableId}`, cache: true }) } /** * Fetches all rows from a table. */ -export const fetchTableData = async tableId => { +export const fetchTableData = async (tableId) => { const rows = await API.get({ url: `/api/${tableId}/rows` }) return await enrichRows(rows, tableId) } @@ -34,3 +34,35 @@ export const searchTableData = async ({ tableId, search, pagination }) => { output.rows = await enrichRows(output.rows, tableId) return output } + +/** + * Searches a table using Lucene. + */ +export const searchTable = async ({ + tableId, + query, + raw, + bookmark, + limit, + sort, + sortOrder, +}) => { + if (!tableId || (!query && !raw)) { + return + } + const res = await API.post({ + url: `/api/search/${tableId}/rows`, + body: { + query, + raw, + bookmark, + limit, + sort, + sortOrder, + }, + }) + return { + rows: await enrichRows(res?.rows, tableId), + bookmark: res.bookmark, + } +} diff --git a/packages/client/src/constants.js b/packages/client/src/constants.js index 3aa302bec9..fc69ac212c 100644 --- a/packages/client/src/constants.js +++ b/packages/client/src/constants.js @@ -5,4 +5,6 @@ export const TableNames = { export const ActionTypes = { ValidateForm: "ValidateForm", RefreshDatasource: "RefreshDatasource", + NextPage: "NextPage", + PrevPage: "PrevPage", } diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js index 4d2865d586..f78ac1773c 100644 --- a/packages/client/src/utils/buttonActions.js +++ b/packages/client/src/utils/buttonActions.js @@ -16,27 +16,27 @@ const saveRowHandler = async (action, context) => { } } -const deleteRowHandler = async action => { +const deleteRowHandler = async (action) => { const { tableId, revId, rowId } = action.parameters if (tableId && revId && rowId) { await deleteRow({ tableId, rowId, revId }) } } -const triggerAutomationHandler = async action => { +const triggerAutomationHandler = async (action) => { const { fields } = action.parameters if (fields) { await triggerAutomation(action.parameters.automationId, fields) } } -const navigationHandler = action => { +const navigationHandler = (action) => { if (action.parameters.url) { routeStore.actions.navigate(action.parameters.url) } } -const queryExecutionHandler = async action => { +const queryExecutionHandler = async (action) => { const { datasourceId, queryId, queryParams } = action.parameters await executeQuery({ datasourceId, @@ -68,7 +68,23 @@ const refreshDatasourceHandler = async (action, context) => { ) } -const loginHandler = async action => { +const nextPageHandler = async (action, context) => { + return await executeActionHandler( + context, + action.parameters.componentId, + ActionTypes.NextPage + ) +} + +const prevPageHandler = async (action, context) => { + return await executeActionHandler( + context, + action.parameters.componentId, + ActionTypes.PrevPage + ) +} + +const loginHandler = async (action) => { const { email, password } = action.parameters await authStore.actions.logIn({ email, password }) } @@ -87,6 +103,8 @@ const handlerMap = { ["Refresh Datasource"]: refreshDatasourceHandler, ["Log In"]: loginHandler, ["Log Out"]: logoutHandler, + ["Next Page"]: nextPageHandler, + ["Previous Page"]: prevPageHandler, } /** @@ -96,9 +114,10 @@ const handlerMap = { export const enrichButtonActions = (actions, context) => { // Prevent button actions in the builder preview if (get(builderStore).inBuilder) { - return () => {} + // TODO uncomment + // return () => {} } - const handlers = actions.map(def => handlerMap[def["##eventHandlerType"]]) + const handlers = actions.map((def) => handlerMap[def["##eventHandlerType"]]) return async () => { for (let i = 0; i < handlers.length; i++) { try { diff --git a/packages/server/src/api/controllers/search/index.js b/packages/server/src/api/controllers/search/index.js index 234c7eb258..648a66c742 100644 --- a/packages/server/src/api/controllers/search/index.js +++ b/packages/server/src/api/controllers/search/index.js @@ -1,12 +1,14 @@ const { QueryBuilder, buildSearchUrl, search } = require("./utils") -exports.rowSearch = async ctx => { +exports.rowSearch = async (ctx) => { const appId = ctx.appId const { tableId } = ctx.params - const { bookmark, query, raw } = ctx.request.body + const { bookmark, query, raw, limit, sort, sortOrder } = ctx.request.body let url if (query) { - url = new QueryBuilder(appId, query, bookmark).addTable(tableId).complete() + url = new QueryBuilder(appId, query, bookmark, limit, sort, sortOrder) + .addTable(tableId) + .complete() } else if (raw) { url = buildSearchUrl({ appId, diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js index e8e26c813b..d18f48533e 100644 --- a/packages/server/src/api/controllers/search/utils.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -10,24 +10,43 @@ const fetch = require("node-fetch") * @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was * returned with query for next set of search results. * @param {number} limit The number of entries to return per query. + * @param {string} sort The column to sort by. + * @param {string} sortOrder The order to sort by. "ascending" or "descending". * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled. * @return {string} The URL which a GET can be performed on to receive results. */ -function buildSearchUrl({ appId, query, bookmark, excludeDocs, limit = 50 }) { +function buildSearchUrl({ + appId, + query, + bookmark, + sort, + sortOrder, + excludeDocs, + limit = 50, +}) { let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search` url += `/${SearchIndexes.ROWS}?q=${query}` url += `&limit=${limit}` if (!excludeDocs) { url += "&include_docs=true" } + if (sort) { + const orderChar = sortOrder === "descending" ? "-" : "" + url += `&sort="${orderChar}${sort.replace(/ /, "_")}"` + } if (bookmark) { url += `&bookmark=${bookmark}` } + console.log(url) return checkSlashesInUrl(url) } +const luceneEscape = (value) => { + return `${value}`.replace(/[ #+\-&|!(){}\[\]^"~*?:\\]/g, "\\$&") +} + class QueryBuilder { - constructor(appId, base) { + constructor(appId, base, bookmark, limit, sort, sortOrder) { this.appId = appId this.query = { string: {}, @@ -35,10 +54,14 @@ class QueryBuilder { range: {}, equal: {}, notEqual: {}, + empty: {}, + notEmpty: {}, ...base, } - this.limit = 50 - this.bookmark = null + this.bookmark = bookmark + this.limit = limit || 50 + this.sort = sort + this.sortOrder = sortOrder || "ascending" } setLimit(limit) { @@ -79,39 +102,73 @@ class QueryBuilder { return this } + addEmpty(key, value) { + this.query.empty[key] = value + return this + } + + addNotEmpty(key, value) { + this.query.notEmpty[key] = value + return this + } + addTable(tableId) { this.query.equal.tableId = tableId return this } complete(rawQuery = null) { - let output = "" + let output = "*:*" function build(structure, queryFn) { for (let [key, value] of Object.entries(structure)) { - if (output.length !== 0) { - output += " AND " + const expression = queryFn(luceneEscape(key.replace(/ /, "_")), value) + if (expression == null) { + continue } - output += queryFn(key, value).replace(/ /, "\\ ") + output += ` AND ${expression}` } } if (this.query.string) { - build(this.query.string, (key, value) => `${key}:${value}*`) + build(this.query.string, (key, value) => { + return value ? `${key}:${luceneEscape(value.toLowerCase())}*` : null + }) } if (this.query.range) { - build( - this.query.range, - (key, value) => `${key}:[${value.low} TO ${value.high}]` - ) + build(this.query.range, (key, value) => { + if (!value) { + return null + } + if (isNaN(value.low) || value.low == null || value.low === "") { + return null + } + if (isNaN(value.high) || value.high == null || value.high === "") { + return null + } + console.log(value) + return `${key}:[${value.low} TO ${value.high}]` + }) } if (this.query.fuzzy) { - build(this.query.fuzzy, (key, value) => `${key}:${value}~`) + build(this.query.fuzzy, (key, value) => { + return value ? `${key}:${luceneEscape(value.toLowerCase())}~` : null + }) } if (this.query.equal) { - build(this.query.equal, (key, value) => `${key}:${value}`) + build(this.query.equal, (key, value) => { + return value ? `${key}:${luceneEscape(value.toLowerCase())}` : null + }) } if (this.query.notEqual) { - build(this.query.notEqual, (key, value) => `!${key}:${value}`) + build(this.query.notEqual, (key, value) => { + return value ? `!${key}:${luceneEscape(value.toLowerCase())}` : null + }) + } + if (this.query.empty) { + build(this.query.empty, (key) => `!${key}:["" TO *]`) + } + if (this.query.notEmpty) { + build(this.query.notEmpty, (key) => `${key}:["" TO *]`) } if (rawQuery) { output = output.length === 0 ? rawQuery : `&${rawQuery}` @@ -121,11 +178,13 @@ class QueryBuilder { query: output, bookmark: this.bookmark, limit: this.limit, + sort: this.sort, + sortOrder: this.sortOrder, }) } } -exports.search = async query => { +exports.search = async (query) => { const response = await fetch(query, { method: "GET", }) @@ -134,7 +193,7 @@ exports.search = async query => { rows: [], } if (json.rows != null && json.rows.length > 0) { - output.rows = json.rows.map(row => row.doc) + output.rows = json.rows.map((row) => row.doc) } if (json.bookmark) { output.bookmark = json.bookmark diff --git a/packages/server/src/db/views/staticViews.js b/packages/server/src/db/views/staticViews.js index 305d042217..62bf58a745 100644 --- a/packages/server/src/db/views/staticViews.js +++ b/packages/server/src/db/views/staticViews.js @@ -25,11 +25,11 @@ const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR * @returns {Promise} The view now exists, please note that the next view of this query will actually build it, * so it may be slow. */ -exports.createLinkView = async appId => { +exports.createLinkView = async (appId) => { const db = new CouchDB(appId) const designDoc = await db.get("_design/database") const view = { - map: function(doc) { + map: function (doc) { // everything in this must remain constant as its going to Pouch, no external variables if (doc.type === "link") { let doc1 = doc.doc1 @@ -57,7 +57,7 @@ exports.createLinkView = async appId => { await db.put(designDoc) } -exports.createRoutingView = async appId => { +exports.createRoutingView = async (appId) => { const db = new CouchDB(appId) const designDoc = await db.get("_design/database") const view = { @@ -84,23 +84,28 @@ async function searchIndex(appId, indexName, fnString) { designDoc.indexes = { [indexName]: { index: fnString, + analyzer: "keyword", }, } await db.put(designDoc) } -exports.createAllSearchIndex = async appId => { +exports.createAllSearchIndex = async (appId) => { await searchIndex( appId, SearchIndexes.ROWS, - function(doc) { + function (doc) { function idx(input, prev) { for (let key of Object.keys(input)) { - const idxKey = prev != null ? `${prev}.${key}` : key - if (key === "_id" || key === "_rev") { + let idxKey = prev != null ? `${prev}.${key}` : key + idxKey = idxKey.replace(/ /, "_") + if (key === "_id" || key === "_rev" || input[key] == null) { continue } - if (typeof input[key] !== "object") { + if (typeof input[key] === "string") { + // eslint-disable-next-line no-undef + index(idxKey, input[key].toLowerCase(), { store: true }) + } else if (typeof input[key] !== "object") { // eslint-disable-next-line no-undef index(idxKey, input[key], { store: true }) } else { diff --git a/packages/standard-components/manifest.json b/packages/standard-components/manifest.json index 163caa3dbc..c20744a1a2 100644 --- a/packages/standard-components/manifest.json +++ b/packages/standard-components/manifest.json @@ -1419,6 +1419,7 @@ "icon": "Data", "styleable": false, "hasChildren": true, + "actions": ["NextPage", "PrevPage"], "settings": [ { "type": "dataSource", @@ -1470,6 +1471,10 @@ { "label": "Loaded", "key": "loaded" + }, + { + "label": "Page Number", + "key": "pageNumber" } ] } diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte index 83de1ceb66..cedc153917 100644 --- a/packages/standard-components/src/DataProvider.svelte +++ b/packages/standard-components/src/DataProvider.svelte @@ -5,7 +5,7 @@ export let filter export let sortColumn export let sortOrder - export let limit + export let limit = 50 const { API, styleable, Provider, ActionTypes } = getContext("sdk") const component = getContext("component") @@ -16,13 +16,18 @@ // Loading flag for the initial load let loaded = false - let allRows = [] + // Provider state + let rows = [] let schema = {} + let bookmarks = [null] + let pageNumber = 0 - $: fetchData(dataSource) - $: filteredRows = filterRows(allRows, filter) - $: sortedRows = sortRows(filteredRows, sortColumn, sortOrder) - $: rows = limitRows(sortedRows, limit) + $: query = dataSource?.type === "table" ? buildLuceneQuery(filter) : null + $: hasNextPage = bookmarks[pageNumber + 1] != null + $: hasPrevPage = pageNumber > 0 + $: fetchData(dataSource, query, limit, sortColumn, sortOrder) + // $: sortedRows = sortRows(filteredRows, sortColumn, sortOrder) + // $: rows = limitRows(sortedRows, limit) $: getSchema(dataSource) $: actions = [ { @@ -30,6 +35,14 @@ callback: () => fetchData(dataSource), metadata: { dataSource }, }, + { + type: ActionTypes.NextPage, + callback: () => nextPage(), + }, + { + type: ActionTypes.PrevPage, + callback: () => prevPage(), + }, ] $: dataContext = { rows, @@ -37,23 +50,82 @@ rowsLength: rows.length, loading, loaded, + pageNumber: pageNumber + 1, + hasNextPage, + hasPrevPage, } - const fetchData = async dataSource => { + const buildLuceneQuery = (filter) => { + let query = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + } + if (Array.isArray(filter)) { + filter.forEach((expression) => { + if (expression.operator.startsWith("range")) { + let range = { + low: Number.MIN_SAFE_INTEGER, + high: Number.MAX_SAFE_INTEGER, + } + if (expression.operator === "rangeLow") { + range.low = expression.value + } else if (expression.operator === "rangeHigh") { + range.high = expression.value + } + query.range[expression.field] = range + } else if (query[expression.operator]) { + query[expression.operator][expression.field] = expression.value + } + }) + } + return query + } + + const fetchData = async (dataSource, query, limit, sortColumn, sortOrder) => { loading = true - allRows = await API.fetchDatasource(dataSource) + if (dataSource?.type === "table") { + const res = await API.searchTable({ + tableId: dataSource.tableId, + query, + limit, + sort: sortColumn, + sortOrder: sortOrder?.toLowerCase() ?? "ascending", + }) + pageNumber = 0 + rows = res.rows + + // Check we have next data + const next = await API.searchTable({ + tableId: dataSource.tableId, + query, + limit: 1, + bookmark: res.bookmark, + sort: sortColumn, + sortOrder: sortOrder?.toLowerCase() ?? "ascending", + }) + if (next.rows?.length) { + bookmarks = [null, res.bookmark] + } else { + bookmarks = [null] + } + } else { + const rows = await API.fetchDatasource(dataSource) + rows = inMemoryFilterRows(rows, filter) + } loading = false loaded = true } - const filterRows = (rows, filter) => { - if (!Object.keys(filter || {}).length) { - return rows - } + const inMemoryFilterRows = (rows, filter) => { let filteredData = [...rows] Object.entries(filter).forEach(([field, value]) => { if (value != null && value !== "") { - filteredData = filteredData.filter(row => { + filteredData = filteredData.filter((row) => { return row[field] === value }) } @@ -84,7 +156,7 @@ return rows.slice(0, numLimit) } - const getSchema = async dataSource => { + const getSchema = async (dataSource) => { if (dataSource?.schema) { schema = dataSource.schema } else if (dataSource?.tableId) { @@ -101,6 +173,51 @@ } }) } + + const nextPage = async () => { + if (!hasNextPage) { + return + } + const res = await API.searchTable({ + tableId: dataSource?.tableId, + query, + bookmark: bookmarks[pageNumber + 1], + limit, + sort: sortColumn, + sortOrder: sortOrder?.toLowerCase() ?? "ascending", + }) + pageNumber++ + rows = res.rows + + // Check we have next data + const next = await API.searchTable({ + tableId: dataSource.tableId, + query, + limit: 1, + bookmark: res.bookmark, + sort: sortColumn, + sortOrder: sortOrder?.toLowerCase() ?? "ascending", + }) + if (next.rows?.length) { + bookmarks[pageNumber + 1] = res.bookmark + } + } + + const prevPage = async () => { + if (!hasPrevPage) { + return + } + const res = await API.searchTable({ + tableId: dataSource?.tableId, + query, + bookmark: bookmarks[pageNumber - 1], + limit, + sort: sortColumn, + sortOrder: sortOrder?.toLowerCase() ?? "ascending", + }) + pageNumber-- + rows = res.rows + }
From 78ae68981e4333cf821b868082af684e08e5b777 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 11 May 2021 11:24:16 +0100 Subject: [PATCH 05/37] Add support for numerical sorting --- .../popovers/EditTablePopover.svelte | 2 - .../FilterEditor/FilterBuilder.svelte | 36 +++++++----- .../FilterEditor/FilterEditor.svelte | 1 - packages/client/src/api/tables.js | 6 +- .../src/api/controllers/search/index.js | 22 +++++++- .../src/api/controllers/search/utils.js | 20 ++++--- .../src/DataProvider.svelte | 55 ++++++++++++++----- yarn.lock | 2 +- 8 files changed, 101 insertions(+), 43 deletions(-) diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte index 2d419f18aa..6cae231be9 100644 --- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte +++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte @@ -40,13 +40,11 @@ if (wasSelectedTable._id === table._id) { $goto("./table") } - editorModal.hide() } async function save() { await tables.save(table) notifications.success("Table renamed successfully") - editorModal.hide() } function checkValid(evt) { diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte index a5c192cace..7afbaa5b7d 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte @@ -9,7 +9,7 @@ import { store, currentAsset } from "builderStore" import { getBindableProperties } from "builderStore/dataBinding" import { createEventDispatcher } from "svelte" - import DrawerBindableInput from "components/common/DrawerBindableInput.svelte" + import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" import { generate } from "shortid" const dispatch = createEventDispatcher() @@ -159,35 +159,44 @@ bind:value={expression.field} options={fieldOptions} on:change={e => onFieldChange(expression, e.detail)} - placeholder="Column" /> + placeholder="Column" + /> + {#if expression.valueType === "Binding"} (expression.value = event.detail)} /> + {:else if ["string", "longform", "number"].includes(expression.type)} + {:else if expression.type === "options"} diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js index 7827850428..3310f89bf7 100644 --- a/packages/server/src/api/controllers/search/utils.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -39,7 +39,6 @@ function buildSearchUrl({ if (bookmark) { url += `&bookmark=${bookmark}` } - console.log(url) return checkSlashesInUrl(url) } @@ -142,13 +141,12 @@ class QueryBuilder { if (!value) { return null } - if (isNaN(value.low) || value.low == null || value.low === "") { + if (value.low == null || value.low === "") { return null } - if (isNaN(value.high) || value.high == null || value.high === "") { + if (value.high == null || value.high === "") { return null } - console.log(value) return `${key}:[${value.low} TO ${value.high}]` }) } diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte index 38dd9f3955..512a4c594d 100644 --- a/packages/standard-components/src/DataProvider.svelte +++ b/packages/standard-components/src/DataProvider.svelte @@ -83,20 +83,21 @@ notEmpty: {}, } if (Array.isArray(filter)) { - filter.forEach(expression => { - if (expression.operator.startsWith("range")) { - let range = { - low: Number.MIN_SAFE_INTEGER, - high: Number.MAX_SAFE_INTEGER, + filter.forEach(({ operator, field, type, value }) => { + if (operator.startsWith("range")) { + if (!query.range[field]) { + query.range[field] = { + low: type === "number" ? Number.MIN_SAFE_INTEGER : "0000", + high: type === "number" ? Number.MAX_SAFE_INTEGER : "9999", + } } - if (expression.operator === "rangeLow") { - range.low = expression.value - } else if (expression.operator === "rangeHigh") { - range.high = expression.value + if (operator === "rangeLow") { + query.range[field].low = value + } else if (operator === "rangeHigh") { + query.range[field].high = value } - query.range[expression.field] = range - } else if (query[expression.operator]) { - query[expression.operator][expression.field] = expression.value + } else if (query[operator]) { + query[operator][field] = value } }) } From 6d1c9c3e448ed7c5d9c2ec6786718f773e3f0ff6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 11 May 2021 19:07:52 +0100 Subject: [PATCH 08/37] Add legacy in-memory filtering and sorting for views and external datasources --- .../EventsEditor/actions/SaveFields.svelte | 2 +- .../FilterEditor/FilterEditor.svelte | 54 +++++++++++++++---- ...lder.svelte => LuceneFilterBuilder.svelte} | 0 .../src/DataProvider.svelte | 10 ++-- 4 files changed, 50 insertions(+), 16 deletions(-) rename packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/{FilterBuilder.svelte => LuceneFilterBuilder.svelte} (100%) diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/SaveFields.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/SaveFields.svelte index f6bd54ffb6..f4a4bea334 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/SaveFields.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/SaveFields.svelte @@ -80,7 +80,7 @@ /> {/each}
- diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte index d90097939a..1662337409 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte @@ -12,22 +12,35 @@ getDatasourceForProvider, getSchemaForDatasource, } from "builderStore/dataBinding" - import FilterBuilder from "./FilterBuilder.svelte" + import LuceneFilterBuilder from "./LuceneFilterBuilder.svelte" import { currentAsset } from "builderStore" + import SaveFields from "../EventsEditor/actions/SaveFields.svelte" const dispatch = createEventDispatcher() - export let value = {} + export let value = [] export let componentInstance let drawer - let tempValue = Array.isArray(value) ? value : [] + let tempValue = value - $: schemaFields = getSchemaFields(componentInstance) + $: numFilters = Array.isArray(tempValue) + ? tempValue.length + : Object.keys(tempValue || {}).length + $: dataSource = getDatasourceForProvider($currentAsset, componentInstance) + $: schema = getSchemaForDatasource(dataSource)?.schema + $: schemaFields = Object.values(schema || {}) + $: internalTable = dataSource?.type === "table" - const getSchemaFields = component => { - const datasource = getDatasourceForProvider($currentAsset, component) - const { schema } = getSchemaForDatasource(datasource) - return Object.values(schema || {}) + // Reset value if value is wrong type for the datasource. + // Lucene editor needs an array, and simple editor needs an object. + $: { + if (internalTable && !Array.isArray(value)) { + tempValue = [] + dispatch("change", []) + } else if (!internalTable && Array.isArray(value)) { + tempValue = {} + dispatch("change", {}) + } } const saveFilter = async () => { @@ -43,14 +56,35 @@ - {#if !Object.keys(tempValue || {}).length} + {#if !numFilters} Add your first filter column. {:else} Results are filtered to only those which match all of the following constaints. {/if} - + {#if internalTable} + + {:else} +
+ (tempValue = e.detail)} + /> +
+ {/if} + + diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/LuceneFilterBuilder.svelte similarity index 100% rename from packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte rename to packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/LuceneFilterBuilder.svelte diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte index 512a4c594d..418d80f7a9 100644 --- a/packages/standard-components/src/DataProvider.svelte +++ b/packages/standard-components/src/DataProvider.svelte @@ -24,7 +24,7 @@ let pageNumber = 0 $: internalTable = dataSource?.type === "table" - $: query = dataSource?.type === "table" ? buildLuceneQuery(filter) : null + $: query = internalTable ? buildLuceneQuery(filter) : null $: hasNextPage = bookmarks[pageNumber + 1] != null $: hasPrevPage = pageNumber > 0 $: getSchema(dataSource) @@ -34,8 +34,8 @@ if (internalTable) { rows = allRows } else { - rows = sortRows(allRows, sortColumn, sortOrder) - rows = limitRows(rows, limit) + const sortedRows = sortRows(allRows, sortColumn, sortOrder) + rows = limitRows(sortedRows, limit) } } $: actions = [ @@ -213,7 +213,7 @@ } const nextPage = async () => { - if (!hasNextPage) { + if (!hasNextPage || !internalTable) { return } const res = await API.searchTable({ @@ -244,7 +244,7 @@ } const prevPage = async () => { - if (!hasPrevPage) { + if (!hasPrevPage || !internalTable) { return } const res = await API.searchTable({ From 5131a2fd74e0c6c18066422691fa49a3798e1e42 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 12 May 2021 14:34:25 +0100 Subject: [PATCH 09/37] Fix chart placeholder, fix spectrum colours and begin work on pagination component --- packages/bbui/src/Table/Table.svelte | 8 ++---- .../DataProviderSelect.svelte | 16 +++++++++++- packages/builder/src/global.css | 4 +-- packages/standard-components/manifest.json | 13 ++++++++++ .../standard-components/src/Pagination.svelte | 25 +++++++++++++++++++ .../src/charts/ApexChart.svelte | 12 ++++++--- 6 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 packages/standard-components/src/Pagination.svelte diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte index 048ded2b5b..56f1ac96a8 100644 --- a/packages/bbui/src/Table/Table.svelte +++ b/packages/bbui/src/Table/Table.svelte @@ -345,7 +345,7 @@ From e29d7512e4fe2c6f7890fa5dc1fddbbeaf369438 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 May 2021 14:44:47 +0100 Subject: [PATCH 10/37] Updating search endpoint to have egress processing. --- packages/server/src/api/controllers/search/index.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/search/index.js b/packages/server/src/api/controllers/search/index.js index 2a309cc7e6..31c81fec9a 100644 --- a/packages/server/src/api/controllers/search/index.js +++ b/packages/server/src/api/controllers/search/index.js @@ -1,4 +1,6 @@ const { QueryBuilder, buildSearchUrl, search } = require("./utils") +const CouchDB = require("../../../db") +const { outputProcessing } = require("../../../utilities/rowProcessor") exports.rowSearch = async ctx => { const appId = ctx.appId @@ -12,6 +14,8 @@ exports.rowSearch = async ctx => { sortOrder, sortType, } = ctx.request.body + const db = new CouchDB(appId) + let url if (query) { url = new QueryBuilder( @@ -32,5 +36,10 @@ exports.rowSearch = async ctx => { bookmark, }) } - ctx.body = await search(url) + const response = await search(url) + const table = await db.get(tableId) + ctx.body = { + rows: await outputProcessing(appId, table, response.rows), + bookmark: response.bookmark, + } } From 4147b7203c5a66441269c64138aa36617c49ff62 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 12 May 2021 14:52:43 +0100 Subject: [PATCH 11/37] Fix two-tone spectrum background colours --- packages/bbui/src/Table/Table.svelte | 4 ++-- packages/builder/src/global.css | 6 +----- packages/standard-components/src/forms/Form.svelte | 1 + 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte index 56f1ac96a8..4278751c5f 100644 --- a/packages/bbui/src/Table/Table.svelte +++ b/packages/bbui/src/Table/Table.svelte @@ -345,7 +345,7 @@ From 521184601a4f7ba976be831e467cfb5f09347216 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 12:23:25 +0100 Subject: [PATCH 12/37] Add spectrum pagination component to BBUI --- packages/bbui/package.json | 1 + .../bbui/src/Pagination/Pagination.svelte | 54 +++++++++++++++++++ packages/bbui/src/index.js | 1 + packages/bbui/yarn.lock | 5 ++ 4 files changed, 61 insertions(+) create mode 100644 packages/bbui/src/Pagination/Pagination.svelte diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 056f4229cb..c4c7e14b4b 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -56,6 +56,7 @@ "@spectrum-css/link": "^3.1.1", "@spectrum-css/menu": "^3.0.1", "@spectrum-css/modal": "^3.0.1", + "@spectrum-css/pagination": "^3.0.3", "@spectrum-css/picker": "^1.0.1", "@spectrum-css/popover": "^3.0.1", "@spectrum-css/progressbar": "^1.0.2", diff --git a/packages/bbui/src/Pagination/Pagination.svelte b/packages/bbui/src/Pagination/Pagination.svelte new file mode 100644 index 0000000000..cfe989ea14 --- /dev/null +++ b/packages/bbui/src/Pagination/Pagination.svelte @@ -0,0 +1,54 @@ + + + + + diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js index 9d47958461..8ed1dadf1f 100644 --- a/packages/bbui/src/index.js +++ b/packages/bbui/src/index.js @@ -51,6 +51,7 @@ export { default as TreeView } from "./TreeView/Tree.svelte" export { default as TreeItem } from "./TreeView/Item.svelte" export { default as Divider } from "./Divider/Divider.svelte" export { default as Search } from "./Form/Search.svelte" +export { default as Pagination } from "./Pagination/Pagination.svelte" // Typography export { default as Body } from "./Typography/Body.svelte" diff --git a/packages/bbui/yarn.lock b/packages/bbui/yarn.lock index e9bad2e162..36e26b0ea6 100644 --- a/packages/bbui/yarn.lock +++ b/packages/bbui/yarn.lock @@ -161,6 +161,11 @@ resolved "https://registry.yarnpkg.com/@spectrum-css/modal/-/modal-3.0.2.tgz#58b6621cab65f90788d310374f40df1f7090473f" integrity sha512-YnIivJhoaao7Otu+HV7sgebPyFbO6sd/oMvTN/Rb2wwgnaMnIIuIRdGandSrcgotN2uNgs+P0knG6mv/xA1/dg== +"@spectrum-css/pagination@^3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@spectrum-css/pagination/-/pagination-3.0.3.tgz#b204c3ada384c4af751a354bc428346d82eeea65" + integrity sha512-OJ/v9GeNXJOZ9Yr9LDBYPrR2NCiLOWP9wANT/a5sqFuugRnQbn/HYMnRp9TBxwpDY6ihaPo0T/wi7kLiAJFdDw== + "@spectrum-css/picker@^1.0.1": version "1.0.2" resolved "https://registry.yarnpkg.com/@spectrum-css/picker/-/picker-1.0.2.tgz#b49429ae3c89f9c5f2c0530787ce45392c9612ff" From b52ccb55583b412b048d8ee03cfce2ce7aa41728 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 12:23:48 +0100 Subject: [PATCH 13/37] Fix bug preventing progress circle component from animating --- packages/bbui/src/ProgressCircle/ProgressCircle.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bbui/src/ProgressCircle/ProgressCircle.svelte b/packages/bbui/src/ProgressCircle/ProgressCircle.svelte index 711517ec7b..9c8181ec7c 100644 --- a/packages/bbui/src/ProgressCircle/ProgressCircle.svelte +++ b/packages/bbui/src/ProgressCircle/ProgressCircle.svelte @@ -42,7 +42,7 @@
From da40086c0b4853bb9996b362f683b3742d9f8008 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 12:26:18 +0100 Subject: [PATCH 14/37] Allow multipage searches and implement optional pagination to data providers --- packages/client/src/api/tables.js | 22 +- packages/server/src/api/controllers/row.js | 40 ---- .../src/api/controllers/search/index.js | 47 ++-- .../src/api/controllers/search/utils.js | 206 ++++++++++++------ packages/server/src/api/routes/row.js | 6 - packages/standard-components/manifest.json | 52 +---- .../src/DataProvider.svelte | 91 +++++--- .../standard-components/src/Search.svelte | 195 ----------------- 8 files changed, 241 insertions(+), 418 deletions(-) delete mode 100644 packages/standard-components/src/Search.svelte diff --git a/packages/client/src/api/tables.js b/packages/client/src/api/tables.js index 693ce8f013..a75a2d368b 100644 --- a/packages/client/src/api/tables.js +++ b/packages/client/src/api/tables.js @@ -17,24 +17,6 @@ export const fetchTableData = async tableId => { return await enrichRows(rows, tableId) } -/** - * Perform a mango query against an internal table - * @param {String} tableId - id of the table to search - * @param {Object} search - Mango Compliant search object - * @param {Object} pagination - the pagination controls - */ -export const searchTableData = async ({ tableId, search, pagination }) => { - const output = await API.post({ - url: `/api/${tableId}/rows/search`, - body: { - query: search, - pagination, - }, - }) - output.rows = await enrichRows(output.rows, tableId) - return output -} - /** * Searches a table using Lucene. */ @@ -47,6 +29,7 @@ export const searchTable = async ({ sort, sortOrder, sortType, + paginate, }) => { if (!tableId || (!query && !raw)) { return @@ -61,10 +44,11 @@ export const searchTable = async ({ sort, sortOrder, sortType, + paginate, }, }) return { + ...res, rows: await enrichRows(res?.rows, tableId), - bookmark: res.bookmark, } } diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 6d0cc08548..1d73abea47 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -16,7 +16,6 @@ const { const { FieldTypes } = require("../../constants") const { isEqual } = require("lodash") const { cloneDeep } = require("lodash/fp") -const { QueryBuilder, search } = require("./search/utils") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -248,45 +247,6 @@ exports.fetchView = async function (ctx) { } } -exports.search = async function (ctx) { - const appId = ctx.appId - const db = new CouchDB(appId) - const { - query, - pagination: { pageSize = 10, bookmark }, - } = ctx.request.body - const tableId = ctx.params.tableId - - const queryBuilder = new QueryBuilder(appId) - .setLimit(pageSize) - .addTable(tableId) - if (bookmark) { - queryBuilder.setBookmark(bookmark) - } - - let searchString - if (ctx.query && ctx.query.raw && ctx.query.raw !== "") { - searchString = queryBuilder.complete(query["RAW"]) - } else { - // make all strings a starts with operation rather than pure equality - for (const [key, queryVal] of Object.entries(query)) { - if (typeof queryVal === "string") { - queryBuilder.addString(key, queryVal) - } else { - queryBuilder.addEqual(key, queryVal) - } - } - searchString = queryBuilder.complete() - } - - const response = await search(searchString) - const table = await db.get(tableId) - ctx.body = { - rows: await outputProcessing(appId, table, response.rows), - bookmark: response.bookmark, - } -} - exports.fetchTableRows = async function (ctx) { const appId = ctx.appId const db = new CouchDB(appId) diff --git a/packages/server/src/api/controllers/search/index.js b/packages/server/src/api/controllers/search/index.js index 31c81fec9a..b8411e9d3a 100644 --- a/packages/server/src/api/controllers/search/index.js +++ b/packages/server/src/api/controllers/search/index.js @@ -1,4 +1,4 @@ -const { QueryBuilder, buildSearchUrl, search } = require("./utils") +const { fullSearch, paginatedSearch } = require("./utils") const CouchDB = require("../../../db") const { outputProcessing } = require("../../../utilities/rowProcessor") @@ -8,38 +8,45 @@ exports.rowSearch = async ctx => { const { bookmark, query, - raw, limit, sort, sortOrder, sortType, + paginate, } = ctx.request.body const db = new CouchDB(appId) - let url - if (query) { - url = new QueryBuilder( + let response + const start = Date.now() + if (paginate) { + response = await paginatedSearch( appId, query, - bookmark, - limit, + tableId, sort, sortOrder, - sortType + sortType, + limit, + bookmark ) - .addTable(tableId) - .complete() - } else if (raw) { - url = buildSearchUrl({ + } else { + response = await fullSearch( appId, - query: raw, - bookmark, - }) + query, + tableId, + sort, + sortOrder, + sortType, + limit + ) } - const response = await search(url) - const table = await db.get(tableId) - ctx.body = { - rows: await outputProcessing(appId, table, response.rows), - bookmark: response.bookmark, + const end = Date.now() + console.log("Time: " + (end - start) / 1000 + " ms") + + if (response.rows && response.rows.length) { + const table = await db.get(tableId) + response.rows = await outputProcessing(appId, table, response.rows) } + + ctx.body = response } diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js index 3310f89bf7..6f2f29628a 100644 --- a/packages/server/src/api/controllers/search/utils.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -3,51 +3,12 @@ const { checkSlashesInUrl } = require("../../../utilities") const env = require("../../../environment") const fetch = require("node-fetch") -/** - * Given a set of inputs this will generate the URL which is to be sent to the search proxy in CouchDB. - * @param {string} appId The ID of the app which we will be searching within. - * @param {string} query The lucene query string which is to be used for searching. - * @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was - * returned with query for next set of search results. - * @param {number} limit The number of entries to return per query. - * @param {string} sort The column to sort by. - * @param {string} sortOrder The order to sort by. "ascending" or "descending". - * @param {string} sortType The type of sort to perform. "string" or "number". - * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled. - * @return {string} The URL which a GET can be performed on to receive results. - */ -function buildSearchUrl({ - appId, - query, - bookmark, - sort, - sortOrder, - sortType, - excludeDocs, - limit = 50, -}) { - let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search` - url += `/${SearchIndexes.ROWS}?q=${query}` - url += `&limit=${Math.min(limit, 200)}` - if (!excludeDocs) { - url += "&include_docs=true" - } - if (sort) { - const orderChar = sortOrder === "descending" ? "-" : "" - url += `&sort="${orderChar}${sort.replace(/ /, "_")}<${sortType}>"` - } - if (bookmark) { - url += `&bookmark=${bookmark}` - } - return checkSlashesInUrl(url) -} - const luceneEscape = value => { return `${value}`.replace(/[ #+\-&|!(){}\[\]^"~*?:\\]/g, "\\$&") } class QueryBuilder { - constructor(appId, base, bookmark, limit, sort, sortOrder, sortType) { + constructor(appId, base) { this.appId = appId this.query = { string: {}, @@ -59,11 +20,14 @@ class QueryBuilder { notEmpty: {}, ...base, } - this.bookmark = bookmark - this.limit = limit || 50 - this.sort = sort - this.sortOrder = sortOrder || "ascending" - this.sortType = sortType || "string" + this.limit = 50 + this.sortOrder = "ascending" + this.sortType = "string" + } + + setTable(tableId) { + this.query.equal.tableId = tableId + return this } setLimit(limit) { @@ -71,6 +35,21 @@ class QueryBuilder { return this } + setSort(sort) { + this.sort = sort + return this + } + + setSortOrder(sortOrder) { + this.sortOrder = sortOrder + return this + } + + setSortType(sortType) { + this.sortType = sortType + return this + } + setBookmark(bookmark) { this.bookmark = bookmark return this @@ -114,12 +93,7 @@ class QueryBuilder { return this } - addTable(tableId) { - this.query.equal.tableId = tableId - return this - } - - complete(rawQuery = null) { + buildSearchURL(excludeDocs = false) { let output = "*:*" function build(structure, queryFn) { for (let [key, value] of Object.entries(structure)) { @@ -171,22 +145,28 @@ class QueryBuilder { if (this.query.notEmpty) { build(this.query.notEmpty, key => `${key}:["" TO *]`) } - if (rawQuery) { - output = output.length === 0 ? rawQuery : `&${rawQuery}` + + let url = `${env.COUCH_DB_URL}/${this.appId}/_design/database/_search` + url += `/${SearchIndexes.ROWS}?q=${output}` + url += `&limit=${Math.min(this.limit, 200)}` + if (!excludeDocs) { + url += "&include_docs=true" } - return buildSearchUrl({ - appId: this.appId, - query: output, - bookmark: this.bookmark, - limit: this.limit, - sort: this.sort, - sortOrder: this.sortOrder, - sortType: this.sortType, - }) + if (this.sort) { + const orderChar = this.sortOrder === "descending" ? "-" : "" + url += `&sort="${orderChar}${this.sort.replace(/ /, "_")}<${ + this.sortType + }>"` + } + if (this.bookmark) { + url += `&bookmark=${this.bookmark}` + } + console.log(url) + return checkSlashesInUrl(url) } } -exports.search = async query => { +const runQuery = async query => { const response = await fetch(query, { method: "GET", }) @@ -203,5 +183,101 @@ exports.search = async query => { return output } -exports.QueryBuilder = QueryBuilder -exports.buildSearchUrl = buildSearchUrl +const recursiveSearch = async ( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + limit, + bookmark, + rows +) => { + if (rows.length >= limit) { + return rows + } + const pageSize = rows.length > limit - 200 ? limit - rows.length : 200 + const url = new QueryBuilder(appId, query) + .setTable(tableId) + .setBookmark(bookmark) + .setLimit(pageSize) + .setSort(sort) + .setSortOrder(sortOrder) + .setSortType(sortType) + .buildSearchURL() + const page = await runQuery(url) + if (!page.rows.length) { + return rows + } + if (page.rows.length < 200) { + return [...rows, ...page.rows] + } + return await recursiveSearch( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + limit, + page.bookmark, + [...rows, ...page.rows] + ) +} + +exports.paginatedSearch = async ( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + limit, + bookmark +) => { + if (limit == null || isNaN(limit) || limit < 0) { + limit = 50 + } + const builder = new QueryBuilder(appId, query) + .setTable(tableId) + .setSort(sort) + .setSortOrder(sortOrder) + .setSortType(sortType) + .setBookmark(bookmark) + .setLimit(limit) + const searchUrl = builder.buildSearchURL() + const nextUrl = builder.setLimit(1).buildSearchURL() + const searchResults = await runQuery(searchUrl) + const nextResults = await runQuery(nextUrl) + return { + ...searchResults, + hasNextPage: nextResults.rows && nextResults.rows.length > 0, + } +} + +exports.fullSearch = async ( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + limit +) => { + if (limit == null || isNaN(limit) || limit < 0) { + limit = 1000 + } + const rows = await recursiveSearch( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + Math.min(limit, 1000), + null, + [] + ) + return { rows } +} diff --git a/packages/server/src/api/routes/row.js b/packages/server/src/api/routes/row.js index be14910f3e..494ea61608 100644 --- a/packages/server/src/api/routes/row.js +++ b/packages/server/src/api/routes/row.js @@ -39,12 +39,6 @@ router usage, rowController.save ) - .post( - "/api/:tableId/rows/search", - paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.READ), - rowController.search - ) .patch( "/api/:tableId/rows/:rowId", paramSubResource("tableId", "rowId"), diff --git a/packages/standard-components/manifest.json b/packages/standard-components/manifest.json index 6a6b00732b..c19deb9d76 100644 --- a/packages/standard-components/manifest.json +++ b/packages/standard-components/manifest.json @@ -65,41 +65,6 @@ "type": "schema" } }, - "search": { - "name": "Search", - "description": "A searchable list of items.", - "icon": "Search", - "styleable": true, - "hasChildren": true, - "settings": [ - { - "type": "table", - "label": "Table", - "key": "table" - }, - { - "type": "multifield", - "label": "Columns", - "key": "columns", - "dependsOn": "table" - }, - { - "type": "number", - "label": "Rows/Page", - "defaultValue": 25, - "key": "pageSize" - }, - { - "type": "text", - "label": "Empty Text", - "key": "noRowsMessage", - "defaultValue": "No rows found." - } - ], - "context": { - "type": "schema" - } - }, "stackedlist": { "name": "Stacked List", "icon": "TaskList", @@ -1446,7 +1411,14 @@ { "type": "number", "label": "Limit", - "key": "limit" + "key": "limit", + "defaultValue": 50 + }, + { + "type": "boolean", + "label": "Paginate", + "key": "paginate", + "defaultValue": true } ], "context": { @@ -1464,14 +1436,6 @@ "label": "Schema", "key": "schema" }, - { - "label": "Loading", - "key": "loading" - }, - { - "label": "Loaded", - "key": "loaded" - }, { "label": "Page Number", "key": "pageNumber" diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte index 418d80f7a9..80ddffedee 100644 --- a/packages/standard-components/src/DataProvider.svelte +++ b/packages/standard-components/src/DataProvider.svelte @@ -1,11 +1,13 @@ -
+
- + {#if !loaded && loading} +
+ +
+ {:else} + + {#if paginate} + + {/if} + {/if}
+ + diff --git a/packages/standard-components/src/Search.svelte b/packages/standard-components/src/Search.svelte deleted file mode 100644 index 29ca3b011b..0000000000 --- a/packages/standard-components/src/Search.svelte +++ /dev/null @@ -1,195 +0,0 @@ - - - -
-
- {#if schema} - {#each columns as field} -
- - {#if schema[field].type === "options"} - - {:else if schema[field].type === "datetime"} - - {:else if schema[field].type === "boolean"} - - {:else if schema[field].type === "number"} - - {:else if schema[field].type === "string"} - - {/if} -
- {/each} - {/if} -
- - -
-
- {#if loaded} - {#if rows.length > 0} - {#if $component.children === 0 && $builderStore.inBuilder} -

Add some components to display.

- {:else} - {#each rows as row} - - - - {/each} - {/if} - {:else if noRowsMessage} -

{noRowsMessage}

- {/if} - {/if} - -
- - - From d9923e1e9885edae05696ea8e2468f6fea1ef459 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 12:30:45 +0100 Subject: [PATCH 15/37] Clean up data source context and remove explicit button actions for pagination --- .../EventsEditor/actions/NextPage.svelte | 38 ------------------- .../EventsEditor/actions/PrevPage.svelte | 38 ------------------- .../EventsEditor/actions/index.js | 10 ----- packages/client/src/constants.js | 2 - packages/client/src/utils/buttonActions.js | 33 ++++------------ packages/standard-components/manifest.json | 1 - .../src/DataProvider.svelte | 19 +--------- 7 files changed, 8 insertions(+), 133 deletions(-) delete mode 100644 packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/NextPage.svelte delete mode 100644 packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/PrevPage.svelte diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/NextPage.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/NextPage.svelte deleted file mode 100644 index a96e1bd3d5..0000000000 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/NextPage.svelte +++ /dev/null @@ -1,38 +0,0 @@ - - -
- - x._instanceName} - getOptionValue={(x) => x._id} - /> -
- - diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js index 95c92a3f6d..4700ea5c8f 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js @@ -6,8 +6,6 @@ import TriggerAutomation from "./TriggerAutomation.svelte" import ValidateForm from "./ValidateForm.svelte" import LogIn from "./LogIn.svelte" import LogOut from "./LogOut.svelte" -import NextPage from "./NextPage.svelte" -import PrevPage from "./PrevPage.svelte" // defines what actions are available, when adding a new one // the component is the setup panel for the action @@ -47,12 +45,4 @@ export default [ name: "Log Out", component: LogOut, }, - { - name: "Next Page", - component: NextPage, - }, - { - name: "Previous Page", - component: PrevPage, - }, ] diff --git a/packages/client/src/constants.js b/packages/client/src/constants.js index fc69ac212c..3aa302bec9 100644 --- a/packages/client/src/constants.js +++ b/packages/client/src/constants.js @@ -5,6 +5,4 @@ export const TableNames = { export const ActionTypes = { ValidateForm: "ValidateForm", RefreshDatasource: "RefreshDatasource", - NextPage: "NextPage", - PrevPage: "PrevPage", } diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js index f78ac1773c..4d2865d586 100644 --- a/packages/client/src/utils/buttonActions.js +++ b/packages/client/src/utils/buttonActions.js @@ -16,27 +16,27 @@ const saveRowHandler = async (action, context) => { } } -const deleteRowHandler = async (action) => { +const deleteRowHandler = async action => { const { tableId, revId, rowId } = action.parameters if (tableId && revId && rowId) { await deleteRow({ tableId, rowId, revId }) } } -const triggerAutomationHandler = async (action) => { +const triggerAutomationHandler = async action => { const { fields } = action.parameters if (fields) { await triggerAutomation(action.parameters.automationId, fields) } } -const navigationHandler = (action) => { +const navigationHandler = action => { if (action.parameters.url) { routeStore.actions.navigate(action.parameters.url) } } -const queryExecutionHandler = async (action) => { +const queryExecutionHandler = async action => { const { datasourceId, queryId, queryParams } = action.parameters await executeQuery({ datasourceId, @@ -68,23 +68,7 @@ const refreshDatasourceHandler = async (action, context) => { ) } -const nextPageHandler = async (action, context) => { - return await executeActionHandler( - context, - action.parameters.componentId, - ActionTypes.NextPage - ) -} - -const prevPageHandler = async (action, context) => { - return await executeActionHandler( - context, - action.parameters.componentId, - ActionTypes.PrevPage - ) -} - -const loginHandler = async (action) => { +const loginHandler = async action => { const { email, password } = action.parameters await authStore.actions.logIn({ email, password }) } @@ -103,8 +87,6 @@ const handlerMap = { ["Refresh Datasource"]: refreshDatasourceHandler, ["Log In"]: loginHandler, ["Log Out"]: logoutHandler, - ["Next Page"]: nextPageHandler, - ["Previous Page"]: prevPageHandler, } /** @@ -114,10 +96,9 @@ const handlerMap = { export const enrichButtonActions = (actions, context) => { // Prevent button actions in the builder preview if (get(builderStore).inBuilder) { - // TODO uncomment - // return () => {} + return () => {} } - const handlers = actions.map((def) => handlerMap[def["##eventHandlerType"]]) + const handlers = actions.map(def => handlerMap[def["##eventHandlerType"]]) return async () => { for (let i = 0; i < handlers.length; i++) { try { diff --git a/packages/standard-components/manifest.json b/packages/standard-components/manifest.json index c19deb9d76..dab68f41ce 100644 --- a/packages/standard-components/manifest.json +++ b/packages/standard-components/manifest.json @@ -1384,7 +1384,6 @@ "icon": "Data", "styleable": false, "hasChildren": true, - "actions": ["NextPage", "PrevPage"], "settings": [ { "type": "dataSource", diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte index 80ddffedee..64e387462a 100644 --- a/packages/standard-components/src/DataProvider.svelte +++ b/packages/standard-components/src/DataProvider.svelte @@ -54,25 +54,8 @@ callback: () => fetchData(dataSource), metadata: { dataSource }, }, - { - type: ActionTypes.NextPage, - callback: () => nextPage(), - }, - { - type: ActionTypes.PrevPage, - callback: () => prevPage(), - }, ] - $: dataContext = { - rows, - schema, - rowsLength: rows.length, - loading, - loaded, - pageNumber: pageNumber + 1, - hasNextPage, - hasPrevPage, - } + $: dataContext = { rows, schema, rowsLength: rows.length } const getSortType = (schema, sortColumn) => { if (!schema || !sortColumn || !schema[sortColumn]) { From 1c07e0a89533ff5bf107a0bbb0300f95c8d283a7 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 12:38:12 +0100 Subject: [PATCH 16/37] Fix bug with determing whether another page exists when performing paginated searches --- packages/server/src/api/controllers/search/utils.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js index 6f2f29628a..b0a6322648 100644 --- a/packages/server/src/api/controllers/search/utils.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -244,11 +244,15 @@ exports.paginatedSearch = async ( .setSort(sort) .setSortOrder(sortOrder) .setSortType(sortType) + const searchUrl = builder .setBookmark(bookmark) .setLimit(limit) - const searchUrl = builder.buildSearchURL() - const nextUrl = builder.setLimit(1).buildSearchURL() + .buildSearchURL() const searchResults = await runQuery(searchUrl) + const nextUrl = builder + .setBookmark(searchResults.bookmark) + .setLimit(1) + .buildSearchURL() const nextResults = await runQuery(nextUrl) return { ...searchResults, From 06d8a0970a85a0fc76e40e3d306f6c5fe700f1d9 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 12:38:34 +0100 Subject: [PATCH 17/37] Remove leftover import of search component --- packages/standard-components/src/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/standard-components/src/index.js b/packages/standard-components/src/index.js index 2ad685033a..7b4d492fa9 100644 --- a/packages/standard-components/src/index.js +++ b/packages/standard-components/src/index.js @@ -27,7 +27,6 @@ export { default as embed } from "./Embed.svelte" export { default as cardhorizontal } from "./CardHorizontal.svelte" export { default as cardstat } from "./CardStat.svelte" export { default as icon } from "./Icon.svelte" -export { default as search } from "./Search.svelte" export { default as backgroundimage } from "./BackgroundImage.svelte" export * from "./charts" export * from "./forms" From c669526c6609c1c6fce9076c34c2a4c25b1d68a6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 12:38:56 +0100 Subject: [PATCH 18/37] Remove search component from component structure --- .../src/components/design/AppPreview/componentStructure.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/builder/src/components/design/AppPreview/componentStructure.json b/packages/builder/src/components/design/AppPreview/componentStructure.json index 91f039fe68..9bc6be093e 100644 --- a/packages/builder/src/components/design/AppPreview/componentStructure.json +++ b/packages/builder/src/components/design/AppPreview/componentStructure.json @@ -4,7 +4,6 @@ "table", "repeater", "button", - "search", { "name": "Form", "icon": "Form", From 8e2aff1716d47058b8f783134cc3744188606e89 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 12:39:46 +0100 Subject: [PATCH 19/37] Remove initial work on pagination standard component --- packages/standard-components/manifest.json | 13 ---------- .../standard-components/src/Pagination.svelte | 25 ------------------- 2 files changed, 38 deletions(-) delete mode 100644 packages/standard-components/src/Pagination.svelte diff --git a/packages/standard-components/manifest.json b/packages/standard-components/manifest.json index dab68f41ce..65aa159e4e 100644 --- a/packages/standard-components/manifest.json +++ b/packages/standard-components/manifest.json @@ -1521,18 +1521,5 @@ "context": { "type": "schema" } - }, - "pagination": { - "name": "Pagination Control", - "icon": "", - "styleable": true, - "hasChildren": "false", - "settings": [ - { - "type": "dataProvider", - "label": "Provider", - "key": "dataProviderId" - } - ] } } diff --git a/packages/standard-components/src/Pagination.svelte b/packages/standard-components/src/Pagination.svelte deleted file mode 100644 index 12919cba8c..0000000000 --- a/packages/standard-components/src/Pagination.svelte +++ /dev/null @@ -1,25 +0,0 @@ - - -{#if hasValidContext} -
- Page {pageNumber} -
-{:else if builderStore.inBuilder} -
Choose a data provider to control with this pagination component.
-{/if} From f54096447d4918c35df4e9425285296e6adf8b4f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 13:11:16 +0100 Subject: [PATCH 20/37] Add jsdoc to search utils --- .../src/api/controllers/search/utils.js | 79 +++++++++++++++++-- 1 file changed, 74 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js index b0a6322648..53648eb989 100644 --- a/packages/server/src/api/controllers/search/utils.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -3,10 +3,20 @@ const { checkSlashesInUrl } = require("../../../utilities") const env = require("../../../environment") const fetch = require("node-fetch") +/** + * Escapes any characters in a string which lucene searches require to be + * escaped. + * @param value The value to escape + * @returns {string} + */ const luceneEscape = value => { return `${value}`.replace(/[ #+\-&|!(){}\[\]^"~*?:\\]/g, "\\$&") } +/** + * Class to build lucene query URLs. + * Optionally takes a base lucene query object. + */ class QueryBuilder { constructor(appId, base) { this.appId = appId @@ -105,6 +115,7 @@ class QueryBuilder { } } + // Construct the actual lucene search query string from JSON structure if (this.query.string) { build(this.query.string, (key, value) => { return value ? `${key}:${luceneEscape(value.toLowerCase())}*` : null @@ -146,6 +157,7 @@ class QueryBuilder { build(this.query.notEmpty, key => `${key}:["" TO *]`) } + // Build the full search URL let url = `${env.COUCH_DB_URL}/${this.appId}/_design/database/_search` url += `/${SearchIndexes.ROWS}?q=${output}` url += `&limit=${Math.min(this.limit, 200)}` @@ -161,11 +173,18 @@ class QueryBuilder { if (this.bookmark) { url += `&bookmark=${this.bookmark}` } + console.log(url) + // Fix any double slashes in the URL return checkSlashesInUrl(url) } } +/** + * Executes a lucene search query. + * @param query The query URL + * @returns {Promise<{rows: []}>} + */ const runQuery = async query => { const response = await fetch(query, { method: "GET", @@ -183,6 +202,22 @@ const runQuery = async query => { return output } +/** + * Gets round the fixed limit of 200 results from a query by fetching as many + * pages as required and concatenating the results. This recursively operates + * until enough results have been found. + * @param appId {string} The app ID to search + * @param query {object} The JSON query structure + * @param tableId {string} The table ID to search + * @param sort {string} The sort column + * @param sortOrder {string} The sort order ("ascending" or "descending") + * @param sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * @param limit {number} The number of results to fetch + * @param bookmark {string|null} Current bookmark in the recursive search + * @param rows {array|null} Current results in the recursive search + * @returns {Promise<*[]|*>} + */ const recursiveSearch = async ( appId, query, @@ -191,8 +226,8 @@ const recursiveSearch = async ( sortOrder, sortType, limit, - bookmark, - rows + bookmark = null, + rows = [] ) => { if (rows.length >= limit) { return rows @@ -226,6 +261,21 @@ const recursiveSearch = async ( ) } +/** + * Performs a paginated search. A bookmark will be returned to allow the next + * page to be fetched. There is a max limit off 200 results per page in a + * paginated search. + * @param appId {string} The app ID to search + * @param query {object} The JSON query structure + * @param tableId {string} The table ID to search + * @param sort {string} The sort column + * @param sortOrder {string} The sort order ("ascending" or "descending") + * @param sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * @param limit {number} The desired page size + * @param bookmark {string} The bookmark to resume from + * @returns {Promise<{hasNextPage: boolean, rows: *[]}>} + */ exports.paginatedSearch = async ( appId, query, @@ -239,6 +289,7 @@ exports.paginatedSearch = async ( if (limit == null || isNaN(limit) || limit < 0) { limit = 50 } + limit = Math.min(limit, 200) const builder = new QueryBuilder(appId, query) .setTable(tableId) .setSort(sort) @@ -249,17 +300,36 @@ exports.paginatedSearch = async ( .setLimit(limit) .buildSearchURL() const searchResults = await runQuery(searchUrl) + + // Try fetching 1 row in the next page to see if another page of results + // exists or not const nextUrl = builder .setBookmark(searchResults.bookmark) .setLimit(1) .buildSearchURL() const nextResults = await runQuery(nextUrl) + return { ...searchResults, hasNextPage: nextResults.rows && nextResults.rows.length > 0, } } +/** + * Performs a full search, fetching multiple pages if required to return the + * desired amount of results. There is a limit of 1000 results to avoid + * heavy performance hits, and to avoid client components breaking from + * handling too much data. + * @param appId {string} The app ID to search + * @param query {object} The JSON query structure + * @param tableId {string} The table ID to search + * @param sort {string} The sort column + * @param sortOrder {string} The sort order ("ascending" or "descending") + * @param sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * @param limit {number} The desired number of results + * @returns {Promise<{rows: *}>} + */ exports.fullSearch = async ( appId, query, @@ -272,6 +342,7 @@ exports.fullSearch = async ( if (limit == null || isNaN(limit) || limit < 0) { limit = 1000 } + limit = Math.min(limit, 1000) const rows = await recursiveSearch( appId, query, @@ -279,9 +350,7 @@ exports.fullSearch = async ( sort, sortOrder, sortType, - Math.min(limit, 1000), - null, - [] + limit ) return { rows } } From b578522bcd529bd5348c4555b7832b312eedc9c3 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 13:11:45 +0100 Subject: [PATCH 21/37] Wait until table schema loads before searching to avoid initial double search --- .../src/DataProvider.svelte | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte index 64e387462a..66414ac39b 100644 --- a/packages/standard-components/src/DataProvider.svelte +++ b/packages/standard-components/src/DataProvider.svelte @@ -17,6 +17,7 @@ // Loading flag for the initial load let loaded = false + let schemaLoaded = false // Provider state let rows = [] @@ -31,16 +32,23 @@ $: hasPrevPage = pageNumber > 0 $: getSchema(dataSource) $: sortType = getSortType(schema, sortColumn) - $: fetchData( - dataSource, - query, - limit, - sortColumn, - sortOrder, - sortType, - paginate - ) $: { + // Wait until schema loads before loading data, so that we can determine + // the correct sort type first time + if (schemaLoaded) { + fetchData( + dataSource, + query, + limit, + sortColumn, + sortOrder, + sortType, + paginate + ) + } + } + $: { + // Sort and limit rows in memory when we aren't searching internal tables if (internalTable) { rows = allRows } else { @@ -195,6 +203,7 @@ } }) schema = fixedSchema + schemaLoaded = true } const nextPage = async () => { From 6de9ccd28bc69ef4b9632ea4555eb2239248fda5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 13:12:59 +0100 Subject: [PATCH 22/37] Fix incorrect background colour in standard-components table background --- packages/standard-components/src/table/Table.svelte | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/standard-components/src/table/Table.svelte b/packages/standard-components/src/table/Table.svelte index f68835d75b..35839e4722 100644 --- a/packages/standard-components/src/table/Table.svelte +++ b/packages/standard-components/src/table/Table.svelte @@ -94,3 +94,9 @@
+ + From 26014f4e5ca087b6c24ff099d0680b00732e5f63 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 15:35:54 +0100 Subject: [PATCH 23/37] Remove sorting by auto ID when output processing rows --- .../bbui/src/Pagination/Pagination.svelte | 1 + .../src/api/controllers/search/utils.js | 5 ++--- packages/server/src/utilities/rowProcessor.js | 20 ------------------- 3 files changed, 3 insertions(+), 23 deletions(-) diff --git a/packages/bbui/src/Pagination/Pagination.svelte b/packages/bbui/src/Pagination/Pagination.svelte index cfe989ea14..e9ee31cc65 100644 --- a/packages/bbui/src/Pagination/Pagination.svelte +++ b/packages/bbui/src/Pagination/Pagination.svelte @@ -47,6 +47,7 @@ From d8bbb5e06d3e92df8826ac1be30678ca6e824d6c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 16:33:19 +0100 Subject: [PATCH 28/37] Simplify loading logic to prevent empty state flashing when loading data in data providers --- packages/client/src/api/tables.js | 8 ++++---- packages/standard-components/src/DataProvider.svelte | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/client/src/api/tables.js b/packages/client/src/api/tables.js index a75a2d368b..59381e35bf 100644 --- a/packages/client/src/api/tables.js +++ b/packages/client/src/api/tables.js @@ -23,7 +23,6 @@ export const fetchTableData = async tableId => { export const searchTable = async ({ tableId, query, - raw, bookmark, limit, sort, @@ -31,14 +30,15 @@ export const searchTable = async ({ sortType, paginate, }) => { - if (!tableId || (!query && !raw)) { - return + if (!tableId || !query) { + return { + rows: [], + } } const res = await API.post({ url: `/api/search/${tableId}/rows`, body: { query, - raw, bookmark, limit, sort, diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte index 66414ac39b..83f30fab86 100644 --- a/packages/standard-components/src/DataProvider.svelte +++ b/packages/standard-components/src/DataProvider.svelte @@ -114,7 +114,6 @@ sortType, paginate ) => { - console.log("FETCH") loading = true if (dataSource?.type === "table") { const res = await API.searchTable({ @@ -248,7 +247,7 @@
- {#if !loaded && loading} + {#if !loaded}
From 753f0befbce9c030ea94c527571dd130e2265424 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 14 May 2021 13:11:23 +0100 Subject: [PATCH 29/37] Add brief docs to Table component explaining the schema prop --- packages/bbui/src/Table/Table.svelte | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte index 4278751c5f..07ba7fa6e9 100644 --- a/packages/bbui/src/Table/Table.svelte +++ b/packages/bbui/src/Table/Table.svelte @@ -4,6 +4,16 @@ import CellRenderer from "./CellRenderer.svelte" import SelectEditRenderer from "./SelectEditRenderer.svelte" + /** + * The expected schema is our normal couch schemas for our tables. + * Each field schema can be enriched with a few extra properties to customise + * the behaviour. + * All of these are optional and do not need to be added. + * displayName: Overrides the field name displayed as the column title + * sortable: Set to false to disable sorting data by a certain column + * editable: Set to false to disable editing a certain column if the + * allowEditColumns prop is true + */ export let data = [] export let schema = {} export let showAutoColumns = false From 47db3df0a12ee47dde5a04ef81d001da9ebc9ead Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 14 May 2021 13:29:14 +0100 Subject: [PATCH 30/37] Add optional info text to components --- .../design/PropertiesPanel/SettingsView.svelte | 9 ++++++--- packages/standard-components/manifest.json | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/design/PropertiesPanel/SettingsView.svelte b/packages/builder/src/components/design/PropertiesPanel/SettingsView.svelte index 9a81a0dba3..eeb4fe1bb3 100644 --- a/packages/builder/src/components/design/PropertiesPanel/SettingsView.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/SettingsView.svelte @@ -156,8 +156,11 @@ {/if} {/each} {:else} -
- This component doesn't have any additional settings. +
This component doesn't have any additional settings.
+ {/if} + {#if componentDefinition?.info} +
+ {@html componentDefinition?.info}
{/if} @@ -185,7 +188,7 @@ height: 100%; gap: var(--spacing-s); } - .empty { + .text { font-size: var(--spectrum-global-dimension-font-size-75); margin-top: var(--spacing-m); color: var(--grey-6); diff --git a/packages/standard-components/manifest.json b/packages/standard-components/manifest.json index 65aa159e4e..1c8502c0f5 100644 --- a/packages/standard-components/manifest.json +++ b/packages/standard-components/manifest.json @@ -1381,6 +1381,7 @@ }, "dataprovider": { "name": "Data Provider", + "info": "Pagination can only be used with data sources that are tables.", "icon": "Data", "styleable": false, "hasChildren": true, From 6479382856d192446977d89994b8047d2c747a48 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 14 May 2021 13:29:37 +0100 Subject: [PATCH 31/37] Don't show pagination control if the data source isn't an internal table --- packages/standard-components/src/DataProvider.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte index 83f30fab86..e0b2ad859a 100644 --- a/packages/standard-components/src/DataProvider.svelte +++ b/packages/standard-components/src/DataProvider.svelte @@ -253,7 +253,7 @@
{:else} - {#if paginate} + {#if paginate && internalTable}