1
0
Fork 0
mirror of synced 2024-06-29 03:20:34 +12:00

Merge pull request #1107 from Budibase/internal-search

Internal search
This commit is contained in:
Martin McKeaveney 2021-02-10 15:44:45 +00:00 committed by GitHub
commit 5269b6be23
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 464 additions and 8077 deletions

8068
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -5,7 +5,6 @@
const rimraf = require("rimraf")
const { join, resolve } = require("path")
// const run = require("../../cli/src/commands/run/runHandler")
const initialiseBudibase = require("../../server/src/utilities/initialiseBudibase")
const homedir = join(require("os").homedir(), ".budibase")

View file

@ -232,7 +232,7 @@ export const getBackendUiStore = () => {
return state
})
},
saveField: ({ originalName, field, primaryDisplay = false }) => {
saveField: ({ originalName, field, primaryDisplay = false, indexes }) => {
store.update(state => {
// delete the original if renaming
// need to handle if the column had no name, empty string
@ -249,6 +249,10 @@ export const getBackendUiStore = () => {
state.draftTable.primaryDisplay = field.name
}
if (indexes) {
state.draftTable.indexes = indexes
}
state.draftTable.schema[field.name] = cloneDeep(field)
store.actions.tables.save(state.draftTable)
return state

View file

@ -1,5 +1,12 @@
<script>
import { Input, Button, TextButton, Select, Toggle } from "@budibase/bbui"
import {
Input,
Button,
Label,
TextButton,
Select,
Toggle,
} from "@budibase/bbui"
import { cloneDeep } from "lodash/fp"
import { backendUiStore } from "builderStore"
import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
@ -24,6 +31,7 @@
let primaryDisplay =
$backendUiStore.selectedTable.primaryDisplay == null ||
$backendUiStore.selectedTable.primaryDisplay === field.name
let indexes = [...($backendUiStore.selectedTable.indexes || [])]
let confirmDeleteDialog
let deletion
@ -41,6 +49,7 @@
originalName,
field,
primaryDisplay,
indexes,
})
return state
})
@ -79,6 +88,18 @@
}
}
function onChangePrimaryIndex(e) {
indexes = e.target.checked ? [field.name] : []
}
function onChangeSecondaryIndex(e) {
if (e.target.checked) {
indexes[1] = field.name
} else {
indexes = indexes.slice(0, 1)
}
}
function confirmDelete() {
confirmDeleteDialog.show()
deletion = true
@ -120,6 +141,20 @@
on:change={onChangePrimaryDisplay}
thin
text="Use as table display column" />
<Label gray small>Search Indexes</Label>
<Toggle
checked={indexes[0] === field.name}
disabled={indexes[1] === field.name}
on:change={onChangePrimaryIndex}
thin
text="Primary" />
<Toggle
checked={indexes[1] === field.name}
disabled={!indexes[0] || indexes[0] === field.name}
on:change={onChangeSecondaryIndex}
thin
text="Secondary" />
{/if}
{#if field.type === 'string'}

View file

@ -41,7 +41,7 @@
.icon {
right: 2px;
top: 26px;
top: 5px;
bottom: 2px;
position: absolute;
align-items: center;

View file

@ -3,6 +3,7 @@
"datagrid",
"list",
"button",
"search",
{
"name": "Form",
"icon": "ri-file-edit-line",

View file

@ -46,6 +46,11 @@
innerVal = value.target.value
}
}
if (type === "number") {
innerVal = parseInt(innerVal)
}
if (typeof innerVal === "string") {
onChange(replaceBindings(innerVal))
} else {
@ -72,6 +77,7 @@
value={safeValue}
on:change={handleChange}
onChange={handleChange}
{type}
{...props}
name={key} />
</div>

View file

@ -1,10 +1,15 @@
import { cloneDeep } from "lodash/fp"
import { fetchTableData } from "./tables"
import { fetchTableData, searchTableData } from "./tables"
import { fetchViewData } from "./views"
import { fetchRelationshipData } from "./relationships"
import { executeQuery } from "./queries"
import { enrichRows } from "./rows"
export const searchTable = async ({ tableId, search, pagination }) => {
const rows = await searchTableData({ tableId, search, pagination })
return await enrichRows(rows, tableId)
}
/**
* Fetches all rows for a particular Budibase data source.
*/

View file

@ -16,3 +16,19 @@ export const fetchTableData = async tableId => {
const rows = await API.get({ url: `/api/${tableId}/rows` })
return await enrichRows(rows, tableId)
}
/**
* Perform a mango query against an internal table
* @param {String} tableId - id of the table to search
* @param {Object} search - Mango Compliant search object
*/
export const searchTableData = async ({ tableId, search, pagination }) => {
const rows = await API.post({
url: `/api/${tableId}/rows/search`,
body: {
query: search,
pagination,
},
})
return await enrichRows(rows, tableId)
}

View file

@ -91,6 +91,7 @@
"pino-pretty": "4.0.0",
"pouchdb": "7.2.1",
"pouchdb-all-dbs": "1.0.2",
"pouchdb-find": "^7.2.2",
"pouchdb-replication-stream": "1.2.9",
"sanitize-s3-objectkey": "0.0.1",
"server-destroy": "1.0.1",

View file

@ -229,6 +229,38 @@ exports.fetchView = async function(ctx) {
}
}
exports.search = async function(ctx) {
const appId = ctx.user.appId
const db = new CouchDB(appId)
const {
query,
pagination: { pageSize = 10, page },
} = ctx.request.body
query.tableId = ctx.params.tableId
const response = await db.find({
selector: query,
limit: pageSize,
skip: pageSize * page,
})
const rows = response.docs
// delete passwords from users
if (query.tableId === ViewNames.USERS) {
for (let row of rows) {
delete row.password
}
}
const table = await db.get(ctx.params.tableId)
ctx.body = await enrichRows(appId, table, rows)
}
exports.fetchTableRows = async function(ctx) {
const appId = ctx.user.appId
const db = new CouchDB(appId)

View file

@ -7,6 +7,7 @@ const {
generateTableID,
generateRowID,
} = require("../../db/utils")
const { isEqual } = require("lodash/fp")
async function checkForColumnUpdates(db, oldTable, updatedTable) {
let updatedRows
@ -128,6 +129,46 @@ exports.save = async function(ctx) {
const result = await db.post(tableToSave)
tableToSave._rev = result.rev
// create relevant search indexes
if (tableToSave.indexes && tableToSave.indexes.length > 0) {
const currentIndexes = await db.getIndexes()
const indexName = `search:${result.id}`
const existingIndex = currentIndexes.indexes.find(
existing => existing.name === indexName
)
if (existingIndex) {
const currentFields = existingIndex.def.fields.map(
field => Object.keys(field)[0]
)
// if index fields have changed, delete the original index
if (!isEqual(currentFields, tableToSave.indexes)) {
await db.deleteIndex(existingIndex)
// create/recreate the index with fields
await db.createIndex({
index: {
fields: tableToSave.indexes,
name: indexName,
ddoc: "search_ddoc",
type: "json",
},
})
}
} else {
// create/recreate the index with fields
await db.createIndex({
index: {
fields: tableToSave.indexes,
name: indexName,
ddoc: "search_ddoc",
type: "json",
},
})
}
}
ctx.eventEmitter &&
ctx.eventEmitter.emitTable(`table:save`, appId, tableToSave)
@ -171,6 +212,15 @@ exports.destroy = async function(ctx) {
// don't remove the table itself until very end
await db.remove(tableToDelete)
// remove table search index
const currentIndexes = await db.getIndexes()
const existingIndex = currentIndexes.indexes.find(
existing => existing.name === `search:${ctx.params.tableId}`
)
if (existingIndex) {
await db.deleteIndex(existingIndex)
}
ctx.eventEmitter &&
ctx.eventEmitter.emitTable(`table:delete`, appId, tableToDelete)
ctx.status = 200

View file

@ -39,6 +39,12 @@ router
usage,
rowController.save
)
.post(
"/api/:tableId/rows/search",
paramResource("tableId"),
authorized(PermissionTypes.TABLE, PermissionLevels.READ),
rowController.search
)
.patch(
"/api/:tableId/rows/:rowId",
paramSubResource("tableId", "rowId"),

View file

@ -14,7 +14,15 @@ const selfhost = require("./selfhost")
const app = new Koa()
// set up top level koa middleware
app.use(koaBody({ multipart: true }))
app.use(
koaBody({
multipart: true,
formLimit: "10mb",
jsonLimit: "10mb",
textLimit: "10mb",
enableTypes: ["json", "form", "text"],
})
)
app.use(
logger({

View file

@ -2,12 +2,14 @@ const PouchDB = require("pouchdb")
const replicationStream = require("pouchdb-replication-stream")
const allDbs = require("pouchdb-all-dbs")
const { budibaseAppsDir } = require("../utilities/budibaseDir")
const find = require("pouchdb-find")
const env = require("../environment")
const COUCH_DB_URL = env.COUCH_DB_URL || `leveldb://${budibaseAppsDir()}/.data/`
const isInMemory = env.NODE_ENV === "jest"
PouchDB.plugin(replicationStream.plugin)
PouchDB.plugin(find)
PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
let POUCH_DB_DEFAULTS = {

View file

@ -3219,6 +3219,13 @@ fd-slicer@~1.1.0:
dependencies:
pend "~1.2.0"
fetch-cookie@0.10.1:
version "0.10.1"
resolved "https://registry.yarnpkg.com/fetch-cookie/-/fetch-cookie-0.10.1.tgz#5ea88f3d36950543c87997c27ae2aeafb4b5c4d4"
integrity sha512-beB+VEd4cNeVG1PY+ee74+PkuCQnik78pgLi5Ah/7qdUfov8IctU0vLUbBT8/10Ma5GMBeI4wtxhGrEfKNYs2g==
dependencies:
tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
fetch-cookie@0.7.3:
version "0.7.3"
resolved "https://registry.yarnpkg.com/fetch-cookie/-/fetch-cookie-0.7.3.tgz#b8d023f421dd2b2f4a0eca9cd7318a967ed4eed8"
@ -6498,6 +6505,20 @@ pouch-stream@^0.4.0:
inherits "^2.0.1"
readable-stream "^1.0.27-1"
pouchdb-abstract-mapreduce@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-abstract-mapreduce/-/pouchdb-abstract-mapreduce-7.2.2.tgz#dd1b10a83f8d24361dce9aaaab054614b39f766f"
integrity sha512-7HWN/2yV2JkwMnGnlp84lGvFtnm0Q55NiBUdbBcaT810+clCGKvhssBCrXnmwShD1SXTwT83aszsgiSfW+SnBA==
dependencies:
pouchdb-binary-utils "7.2.2"
pouchdb-collate "7.2.2"
pouchdb-collections "7.2.2"
pouchdb-errors "7.2.2"
pouchdb-fetch "7.2.2"
pouchdb-mapreduce-utils "7.2.2"
pouchdb-md5 "7.2.2"
pouchdb-utils "7.2.2"
pouchdb-adapter-leveldb-core@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-adapter-leveldb-core/-/pouchdb-adapter-leveldb-core-7.2.2.tgz#e0aa6a476e2607d7ae89f4a803c9fba6e6d05a8a"
@ -6557,6 +6578,11 @@ pouchdb-binary-utils@7.2.2:
dependencies:
buffer-from "1.1.1"
pouchdb-collate@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-collate/-/pouchdb-collate-7.2.2.tgz#fc261f5ef837c437e3445fb0abc3f125d982c37c"
integrity sha512-/SMY9GGasslknivWlCVwXMRMnQ8myKHs4WryQ5535nq1Wj/ehpqWloMwxEQGvZE1Sda3LOm7/5HwLTcB8Our+w==
pouchdb-collections@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-collections/-/pouchdb-collections-7.2.2.tgz#aeed77f33322429e3f59d59ea233b48ff0e68572"
@ -6569,6 +6595,28 @@ pouchdb-errors@7.2.2:
dependencies:
inherits "2.0.4"
pouchdb-fetch@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-fetch/-/pouchdb-fetch-7.2.2.tgz#492791236d60c899d7e9973f9aca0d7b9cc02230"
integrity sha512-lUHmaG6U3zjdMkh8Vob9GvEiRGwJfXKE02aZfjiVQgew+9SLkuOxNw3y2q4d1B6mBd273y1k2Lm0IAziRNxQnA==
dependencies:
abort-controller "3.0.0"
fetch-cookie "0.10.1"
node-fetch "2.6.0"
pouchdb-find@^7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-find/-/pouchdb-find-7.2.2.tgz#1227afdd761812d508fe0794b3e904518a721089"
integrity sha512-BmFeFVQ0kHmDehvJxNZl9OmIztCjPlZlVSdpijuFbk/Fi1EFPU1BAv3kLC+6DhZuOqU/BCoaUBY9sn66pPY2ag==
dependencies:
pouchdb-abstract-mapreduce "7.2.2"
pouchdb-collate "7.2.2"
pouchdb-errors "7.2.2"
pouchdb-fetch "7.2.2"
pouchdb-md5 "7.2.2"
pouchdb-selector-core "7.2.2"
pouchdb-utils "7.2.2"
pouchdb-json@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-json/-/pouchdb-json-7.2.2.tgz#b939be24b91a7322e9a24b8880a6e21514ec5e1f"
@ -6576,6 +6624,16 @@ pouchdb-json@7.2.2:
dependencies:
vuvuzela "1.0.3"
pouchdb-mapreduce-utils@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-mapreduce-utils/-/pouchdb-mapreduce-utils-7.2.2.tgz#13a46a3cc2a3f3b8e24861da26966904f2963146"
integrity sha512-rAllb73hIkU8rU2LJNbzlcj91KuulpwQu804/F6xF3fhZKC/4JQMClahk+N/+VATkpmLxp1zWmvmgdlwVU4HtQ==
dependencies:
argsarray "0.0.1"
inherits "2.0.4"
pouchdb-collections "7.2.2"
pouchdb-utils "7.2.2"
pouchdb-md5@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-md5/-/pouchdb-md5-7.2.2.tgz#415401acc5a844112d765bd1fb4e5d9f38fb0838"
@ -6616,6 +6674,14 @@ pouchdb-replication-stream@1.2.9:
pouchdb-promise "^6.0.4"
through2 "^2.0.0"
pouchdb-selector-core@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-selector-core/-/pouchdb-selector-core-7.2.2.tgz#264d7436a8c8ac3801f39960e79875ef7f3879a0"
integrity sha512-XYKCNv9oiNmSXV5+CgR9pkEkTFqxQGWplnVhO3W9P154H08lU0ZoNH02+uf+NjZ2kjse7Q1fxV4r401LEcGMMg==
dependencies:
pouchdb-collate "7.2.2"
pouchdb-utils "7.2.2"
pouchdb-utils@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-utils/-/pouchdb-utils-7.2.2.tgz#c17c4788f1d052b0daf4ef8797bbc4aaa3945aa4"
@ -6724,7 +6790,7 @@ pseudomap@^1.0.2:
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM=
psl@^1.1.28:
psl@^1.1.28, psl@^1.1.33:
version "1.8.0"
resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
@ -8090,6 +8156,15 @@ tough-cookie@^2.3.3, tough-cookie@^2.3.4, tough-cookie@^2.4.3, tough-cookie@~2.5
psl "^1.1.28"
punycode "^2.1.1"
"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0":
version "4.0.0"
resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4"
integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==
dependencies:
psl "^1.1.33"
punycode "^2.1.1"
universalify "^0.1.2"
tr46@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09"
@ -8217,7 +8292,7 @@ unique-string@^2.0.0:
dependencies:
crypto-random-string "^2.0.0"
universalify@^0.1.0:
universalify@^0.1.0, universalify@^0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==

View file

@ -111,6 +111,44 @@
"type": "datasource",
"label": "Data",
"key": "datasource"
},
{
"type": "text",
"label": "No Rows Message",
"key": "noRowsMessage"
}
]
},
"search": {
"name": "Search",
"description": "A searchable list of items.",
"icon": "ri-search-line",
"styleable": true,
"hasChildren": true,
"dataProvider": true,
"settings": [
{
"type": "table",
"label": "Table",
"key": "table"
},
{
"type": "multifield",
"label": "Columns",
"key": "columns",
"dependsOn": "table"
},
{
"type": "number",
"label": "Rows Per Page",
"defaultValue": 25,
"key": "pageSize"
},
{
"type": "text",
"label": "No Rows Message",
"key": "noRowsMessage",
"defaultValue": "No Rows"
}
]
},

View file

@ -6,6 +6,7 @@
const component = getContext("component")
export let datasource = []
export let noRowsMessage = "Feed me some data"
let rows = []
let loaded = false
@ -32,7 +33,7 @@
{/each}
{/if}
{:else if loaded && $builderStore.inBuilder}
<p>Feed me some data</p>
<p>{noRowsMessage}</p>
{/if}
</div>

View file

@ -1,6 +1,8 @@
<script>
import { getContext } from "svelte"
const ENTER_KEY = 13
const { authStore, styleable } = getContext("sdk")
const component = getContext("component")
@ -27,8 +29,15 @@
await authStore.actions.logIn({ email, password })
loading = false
}
function handleKeydown(evt) {
if (evt.keyCode === ENTER_KEY) {
login()
}
}
</script>
<svelte:window on:keydown={handleKeydown} />
<div class="root" use:styleable={$component.styles}>
<div class="content">
{#if logo}

View file

@ -0,0 +1,166 @@
<script>
import { getContext } from "svelte"
import { isEmpty } from "lodash/fp"
import {
Button,
DatePicker,
Label,
Select,
Toggle,
Input,
} from "@budibase/bbui"
const { API, styleable, DataProvider, builderStore } = getContext("sdk")
const component = getContext("component")
export let table = []
export let columns = []
export let pageSize
export let noRowsMessage
let rows = []
let loaded = false
let search = {}
let tableDefinition
let schema
// pagination
let page = 0
$: fetchData(table, page)
// omit empty strings
$: parsedSearch = Object.keys(search).reduce(
(acc, next) =>
search[next] === "" ? acc : { ...acc, [next]: search[next] },
{}
)
async function fetchData(table, page) {
if (!isEmpty(table)) {
const tableDef = await API.fetchTableDefinition(table)
schema = tableDef.schema
rows = await API.searchTable({
tableId: table,
search: parsedSearch,
pagination: {
pageSize,
page,
},
})
}
loaded = true
}
function nextPage() {
page += 1
}
function previousPage() {
page -= 1
}
</script>
<div use:styleable={$component.styles}>
<div class="query-builder">
{#if schema}
{#each columns as field}
<div class="form-field">
<Label extraSmall grey>{schema[field].name}</Label>
{#if schema[field].type === 'options'}
<Select secondary bind:value={search[field]}>
<option value="">Choose an option</option>
{#each schema[field].constraints.inclusion as opt}
<option>{opt}</option>
{/each}
</Select>
{:else if schema[field].type === 'datetime'}
<DatePicker bind:value={search[field]} />
{:else if schema[field].type === 'boolean'}
<Toggle text={schema[field].name} bind:checked={search[field]} />
{:else if schema[field].type === 'number'}
<Input type="number" bind:value={search[field]} />
{:else if schema[field].type === 'string'}
<Input bind:value={search[field]} />
{/if}
</div>
{/each}
{/if}
<div class="actions">
<Button
secondary
on:click={() => {
search = {}
page = 0
}}>
Reset
</Button>
<Button
primary
on:click={() => {
page = 0
fetchData(table, page)
}}>
Search
</Button>
</div>
</div>
{#if loaded}
{#if rows.length > 0}
{#if $component.children === 0 && $builderStore.inBuilder}
<p>Add some components too</p>
{:else}
{#each rows as row}
<DataProvider {row}>
<slot />
</DataProvider>
{/each}
{/if}
{:else if $builderStore.inBuilder}
<p>Feed me some data</p>
{:else}
<p>{noRowsMessage}</p>
{/if}
{/if}
<div class="pagination">
{#if page > 0}
<Button primary on:click={previousPage}>Back</Button>
{/if}
{#if rows.length === pageSize}
<Button primary on:click={nextPage}>Next</Button>
{/if}
</div>
</div>
<style>
p {
display: grid;
place-items: center;
background: #f5f5f5;
border: #ccc 1px solid;
padding: var(--spacing-m);
}
.query-builder {
padding: var(--spacing-m);
border-radius: var(--border-radius-s);
}
.actions {
display: grid;
grid-gap: var(--spacing-s);
justify-content: flex-end;
grid-auto-flow: column;
}
.form-field {
margin-bottom: var(--spacing-m);
}
.pagination {
display: grid;
grid-gap: var(--spacing-s);
justify-content: flex-end;
margin-top: var(--spacing-m);
grid-auto-flow: column;
}
</style>

View file

@ -25,4 +25,5 @@ export { default as cardhorizontal } from "./CardHorizontal.svelte"
export { default as cardstat } from "./CardStat.svelte"
export { default as newrow } from "./NewRow.svelte"
export { default as icon } from "./Icon.svelte"
export { default as search } from "./Search.svelte"
export * from "./charts"