CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-09-29 16:51:33 +13:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4214 from Budibase/feature/formula-filtering

Browse source 
Feature/formula filtering
This commit is contained in:
Michael Drury 2022-01-31 15:56:04 +00:00 committed by GitHub
commit 5ba6afb399
29 changed files with 709 additions and 214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
packages/bbui/src/Form/Field.svelte
View file

@ -6,11 +6,12 @@
export let label = null
export let labelPosition = "above"
export let error = null
export let tooltip = ""
</script>
<div class="spectrum-Form-item" class:above={labelPosition === "above"}>
{#if label}
<FieldLabel forId={id} {label} position={labelPosition} />
<FieldLabel forId={id} {label} position={labelPosition} {tooltip} />
{/if}
<div class="spectrum-Form-itemField">
<slot />

17
packages/bbui/src/Form/FieldLabel.svelte
View file

@ -1,19 +1,24 @@
<script>
import TooltipWrapper from "../Tooltip/TooltipWrapper.svelte"
import "@spectrum-css/fieldlabel/dist/index-vars.css"
export let forId
export let label
export let position = "above"
export let tooltip = ""
$: className = position === "above" ? "" : `spectrum-FieldLabel--${position}`
</script>
<label
for={forId}
class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${className}`}
>
{label || ""}
</label>
<TooltipWrapper {tooltip} size="S">
<label
for={forId}
class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${className}`}
>
{label || ""}
</label>
</TooltipWrapper>
<style>
label {

3
packages/bbui/src/Form/Select.svelte
View file

@ -17,6 +17,7 @@
export let quiet = false
export let autoWidth = false
export let sort = false
export let tooltip = ""
const dispatch = createEventDispatcher()
const onChange = e => {
@ -32,7 +33,7 @@
}
</script>
<Field {label} {labelPosition} {error}>
<Field {label} {labelPosition} {error} {tooltip}>
<Select
{quiet}
{error}

59
packages/bbui/src/Label/Label.svelte
View file

@ -1,73 +1,20 @@
<script>
import "@spectrum-css/fieldlabel/dist/index-vars.css"
import Tooltip from "../Tooltip/Tooltip.svelte"
import Icon from "../Icon/Icon.svelte"
import TooltipWrapper from "../Tooltip/TooltipWrapper.svelte"
export let size = "M"
export let tooltip = ""
export let showTooltip = false
</script>
{#if tooltip}
<div class="container">
<label
for=""
class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}
>
<slot />
</label>
<div class="icon-container">
<div
class="icon"
class:icon-small={size === "M" || size === "S"}
on:mouseover={() => (showTooltip = true)}
on:mouseleave={() => (showTooltip = false)}
>
<Icon name="InfoOutline" size="S" disabled={true} />
</div>
{#if showTooltip}
<div class="tooltip">
<Tooltip textWrapping={true} direction={"bottom"} text={tooltip} />
</div>
{/if}
</div>
</div>
{:else}
<TooltipWrapper {tooltip} {size}>
<label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}>
<slot />
</label>
{/if}
</TooltipWrapper>
<style>
label {
padding: 0;
white-space: nowrap;
}
.container {
display: flex;
align-items: center;
}
.icon-container {
position: relative;
display: flex;
justify-content: center;
margin-top: 1px;
margin-left: 5px;
margin-right: 5px;
}
.tooltip {
position: absolute;
display: flex;
justify-content: center;
top: 15px;
z-index: 1;
width: 160px;
}
.icon {
transform: scale(0.75);
}
.icon-small {
margin-top: -2px;
margin-bottom: -5px;
}
</style>

60
packages/bbui/src/Tooltip/TooltipWrapper.svelte Normal file
View file

@ -0,0 +1,60 @@
<script>
import Tooltip from "./Tooltip.svelte"
import Icon from "../Icon/Icon.svelte"
export let tooltip = ""
export let size = "M"
let showTooltip = false
</script>
<div class:container={!!tooltip}>
<slot />
{#if tooltip}
<div class="icon-container">
<div
class="icon"
class:icon-small={size === "M" || size === "S"}
on:mouseover={() => (showTooltip = true)}
on:mouseleave={() => (showTooltip = false)}
>
<Icon name="InfoOutline" size="S" disabled={true} />
</div>
{#if showTooltip}
<div class="tooltip">
<Tooltip textWrapping={true} direction={"bottom"} text={tooltip} />
</div>
{/if}
</div>
{/if}
</div>
<style>
.container {
display: flex;
align-items: center;
}
.icon-container {
position: relative;
display: flex;
justify-content: center;
margin-top: 1px;
margin-left: 5px;
margin-right: 5px;
}
.tooltip {
position: absolute;
display: flex;
justify-content: center;
top: 15px;
z-index: 1;
width: 160px;
}
.icon {
transform: scale(0.75);
}
.icon-small {
margin-top: -2px;
margin-bottom: -5px;
}
</style>

22
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View file

@ -152,6 +152,7 @@
delete field.subtype
delete field.tableId
delete field.relationshipType
delete field.formulaType
// Add in defaults and initial definition
const definition = fieldDefinitions[event.detail?.toUpperCase()]
@ -163,6 +164,9 @@
if (field.type === LINK_TYPE) {
field.relationshipType = RelationshipTypes.MANY_TO_MANY
}
if (field.type === FORMULA_TYPE) {
field.formulaType = "dynamic"
}
}
function onChangeRequired(e) {
@ -438,8 +442,22 @@
error={errors.relatedName}
/>
{:else if field.type === FORMULA_TYPE}
{#if !table.sql}
<Select
label="Formula type"
bind:value={field.formulaType}
options={[
{ label: "Dynamic", value: "dynamic" },
{ label: "Static", value: "static" },
]}
getOptionLabel={option => option.label}
getOptionValue={option => option.value}
tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered,
while static formula are calculated when the row is saved."
/>
{/if}
<ModalBindableInput
title="Handlebars Formula"
title="Formula"
label="Formula"
value={field.formula}
on:change={e => (field.formula = e.detail)}
@ -448,7 +466,7 @@
/>
{:else if field.type === AUTO_TYPE}
<Select
label="Auto Column Type"
label="Auto column type"
value={field.subtype}
on:change={e => (field.subtype = e.detail)}
options={Object.entries(getAutoColumnInformation())}

2
packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte
View file

@ -131,7 +131,7 @@
{bindings}
on:change={event => (filter.value = event.detail)}
/>
{:else if ["string", "longform", "number"].includes(filter.type)}
{:else if ["string", "longform", "number", "formula"].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if ["options", "array"].includes(filter.type)}
<Combobox

54
packages/builder/src/constants/lucene.js
View file

@ -59,24 +59,26 @@ export const NoEmptyFilterStrings = [
*/
export const getValidOperatorsForType = type => {
const Op = OperatorOptions
const stringOps = [
Op.Equals,
Op.NotEquals,
Op.StartsWith,
Op.Like,
Op.Empty,
Op.NotEmpty,
]
const numOps = [
Op.Equals,
Op.NotEquals,
Op.MoreThan,
Op.LessThan,
Op.Empty,
Op.NotEmpty,
]
if (type === "string") {
return [
Op.Equals,
Op.NotEquals,
Op.StartsWith,
Op.Like,
Op.Empty,
Op.NotEmpty,
]
return stringOps
} else if (type === "number") {
return [
Op.Equals,
Op.NotEquals,
Op.MoreThan,
Op.LessThan,
Op.Empty,
Op.NotEmpty,
]
return numOps
} else if (type === "options") {
return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
} else if (type === "array") {
@ -84,23 +86,11 @@ export const getValidOperatorsForType = type => {
} else if (type === "boolean") {
return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
} else if (type === "longform") {
return [
Op.Equals,
Op.NotEquals,
Op.StartsWith,
Op.Like,
Op.Empty,
Op.NotEmpty,
]
return stringOps
} else if (type === "datetime") {
return [
Op.Equals,
Op.NotEquals,
Op.MoreThan,
Op.LessThan,
Op.Empty,
Op.NotEmpty,
]
return numOps
} else if (type === "formula") {
return stringOps.concat([Op.MoreThan, Op.LessThan])
}
return []
}

5
packages/builder/src/helpers/searchFields.js
View file

@ -27,5 +27,8 @@ export function getFields(fields, { allowLinks } = { allowLinks: true }) {
filteredFields = filteredFields.concat(getTableFields(linkField))
}
}
return filteredFields
const staticFormulaFields = fields.filter(
field => field.type === "formula" && field.formulaType === "static"
)
return filteredFields.concat(staticFormulaFields)
}

1
packages/client/src/components/app/blocks/CardsBlock.svelte
View file

@ -39,6 +39,7 @@
number: "numberfield",
datetime: "datetimefield",
boolean: "booleanfield",
formula: "stringfield",
}
let formId

6
packages/client/src/components/app/blocks/TableBlock.svelte
View file

@ -35,6 +35,7 @@
number: "numberfield",
datetime: "datetimefield",
boolean: "booleanfield",
formula: "stringfield",
}
let formId
@ -60,10 +61,11 @@
let enrichedFilter = [...(filter || [])]
columns?.forEach(column => {
const safePath = column.name.split(".").map(safe).join(".")
const stringType = column.type === "string" || column.type === "formula"
enrichedFilter.push({
field: column.name,
operator: column.type === "string" ? "string" : "equal",
type: column.type === "string" ? "string" : "number",
operator: stringType ? "string" : "equal",
type: stringType ? "string" : "number",
valueType: "Binding",
value: `{{ ${safe(formId)}.${safePath} }}`,
})

10
packages/client/src/components/app/dynamic-filter/FilterModal.svelte
View file

@ -19,10 +19,14 @@
export let schemaFields
export let filters = []
const BannedTypes = ["link", "attachment", "formula", "json"]
const BannedTypes = ["link", "attachment", "json"]
$: fieldOptions = (schemaFields ?? [])
.filter(field => !BannedTypes.includes(field.type))
.filter(
field =>
!BannedTypes.includes(field.type) ||
(field.type === "formula" && field.formulaType === "static")
)
.map(field => field.name)
const addFilter = () => {
@ -114,7 +118,7 @@
on:change={e => onOperatorChange(filter, e.detail)}
placeholder={null}
/>
{#if ["string", "longform", "number"].includes(filter.type)}
{#if ["string", "longform", "number", "formula"].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if ["options", "array"].includes(filter.type)}
<Combobox

3
packages/client/src/components/app/forms/Field.svelte
View file

@ -32,6 +32,7 @@
validation,
formStep
)
$: schemaType = fieldSchema?.type !== "formula" ? fieldSchema?.type : "string"
// Focus label when editing
let labelNode
@ -72,7 +73,7 @@
<Placeholder
text="Add the Field setting to start using your component"
/>
{:else if fieldSchema?.type && fieldSchema?.type !== type && type !== "options"}
{:else if schemaType && schemaType !== type && type !== "options"}
<Placeholder
text="This Field setting is the wrong data type for this component"
/>

79
packages/server/src/api/controllers/row/internal.js
View file

@ -1,8 +1,8 @@
const CouchDB = require("../../../db")
const linkRows = require("../../../db/linkedRows")
const {
getRowParams,
generateRowID,
getRowParams,
DocumentTypes,
InternalTables,
} = require("../../../db/utils")
@ -10,11 +10,9 @@ const userController = require("../user")
const {
inputProcessing,
outputProcessing,
processAutoColumn,
cleanupAttachments,
} = require("../../../utilities/rowProcessor")
const { FieldTypes } = require("../../../constants")
const { isEqual } = require("lodash")
const { validate, findRow } = require("./utils")
const { fullSearch, paginatedSearch } = require("./internalSearch")
const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
@ -27,6 +25,7 @@ const {
getFromMemoryDoc,
} = require("../view/utils")
const { cloneDeep } = require("lodash/fp")
const { finaliseRow, updateRelatedFormula } = require("./staticFormula")
const CALCULATION_TYPES = {
SUM: "sum",
@ -34,51 +33,6 @@ const CALCULATION_TYPES = {
STATS: "stats",
}
async function storeResponse(ctx, db, row, oldTable, table) {
row.type = "row"
// don't worry about rev, tables handle rev/lastID updates
// if another row has been written since processing this will
// handle the auto ID clash
if (!isEqual(oldTable, table)) {
try {
await db.put(table)
} catch (err) {
if (err.status === 409) {
const updatedTable = await db.get(table._id)
let response = processAutoColumn(null, updatedTable, row, {
reprocessing: true,
})
await db.put(response.table)
row = response.row
} else {
throw err
}
}
}
const response = await db.put(row)
row._rev = response.rev
// process the row before return, to include relationships
row = await outputProcessing(ctx, table, row, { squash: false })
return { row, table }
}
// doesn't do the outputProcessing
async function getRawTableData(ctx, db, tableId) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows
}
async function getView(db, viewName) {
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
@ -105,6 +59,22 @@ async function getView(db, viewName) {
return viewInfo
}
async function getRawTableData(ctx, db, tableId) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows
}
exports.patch = async ctx => {
const appId = ctx.appId
const db = new CouchDB(appId)
@ -162,7 +132,10 @@ exports.patch = async ctx => {
return { row: ctx.body, table }
}
return storeResponse(ctx, db, row, dbTable, table)
return finaliseRow(ctx.appId, table, row, {
oldTable: dbTable,
updateFormula: true,
})
}
exports.save = async function (ctx) {
@ -196,7 +169,10 @@ exports.save = async function (ctx) {
table,
})
return storeResponse(ctx, db, row, dbTable, table)
return finaliseRow(ctx.appId, table, row, {
oldTable: dbTable,
updateFormula: true,
})
}
exports.fetchView = async ctx => {
@ -302,6 +278,8 @@ exports.destroy = async function (ctx) {
})
// remove any attachments that were on the row from object storage
await cleanupAttachments(appId, table, { row })
// remove any static formula
await updateRelatedFormula(appId, table, row)
let response
if (ctx.params.tableId === InternalTables.USER_METADATA) {
@ -350,6 +328,7 @@ exports.bulkDestroy = async ctx => {
}
// remove any attachments that were on the rows from object storage
await cleanupAttachments(appId, table, { rows })
await updateRelatedFormula(appId, table, rows)
await Promise.all(updates)
return { response: { ok: true }, rows }
}

16
packages/server/src/api/controllers/row/internalSearch.js
View file

@ -37,22 +37,30 @@ class QueryBuilder {
}
setLimit(limit) {
this.limit = limit
if (limit != null) {
this.limit = limit
}
return this
}
setSort(sort) {
this.sort = sort
if (sort != null) {
this.sort = sort
}
return this
}
setSortOrder(sortOrder) {
this.sortOrder = sortOrder
if (sortOrder != null) {
this.sortOrder = sortOrder
}
return this
}
setSortType(sortType) {
this.sortType = sortType
if (sortType != null) {
this.sortType = sortType
}
return this
}

157
packages/server/src/api/controllers/row/staticFormula.js Normal file
View file

@ -0,0 +1,157 @@
const CouchDB = require("../../../db")
const { getRowParams } = require("../../../db/utils")
const {
outputProcessing,
processAutoColumn,
processFormulas,
} = require("../../../utilities/rowProcessor")
const { FieldTypes, FormulaTypes } = require("../../../constants")
const { isEqual } = require("lodash")
const { cloneDeep } = require("lodash/fp")
/**
* This function runs through a list of enriched rows, looks at the rows which
* are related and then checks if they need the state of their formulas
* updated.
* NOTE: this will only for affect static formulas.
*/
exports.updateRelatedFormula = async (appId, table, enrichedRows) => {
const db = new CouchDB(appId)
// no formula to update, we're done
if (!table.relatedFormula) {
return
}
let promises = []
for (let enrichedRow of Array.isArray(enrichedRows)
? enrichedRows
: [enrichedRows]) {
// the related rows by tableId
let relatedRows = {}
for (let [key, field] of Object.entries(enrichedRow)) {
const columnDefinition = table.schema[key]
if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
const relatedTableId = columnDefinition.tableId
if (!relatedRows[relatedTableId]) {
relatedRows[relatedTableId] = []
}
relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
}
}
for (let tableId of table.relatedFormula) {
let relatedTable
try {
// no rows to update, skip
if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
continue
}
relatedTable = await db.get(tableId)
} catch (err) {
// no error scenario, table doesn't seem to exist anymore, ignore
}
for (let column of Object.values(relatedTable.schema)) {
// needs updated in related rows
if (
column.type === FieldTypes.FORMULA &&
column.formulaType === FormulaTypes.STATIC
) {
// re-enrich rows for all the related, don't update the related formula for them
promises = promises.concat(
relatedRows[tableId].map(related =>
exports.finaliseRow(appId, relatedTable, related, {
updateFormula: false,
})
)
)
}
}
}
}
await Promise.all(promises)
}
exports.updateAllFormulasInTable = async (appId, table) => {
const db = new CouchDB(appId)
// start by getting the raw rows (which will be written back to DB after update)
let rows = (
await db.allDocs(
getRowParams(table._id, null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
// now enrich the rows, note the clone so that we have the base state of the
// rows so that we don't write any of the enriched information back
let enrichedRows = await outputProcessing({ appId }, table, cloneDeep(rows), {
squash: false,
})
const updatedRows = []
for (let row of rows) {
// find the enriched row, if found process the formulas
const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id)
if (enrichedRow) {
const processed = processFormulas(table, cloneDeep(row), {
dynamic: false,
contextRows: enrichedRow,
})
// values have changed, need to add to bulk docs to update
if (!isEqual(processed, row)) {
updatedRows.push(processed)
}
}
}
await db.bulkDocs(updatedRows)
}
/**
* This function runs at the end of the save/patch functions of the row controller, all this
* really does is enrich the row, handle any static formula processing, then return the enriched
* row. The reason we need to return the enriched row is that the automation row created trigger
* expects the row to be totally enriched/contain all relationships.
*/
exports.finaliseRow = async (
appId,
table,
row,
{ oldTable, updateFormula } = { updateFormula: true }
) => {
const db = new CouchDB(appId)
row.type = "row"
// process the row before return, to include relationships
let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), {
squash: false,
})
// use enriched row to generate formulas for saving, specifically only use as context
row = processFormulas(table, row, {
dynamic: false,
contextRows: enrichedRow,
})
// don't worry about rev, tables handle rev/lastID updates
// if another row has been written since processing this will
// handle the auto ID clash
if (oldTable && !isEqual(oldTable, table)) {
try {
await db.put(table)
} catch (err) {
if (err.status === 409) {
const updatedTable = await db.get(table._id)
let response = processAutoColumn(null, updatedTable, row, {
reprocessing: true,
})
await db.put(response.table)
row = response.row
} else {
throw err
}
}
}
const response = await db.put(row)
// for response, calculate the formulas for the enriched row
enrichedRow._rev = response.rev
enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
// this updates the related formulas in other rows based on the relations to this row
if (updateFormula) {
await exports.updateRelatedFormula(appId, table, enrichedRow)
}
return { row: enrichedRow, table }
}

183
packages/server/src/api/controllers/table/bulkFormula.js Normal file
View file

@ -0,0 +1,183 @@
const CouchDB = require("../../../db")
const { FieldTypes, FormulaTypes } = require("../../../constants")
const { getAllInternalTables, clearColumns } = require("./utils")
const { doesContainStrings } = require("@budibase/string-templates")
const { cloneDeep } = require("lodash/fp")
const { isEqual, uniq } = require("lodash")
const { updateAllFormulasInTable } = require("../row/staticFormula")
function isStaticFormula(column) {
return (
column.type === FieldTypes.FORMULA &&
column.formulaType === FormulaTypes.STATIC
)
}
/**
* This retrieves the formula columns from a table schema that use a specified column name
* in the formula.
*/
function getFormulaThatUseColumn(table, columnNames) {
let formula = []
columnNames = Array.isArray(columnNames) ? columnNames : [columnNames]
for (let column of Object.values(table.schema)) {
// not a static formula, or doesn't contain a relationship
if (!isStaticFormula(column)) {
continue
}
if (!doesContainStrings(column.formula, columnNames)) {
continue
}
formula.push(column.name)
}
return formula
}
/**
* This functions checks for when a related table, column or related column is deleted, if any
* tables need to have the formula column removed.
*/
async function checkIfFormulaNeedsCleared(
appId,
table,
{ oldTable, deletion }
) {
const db = new CouchDB(appId)
// start by retrieving all tables, remove the current table from the list
const tables = (await getAllInternalTables(appId)).filter(
tbl => tbl._id !== table._id
)
const schemaToUse = oldTable ? oldTable.schema : table.schema
let removedColumns = Object.values(schemaToUse).filter(
column => deletion || !table.schema[column.name]
)
// remove any formula columns that used related columns
for (let removed of removedColumns) {
let tableToUse = table
// if relationship, get the related table
if (removed.type === FieldTypes.LINK) {
tableToUse = tables.find(table => table._id === removed.tableId)
}
const columnsToDelete = getFormulaThatUseColumn(tableToUse, removed.name)
if (columnsToDelete.length > 0) {
await clearColumns(db, table, columnsToDelete)
}
// need a special case, where a column has been removed from this table, but was used
// in a different, related tables formula
if (!table.relatedFormula) {
continue
}
for (let relatedTableId of table.relatedFormula) {
const relatedColumns = Object.values(table.schema).filter(
column => column.tableId === relatedTableId
)
const relatedTable = tables.find(table => table._id === relatedTableId)
// look to see if the column was used in a relationship formula,
// relationships won't be used for this
if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
let relatedFormulaToRemove = []
for (let column of relatedColumns) {
relatedFormulaToRemove = relatedFormulaToRemove.concat(
getFormulaThatUseColumn(relatedTable, [
column.fieldName,
removed.name,
])
)
}
if (relatedFormulaToRemove.length > 0) {
await clearColumns(db, relatedTable, uniq(relatedFormulaToRemove))
}
}
}
}
}
/**
* This function adds a note to related tables that they are
* used in a static formula - so that the link controller
* can manage hydrating related rows formula fields. This is
* specifically only for static formula.
*/
async function updateRelatedFormulaLinksOnTables(
appId,
table,
{ deletion } = { deletion: false }
) {
const db = new CouchDB(appId)
// start by retrieving all tables, remove the current table from the list
const tables = (await getAllInternalTables(appId)).filter(
tbl => tbl._id !== table._id
)
// clone the tables, so we can compare at end
const initialTables = cloneDeep(tables)
// first find the related column names
const relatedColumns = Object.values(table.schema).filter(
col => col.type === FieldTypes.LINK
)
// we start by removing the formula field from all tables
for (let otherTable of tables) {
if (!otherTable.relatedFormula) {
continue
}
const index = otherTable.relatedFormula.indexOf(table._id)
if (index !== -1) {
otherTable.relatedFormula.splice(index, 1)
}
}
// if deleting, just remove the table IDs, don't try add
if (!deletion) {
for (let relatedCol of relatedColumns) {
let columns = getFormulaThatUseColumn(table, relatedCol.name)
if (!columns || columns.length === 0) {
continue
}
const relatedTable = tables.find(
related => related._id === relatedCol.tableId
)
// check if the table is already in the list of related formula, if it isn't, then add it
if (
relatedTable &&
(!relatedTable.relatedFormula ||
!relatedTable.relatedFormula.includes(table._id))
) {
relatedTable.relatedFormula = relatedTable.relatedFormula
? [...relatedTable.relatedFormula, table._id]
: [table._id]
}
}
}
// now we just need to compare all the tables and see if any need saved
for (let initial of initialTables) {
const found = tables.find(tbl => initial._id === tbl._id)
if (found && !isEqual(initial, found)) {
await db.put(found)
}
}
}
async function checkIfFormulaUpdated(appId, table, { oldTable }) {
// look to see if any formula values have changed
const shouldUpdate = Object.values(table.schema).find(
column =>
isStaticFormula(column) &&
(!oldTable ||
!oldTable.schema[column.name] ||
!isEqual(oldTable.schema[column.name], column))
)
// if a static formula column has updated, then need to run the update
if (shouldUpdate != null) {
await updateAllFormulasInTable(appId, table)
}
}
exports.runStaticFormulaChecks = async (
appId,
table,
{ oldTable, deletion }
) => {
await updateRelatedFormulaLinksOnTables(appId, table, { deletion })
await checkIfFormulaNeedsCleared(appId, table, { oldTable, deletion })
if (!deletion) {
await checkIfFormulaUpdated(appId, table, { oldTable })
}
}

20
packages/server/src/api/controllers/table/index.js
View file

@ -3,12 +3,8 @@ const internal = require("./internal")
const external = require("./external")
const csvParser = require("../../../utilities/csvParser")
const { isExternalTable, isSQL } = require("../../../integrations/utils")
const {
getTableParams,
getDatasourceParams,
BudibaseInternalDB,
} = require("../../../db/utils")
const { getTable } = require("./utils")
const { getDatasourceParams } = require("../../../db/utils")
const { getTable, getAllInternalTables } = require("./utils")
function pickApi({ tableId, table }) {
if (table && !tableId) {
@ -26,17 +22,7 @@ function pickApi({ tableId, table }) {
exports.fetch = async function (ctx) {
const db = new CouchDB(ctx.appId)
const internalTables = await db.allDocs(
getTableParams(null, {
include_docs: true,
})
)
const internal = internalTables.rows.map(tableDoc => ({
...tableDoc.doc,
type: "internal",
sourceId: BudibaseInternalDB._id,
}))
const internal = await getAllInternalTables(ctx.appId)
const externalTables = await db.allDocs(
getDatasourceParams("plus", {

8
packages/server/src/api/controllers/table/internal.js
View file

@ -9,6 +9,8 @@ const {
handleDataImport,
} = require("./utils")
const usageQuota = require("../../../utilities/usageQuota")
const { cleanupAttachments } = require("../../../utilities/rowProcessor")
const { runStaticFormulaChecks } = require("./bulkFormula")
exports.save = async function (ctx) {
const appId = ctx.appId
@ -104,7 +106,8 @@ exports.save = async function (ctx) {
tableToSave._rev = result.rev
tableToSave = await tableSaveFunctions.after(tableToSave)
// has to run after, make sure it has _id
await runStaticFormulaChecks(appId, tableToSave, { oldTable })
return tableToSave
}
@ -141,6 +144,9 @@ exports.destroy = async function (ctx) {
await db.deleteIndex(existingIndex)
}
// has to run after, make sure it has _id
await runStaticFormulaChecks(appId, tableToDelete, { deletion: true })
await cleanupAttachments(appId, tableToDelete, { rows })
return tableToDelete
}

56
packages/server/src/api/controllers/table/utils.js
View file

@ -4,10 +4,15 @@ const {
getRowParams,
generateRowID,
InternalTables,
getTableParams,
BudibaseInternalDB,
} = require("../../../db/utils")
const { isEqual } = require("lodash/fp")
const { isEqual } = require("lodash")
const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
const { inputProcessing } = require("../../../utilities/rowProcessor")
const {
inputProcessing,
cleanupAttachments,
} = require("../../../utilities/rowProcessor")
const {
USERS_TABLE_SCHEMA,
SwitchableTypes,
@ -21,8 +26,24 @@ const {
const { getViews, saveView } = require("../view/utils")
const viewTemplate = require("../view/viewBuilder")
const usageQuota = require("../../../utilities/usageQuota")
const { cloneDeep } = require("lodash/fp")
exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
exports.clearColumns = async (appId, table, columnNames) => {
const db = new CouchDB(appId)
const rows = await db.allDocs(
getRowParams(table._id, null, {
include_docs: true,
})
)
return db.bulkDocs(
rows.rows.map(({ doc }) => {
columnNames.forEach(colName => delete doc[colName])
return doc
})
)
}
exports.checkForColumnUpdates = async (appId, db, oldTable, updatedTable) => {
let updatedRows = []
const rename = updatedTable._rename
let deletedColumns = []
@ -39,16 +60,20 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
include_docs: true,
})
)
updatedRows = rows.rows.map(({ doc }) => {
const rawRows = rows.rows.map(({ doc }) => doc)
updatedRows = rawRows.map(row => {
row = cloneDeep(row)
if (rename) {
doc[rename.updated] = doc[rename.old]
delete doc[rename.old]
row[rename.updated] = row[rename.old]
delete row[rename.old]
} else if (deletedColumns.length !== 0) {
deletedColumns.forEach(colName => delete doc[colName])
deletedColumns.forEach(colName => delete row[colName])
}
return doc
return row
})
// cleanup any attachments from object storage for deleted attachment columns
await cleanupAttachments(appId, updatedTable, { oldTable, rows: rawRows })
// Update views
await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
delete updatedTable._rename
@ -209,6 +234,7 @@ class TableSaveFunctions {
// when confirmed valid
async mid(table) {
let response = await exports.checkForColumnUpdates(
this.appId,
this.db,
this.oldTable,
table
@ -234,6 +260,20 @@ class TableSaveFunctions {
}
}
exports.getAllInternalTables = async appId => {
const db = new CouchDB(appId)
const internalTables = await db.allDocs(
getTableParams(null, {
include_docs: true,
})
)
return internalTables.rows.map(tableDoc => ({
...tableDoc.doc,
type: "internal",
sourceId: BudibaseInternalDB._id,
}))
}
exports.getAllExternalTables = async (appId, datasourceId) => {
const db = new CouchDB(appId)
const datasource = await db.get(datasourceId)

5
packages/server/src/constants/index.js
View file

@ -61,6 +61,11 @@ exports.RelationshipTypes = {
MANY_TO_MANY: "many-to-many",
}
exports.FormulaTypes = {
STATIC: "static",
DYNAMIC: "dynamic",
}
exports.AuthTypes = {
APP: "app",
BUILDER: "builder",

9
packages/server/src/db/linkedRows/index.js
View file

@ -72,7 +72,7 @@ async function getLinksForRows(appId, rows) {
)
}
async function getFullLinkedDocs(ctx, appId, links) {
async function getFullLinkedDocs(appId, links) {
// create DBs
const db = new CouchDB(appId)
const linkedRowIds = links.map(link => link.id)
@ -146,13 +146,12 @@ exports.updateLinks = async function (args) {
/**
* Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
* This is required for formula fields, this may only be utilised internally (for now).
* @param {object} ctx The request which is looking for rows.
* @param {string} appId The ID of the app which this request is in the context of.
* @param {object} table The table from which the rows originated.
* @param {array<object>} rows The rows which are to be enriched.
* @return {Promise<*>} returns the rows with all of the enriched relationships on it.
*/
exports.attachFullLinkedDocs = async (ctx, table, rows) => {
const appId = ctx.appId
exports.attachFullLinkedDocs = async (appId, table, rows) => {
const linkedTableIds = getLinkedTableIDs(table)
if (linkedTableIds.length === 0) {
return rows
@ -166,7 +165,7 @@ exports.attachFullLinkedDocs = async (ctx, table, rows) => {
// clear any existing links that could be dupe'd
rows = clearRelationshipFields(table, rows)
// now get the docs and combine into the rows
let linked = await getFullLinkedDocs(ctx, appId, links)
let linked = await getFullLinkedDocs(appId, links)
const linkedTables = []
for (let row of rows) {
for (let link of links.filter(link => link.thisId === row._id)) {

3
packages/server/src/definitions/common.ts
View file

@ -17,6 +17,8 @@ export interface FieldSchema {
autocolumn?: boolean
throughFrom?: string
throughTo?: string
formula?: string
formulaType?: string
main?: boolean
meta?: {
toTable: string
@ -46,6 +48,7 @@ export interface Table extends Base {
schema: TableSchema
primaryDisplay?: string
sourceId?: string
relatedFormula?: string[]
constrained?: string[]
}

36
packages/server/src/utilities/rowProcessor/index.js
View file

@ -180,6 +180,8 @@ function processAutoColumn(
}
exports.processAutoColumn = processAutoColumn
exports.processFormulas = processFormulas
/**
* This will coerce a value to the correct types based on the type transform map
* @param {object} row The value to coerce
@ -229,11 +231,12 @@ exports.inputProcessing = (
}
continue
}
// specific case to delete formula values if they get saved
// type coercion cannot completely remove the field, so have to do it here
// remove any formula values, they are to be generated
if (field.type === FieldTypes.FORMULA) {
delete clonedRow[key]
} else {
}
// otherwise coerce what is there to correct types
else {
clonedRow[key] = exports.coerce(value, field.type)
}
}
@ -250,7 +253,7 @@ exports.inputProcessing = (
/**
* This function enriches the input rows with anything they are supposed to contain, for example
* link records or attachment links.
* @param {object} ctx the request which is looking for enriched rows.
* @param {string} appId the app in which the request is looking for enriched rows.
* @param {object} table the table from which these rows came from originally, this is used to determine
* the schema of the rows and then enrich.
* @param {object[]|object} rows the rows which are to be enriched.
@ -258,22 +261,21 @@ exports.inputProcessing = (
* @returns {object[]|object} the enriched rows will be returned.
*/
exports.outputProcessing = async (
ctx,
{ appId },
table,
rows,
opts = { squash: true }
) => {
const appId = ctx.appId
let wasArray = true
if (!(rows instanceof Array)) {
rows = [rows]
wasArray = false
}
// attach any linked row information
let enriched = await linkRows.attachFullLinkedDocs(ctx, table, rows)
let enriched = await linkRows.attachFullLinkedDocs(appId, table, rows)
// process formulas
enriched = processFormulas(table, enriched)
enriched = processFormulas(table, enriched, { dynamic: true })
// update the attachments URL depending on hosting
for (let [property, column] of Object.entries(table.schema)) {
@ -305,9 +307,15 @@ exports.outputProcessing = async (
* @param {any} row optional - the row being removed.
* @param {any} rows optional - if multiple rows being deleted can do this in bulk.
* @param {any} oldRow optional - if updating a row this will determine the difference.
* @param {any} oldTable optional - if updating a table, can supply the old table to look for
* deleted attachment columns.
* @return {Promise<void>} When all attachments have been removed this will return.
*/
exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
exports.cleanupAttachments = async (
appId,
table,
{ row, rows, oldRow, oldTable }
) => {
if (!isProdAppID(appId)) {
const prodAppId = getDeployedAppID(appId)
// if prod exists, then don't allow deleting
@ -322,12 +330,16 @@ exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
files = files.concat(row[key].map(attachment => attachment.key))
}
}
for (let [key, schema] of Object.entries(table.schema)) {
const schemaToUse = oldTable ? oldTable.schema : table.schema
for (let [key, schema] of Object.entries(schemaToUse)) {
if (schema.type !== FieldTypes.ATTACHMENT) {
continue
}
// if updating, need to manage the differences
if (oldRow && row) {
// old table had this column, new table doesn't - delete it
if (oldTable && !table.schema[key]) {
rows.forEach(row => addFiles(row, key))
} else if (oldRow && row) {
// if updating, need to manage the differences
files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
} else if (row) {
addFiles(row, key)

30
packages/server/src/utilities/rowProcessor/utils.js
View file

@ -1,23 +1,39 @@
const { FieldTypes } = require("../../constants")
const { FieldTypes, FormulaTypes } = require("../../constants")
const { processStringSync } = require("@budibase/string-templates")
/**
* Looks through the rows provided and finds formulas - which it then processes.
*/
exports.processFormulas = (table, rows) => {
exports.processFormulas = (
table,
rows,
{ dynamic, contextRows } = { dynamic: true }
) => {
const single = !Array.isArray(rows)
if (single) {
rows = [rows]
contextRows = contextRows ? [contextRows] : contextRows
}
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.FORMULA) {
const isStatic = schema.formulaType === FormulaTypes.STATIC
if (
schema.type !== FieldTypes.FORMULA ||
(dynamic && isStatic) ||
(!dynamic && !isStatic)
) {
continue
}
// iterate through rows and process formula
rows = rows.map(row => ({
...row,
[column]: processStringSync(schema.formula, row),
}))
for (let i = 0; i < rows.length; i++) {
if (schema.formula) {
let row = rows[i]
let context = contextRows ? contextRows[i] : row
rows[i] = {
...row,
[column]: processStringSync(schema.formula, context),
}
}
}
}
return single ? rows[0] : rows
}

2
packages/string-templates/src/index.cjs
View file

@ -15,6 +15,8 @@ module.exports.processStringSync = templates.processStringSync
module.exports.processObjectSync = templates.processObjectSync
module.exports.processString = templates.processString
module.exports.processObject = templates.processObject
module.exports.doesContainStrings = templates.doesContainStrings
module.exports.doesContainString = templates.doesContainString
/**
* Use vm2 to run JS scripts in a node env

53
packages/string-templates/src/index.js
View file

@ -3,6 +3,7 @@ const { registerAll } = require("./helpers/index")
const processors = require("./processors")
const { atob, btoa } = require("./utilities")
const manifest = require("../manifest.json")
const { FIND_HBS_REGEX } = require("./utilities")
const hbsInstance = handlebars.create()
registerAll(hbsInstance)
@ -26,7 +27,7 @@ function testObject(object) {
* @param {object|array} object The input structure which is to be recursed, it is important to note that
* if the structure contains any cycles then this will fail.
* @param {object} context The context that handlebars should fill data from.
* @param {object} opts optional - specify some options for processing.
* @param {object|null} opts optional - specify some options for processing.
* @returns {Promise<object|array>} The structure input, as fully updated as possible.
*/
module.exports.processObject = async (object, context, opts) => {
@ -57,7 +58,7 @@ module.exports.processObject = async (object, context, opts) => {
* then nothing will occur.
* @param {string} string The template string which is the filled from the context object.
* @param {object} context An object of information which will be used to enrich the string.
* @param {object} opts optional - specify some options for processing.
* @param {object|null} opts optional - specify some options for processing.
* @returns {Promise<string>} The enriched string, all templates should have been replaced if they can be.
*/
module.exports.processString = async (string, context, opts) => {
@ -71,7 +72,7 @@ module.exports.processString = async (string, context, opts) => {
* @param {object|array} object The input structure which is to be recursed, it is important to note that
* if the structure contains any cycles then this will fail.
* @param {object} context The context that handlebars should fill data from.
* @param {object} opts optional - specify some options for processing.
* @param {object|null} opts optional - specify some options for processing.
* @returns {object|array} The structure input, as fully updated as possible.
*/
module.exports.processObjectSync = (object, context, opts) => {
@ -92,7 +93,7 @@ module.exports.processObjectSync = (object, context, opts) => {
* then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call.
* @param {string} string The template string which is the filled from the context object.
* @param {object} context An object of information which will be used to enrich the string.
* @param {object} opts optional - specify some options for processing.
* @param {object|null} opts optional - specify some options for processing.
* @returns {string} The enriched string, all templates should have been replaced if they can be.
*/
module.exports.processStringSync = (string, context, opts) => {
@ -222,3 +223,47 @@ module.exports.decodeJSBinding = handlebars => {
}
return atob(match[1])
}
/**
* Same as the doesContainString function, but will check for all the strings
* before confirming it contains.
* @param {string} template The template string to search.
* @param {string[]} strings The strings to look for.
* @returns {boolean} Will return true if all strings found in HBS statement.
*/
module.exports.doesContainStrings = (template, strings) => {
let regexp = new RegExp(FIND_HBS_REGEX)
let matches = template.match(regexp)
if (matches == null) {
return false
}
for (let match of matches) {
let hbs = match
if (exports.isJSBinding(match)) {
hbs = exports.decodeJSBinding(match)
}
let allFound = true
for (let string of strings) {
if (!hbs.includes(string)) {
allFound = false
}
}
if (allFound) {
return true
}
}
return false
}
/**
* This function looks in the supplied template for handlebars instances, if they contain
* JS the JS will be decoded and then the supplied string will be looked for. For example
* if the template "Hello, your name is {{ related }}" this function would return that true
* for the string "related" but not for "name" as it is not within the handlebars statement.
* @param {string} template A template string to search for handlebars instances.
* @param {string} string The word or sentence to search for.
* @returns {boolean} The this return true if the string is found, false if not.
*/
module.exports.doesContainString = (template, string) => {
return exports.doesContainStrings(template, [string])
}

2
packages/string-templates/src/index.mjs
View file

@ -15,6 +15,8 @@ export const processStringSync = templates.processStringSync
export const processObjectSync = templates.processObjectSync
export const processString = templates.processString
export const processObject = templates.processObject
export const doesContainStrings = templates.doesContainStrings
export const doesContainString = templates.doesContainString
/**
* Use polyfilled vm to run JS scripts in a browser Env

19
packages/string-templates/test/basic.spec.js
View file

@ -4,6 +4,8 @@ const {
isValid,
makePropSafe,
getManifest,
encodeJSBinding,
doesContainString,
} = require("../src/index.cjs")
describe("Test that the string processing works correctly", () => {
@ -157,3 +159,20 @@ describe("check full stops that are safe", () => {
expect(output).toEqual("1")
})
})
describe("check does contain string function", () => {
it("should work for a simple case", () => {
const hbs = "hello {{ name }}"
expect(doesContainString(hbs, "name")).toEqual(true)
})
it("should reject a case where its in the string, but not the handlebars", () => {
const hbs = "hello {{ name }}"
expect(doesContainString(hbs, "hello")).toEqual(false)
})
it("should handle if its in javascript", () => {
const js = encodeJSBinding(`return $("foo")`)
expect(doesContainString(js, "foo")).toEqual(true)
})
})