diff --git a/.tool-versions b/.tool-versions index 946d5198ce..cf78481d93 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ nodejs 20.10.0 python 3.10.0 -yarn 1.22.19 +yarn 1.22.22 diff --git a/lerna.json b/lerna.json index 10d36c9eaf..092e9a133e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.8", + "version": "2.32.10", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 0130c39715..a1bd54715b 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -21,6 +21,7 @@ PROTECTED_EXTERNAL_COLUMNS, canHaveDefaultColumn, } from "@budibase/shared-core" + import { makePropSafe } from "@budibase/string-templates" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -46,6 +47,7 @@ import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" import OptionsEditor from "./OptionsEditor.svelte" import { isEnabled } from "helpers/featureFlags" + import { getUserBindings } from "dataBinding" const AUTO_TYPE = FieldType.AUTO const FORMULA_TYPE = FieldType.FORMULA @@ -191,6 +193,19 @@ fieldId: makeFieldId(t.type, t.subtype), ...t, })) + $: defaultValueBindings = [ + { + type: "context", + runtimeBinding: `${makePropSafe("now")}`, + readableBinding: `Date`, + category: "Date", + icon: "Date", + display: { + name: "Server date", + }, + }, + ...getUserBindings(), + ] const fieldDefinitions = Object.values(FIELDS).reduce( // Storing the fields by complex field id @@ -781,9 +796,8 @@ setRequired(false) } }} - bindings={getBindings({ table })} + bindings={defaultValueBindings} allowJS - context={rowGoldenSample} /> {/if} diff --git a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte index 2040f66706..d031e752cd 100644 --- a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte @@ -2,7 +2,12 @@ import { getContext } from "svelte" import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte" - const { datasource } = getContext("grid") + const { datasource, rows } = getContext("grid") + + const onUpdate = async () => { + await datasource.actions.refreshDefinition() + await rows.actions.refreshData() + } - + diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 68958da8e7..66c755b881 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -3,14 +3,9 @@ import { ViewV2, SearchRowResponse, SearchViewRowRequest, - SearchFilterKey, - LogicalOperator, } from "@budibase/types" -import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" -import { db, context, features } from "@budibase/backend-core" -import { enrichSearchContext } from "./utils" -import { isExternalTableID } from "../../../integrations/utils" +import { context } from "@budibase/backend-core" export async function searchView( ctx: UserCtx @@ -27,58 +22,23 @@ export async function searchView( const { body } = ctx.request - // Enrich saved query with ephemeral query params. - // We prevent searching on any fields that are saved as part of the query, as - // that could let users find rows they should not be allowed to access. - let query = dataFilters.buildQuery(view.query || []) - if (body.query) { - // Delete extraneous search params that cannot be overridden - delete body.query.onEmptyFilter - - if ( - !isExternalTableID(view.tableId) && - !(await features.flags.isEnabled("SQS")) - ) { - // Extract existing fields - const existingFields = - view.query - ?.filter(filter => filter.field) - .map(filter => db.removeKeyNumbering(filter.field)) || [] - - // Carry over filters for unused fields - Object.keys(body.query).forEach(key => { - const operator = key as Exclude - Object.keys(body.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { - query[operator]![field] = body.query[operator]![field] - } - }) - }) - } else { - query = { - $and: { - conditions: [query, body.query], - }, - } - } - } - await context.ensureSnippetContext(true) - const enrichedQuery = await enrichSearchContext(query, { - user: sdk.users.getUserContextBindings(ctx.user), - }) - - const result = await sdk.rows.search({ - viewId: view.id, - tableId: view.tableId, - query: enrichedQuery, - ...getSortOptions(body, view), - limit: body.limit, - bookmark: body.bookmark, - paginate: body.paginate, - countRows: body.countRows, - }) + const result = await sdk.rows.search( + { + viewId: view.id, + tableId: view.tableId, + query: body.query, + ...getSortOptions(body, view), + limit: body.limit, + bookmark: body.bookmark, + paginate: body.paginate, + countRows: body.countRows, + }, + { + user: sdk.users.getUserContextBindings(ctx.user), + } + ) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 090514250d..1ec5ca792a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -408,7 +408,6 @@ describe.each([ }) }) - // We've decided not to try and support binding for in-memory search just now. !isInMemory && describe("bindings", () => { let globalUsers: any = [] @@ -528,6 +527,20 @@ describe.each([ ]) }) + !isLucene && + it("should return all rows matching the session user firstname when logical operator used", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { name: "{{ [user].firstName }}" } }], + }, + }).toContainExactly([ + { + name: config.getUser().firstName, + appointment: future.toISOString(), + }, + ]) + }) + it("should parse the date binding and return all rows after the resolved value", async () => { await tk.withFreeze(serverTime, async () => { await expectQuery({ diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index aab846e704..09273abdce 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1738,6 +1738,40 @@ describe.each([ }) }) + it("views filters are respected even if the column is hidden", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + query: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: false }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: two._id }), + ]) + }) + it("views without data can be returned", async () => { const response = await config.api.viewV2.search(view.id) expect(response.rows).toHaveLength(0) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 7ef776a989..c4f4a1eb2c 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -16,11 +16,11 @@ export const removeInvalidFilters = ( validFields = validFields.map(f => f.toLowerCase()) for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) { - const filter = result[filterKey] - if (!filter || typeof filter !== "object") { - continue - } if (isLogicalSearchOperator(filterKey)) { + const filter = result[filterKey] + if (!filter || typeof filter !== "object") { + continue + } const resultingConditions: SearchFilters[] = [] for (const condition of filter.conditions) { const resultingCondition = removeInvalidFilters(condition, validFields) @@ -36,6 +36,11 @@ export const removeInvalidFilters = ( continue } + const filter = result[filterKey] + if (!filter || typeof filter !== "object") { + continue + } + for (const columnKey of Object.keys(filter)) { const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map( c => c.toLowerCase() diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 809bd73d1f..8de5818805 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,7 +1,10 @@ import { EmptyFilterOption, + LogicalOperator, Row, RowSearchParams, + SearchFilterKey, + SearchFilters, SearchResponse, SortOrder, Table, @@ -14,9 +17,10 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" -import { features } from "@budibase/backend-core" +import { db, features } from "@budibase/backend-core" import tracer from "dd-trace" import { getQueryableFields, removeInvalidFilters } from "./queryUtils" +import { enrichSearchContext } from "../../../api/controllers/row/utils" export { isValidFilter } from "../../../integrations/utils" @@ -34,7 +38,8 @@ function pickApi(tableId: any) { } export async function search( - options: RowSearchParams + options: RowSearchParams, + context?: Record ): Promise> { return await tracer.trace("search", async span => { span?.addTags({ @@ -51,7 +56,73 @@ export async function search( countRows: options.countRows, }) - options.query = dataFilters.cleanupQuery(options.query || {}) + let source: Table | ViewV2 + let table: Table + if (options.viewId) { + source = await sdk.views.get(options.viewId) + table = await sdk.views.getTable(source) + options = searchInputMapping(table, options) + } else if (options.tableId) { + source = await sdk.tables.getTable(options.tableId) + table = source + } else { + throw new Error(`Must supply either a view ID or a table ID`) + } + + const isExternalTable = isExternalTableID(table._id!) + + if (options.query) { + const visibleFields = ( + options.fields || Object.keys(table.schema) + ).filter(field => table.schema[field].visible !== false) + + const queryableFields = await getQueryableFields(table, visibleFields) + options.query = removeInvalidFilters(options.query, queryableFields) + } else { + options.query = {} + } + + if (options.viewId) { + const view = await sdk.views.get(options.viewId) + // Enrich saved query with ephemeral query params. + // We prevent searching on any fields that are saved as part of the query, as + // that could let users find rows they should not be allowed to access. + let viewQuery = dataFilters.buildQuery(view.query || []) + + if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) { + // Lucene does not accept conditional filters, so we need to keep the old logic + const query: SearchFilters = viewQuery + + // Extract existing fields + const existingFields = + view.query + ?.filter(filter => filter.field) + .map(filter => db.removeKeyNumbering(filter.field)) || [] + + // Carry over filters for unused fields + Object.keys(options.query || {}).forEach(key => { + const operator = key as Exclude + Object.keys(options.query[operator] || {}).forEach(field => { + if (!existingFields.includes(db.removeKeyNumbering(field))) { + query[operator]![field] = options.query[operator]![field] + } + }) + }) + options.query = query + } else { + options.query = { + $and: { + conditions: [viewQuery, options.query], + }, + } + } + } + + if (context) { + options.query = await enrichSearchContext(options.query, context) + } + + options.query = dataFilters.cleanupQuery(options.query) options.query = dataFilters.fixupFilterArrays(options.query) span.addTags({ @@ -72,30 +143,8 @@ export async function search( options.sortOrder = options.sortOrder.toLowerCase() as SortOrder } - let source: Table | ViewV2 - let table: Table - if (options.viewId) { - source = await sdk.views.get(options.viewId) - table = await sdk.views.getTable(source) - options = searchInputMapping(table, options) - } else if (options.tableId) { - source = await sdk.tables.getTable(options.tableId) - table = source - options = searchInputMapping(table, options) - } else { - throw new Error(`Must supply either a view ID or a table ID`) - } + options = searchInputMapping(table, options) - if (options.query) { - const visibleFields = ( - options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible !== false) - - const queryableFields = await getQueryableFields(table, visibleFields) - options.query = removeInvalidFilters(options.query, queryableFields) - } - - const isExternalTable = isExternalTableID(table._id!) let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 6548f963b8..1dba420a28 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -11,7 +11,7 @@ import { RowSearchParams, } from "@budibase/types" import { db as dbCore, context } from "@budibase/backend-core" -import { utils } from "@budibase/shared-core" +import { utils, dataFilters } from "@budibase/shared-core" export async function paginatedSearch( query: SearchFilters, @@ -31,13 +31,13 @@ export async function fullSearch( function findColumnInQueries( column: string, - options: RowSearchParams, + filters: SearchFilters, callback: (filter: any) => any ) { - if (!options.query) { + if (!filters) { return } - for (let filterBlock of Object.values(options.query)) { + for (let filterBlock of Object.values(filters)) { if (typeof filterBlock !== "object") { continue } @@ -49,8 +49,8 @@ function findColumnInQueries( } } -function userColumnMapping(column: string, options: RowSearchParams) { - findColumnInQueries(column, options, (filterValue: any): any => { +function userColumnMapping(column: string, filters: SearchFilters) { + findColumnInQueries(column, filters, (filterValue: any): any => { const isArray = Array.isArray(filterValue), isString = typeof filterValue === "string" if (!isString && !isArray) { @@ -83,26 +83,31 @@ function userColumnMapping(column: string, options: RowSearchParams) { // maps through the search parameters to check if any of the inputs are invalid // based on the table schema, converts them to something that is valid. export function searchInputMapping(table: Table, options: RowSearchParams) { - for (let [key, column] of Object.entries(table.schema || {})) { - switch (column.type) { - case FieldType.BB_REFERENCE_SINGLE: { - const subtype = column.subtype - switch (subtype) { - case BBReferenceFieldSubType.USER: - userColumnMapping(key, options) - break + // need an internal function to loop over filters, because this takes the full options + function checkFilters(filters: SearchFilters) { + for (let [key, column] of Object.entries(table.schema || {})) { + switch (column.type) { + case FieldType.BB_REFERENCE_SINGLE: { + const subtype = column.subtype + switch (subtype) { + case BBReferenceFieldSubType.USER: + userColumnMapping(key, filters) + break - default: - utils.unreachable(subtype) + default: + utils.unreachable(subtype) + } + break + } + case FieldType.BB_REFERENCE: { + userColumnMapping(key, filters) + break } - break - } - case FieldType.BB_REFERENCE: { - userColumnMapping(key, options) - break } } + return dataFilters.recurseLogicalOperators(filters, checkFilters) } + options.query = checkFilters(options.query) return options } diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index 0194e900fe..74389a1444 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -130,6 +130,26 @@ export function getUserContextBindings(user: ContextUser) { return {} } // Current user context for bindable search - const { _id, _rev, firstName, lastName, email, status, roleId } = user - return { _id, _rev, firstName, lastName, email, status, roleId } + const { + _id, + _rev, + firstName, + lastName, + email, + status, + roleId, + globalId, + userId, + } = user + return { + _id, + _rev, + firstName, + lastName, + email, + status, + roleId, + globalId, + userId, + } } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 45e9a7c6d0..ef0500b01a 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -124,7 +124,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters[logical]) { + if (filters?.[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 1d5b36031c..647a9e7d00 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -68,6 +68,8 @@ type RangeFilter = Record< [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never } +type LogicalFilter = { conditions: SearchFilters[] } + export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter export interface SearchFilters { @@ -92,12 +94,8 @@ export interface SearchFilters { // specific document type (such as just rows) documentType?: DocumentType - [LogicalOperator.AND]?: { - conditions: SearchFilters[] - } - [LogicalOperator.OR]?: { - conditions: SearchFilters[] - } + [LogicalOperator.AND]?: LogicalFilter + [LogicalOperator.OR]?: LogicalFilter } export type SearchFilterKey = keyof Omit<