Merge branch 'master' into budi-8664-cron-helper-ai-feature

.tool-versions

@@ -1,3 +1,3 @@
 nodejs 20.10.0
 python 3.10.0
-yarn 1.22.19
+yarn 1.22.22

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.8",
+  "version": "2.32.10",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -21,6 +21,7 @@
     PROTECTED_EXTERNAL_COLUMNS,
     canHaveDefaultColumn,
   } from "@budibase/shared-core"
+  import { makePropSafe } from "@budibase/string-templates"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/builder"
@@ -46,6 +47,7 @@
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
   import OptionsEditor from "./OptionsEditor.svelte"
   import { isEnabled } from "helpers/featureFlags"
+  import { getUserBindings } from "dataBinding"
 
   const AUTO_TYPE = FieldType.AUTO
   const FORMULA_TYPE = FieldType.FORMULA
@@ -191,6 +193,19 @@
     fieldId: makeFieldId(t.type, t.subtype),
     ...t,
   }))
+  $: defaultValueBindings = [
+    {
+      type: "context",
+      runtimeBinding: `${makePropSafe("now")}`,
+      readableBinding: `Date`,
+      category: "Date",
+      icon: "Date",
+      display: {
+        name: "Server date",
+      },
+    },
+    ...getUserBindings(),
+  ]
 
   const fieldDefinitions = Object.values(FIELDS).reduce(
     // Storing the fields by complex field id
@@ -781,9 +796,8 @@
             setRequired(false)
           }
         }}
-        bindings={getBindings({ table })}
+        bindings={defaultValueBindings}
         allowJS
-        context={rowGoldenSample}
       />
     </div>
   {/if}

packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte

@@ -2,7 +2,12 @@
   import { getContext } from "svelte"
   import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte"
 
-  const { datasource } = getContext("grid")
+  const { datasource, rows } = getContext("grid")
+
+  const onUpdate = async () => {
+    await datasource.actions.refreshDefinition()
+    await rows.actions.refreshData()
+  }
 </script>
 
-<CreateEditColumn on:updatecolumns={datasource.actions.refreshDefinition} />
+<CreateEditColumn on:updatecolumns={onUpdate} />

packages/server/src/api/controllers/row/views.ts

@@ -3,14 +3,9 @@ import {
   ViewV2,
   SearchRowResponse,
   SearchViewRowRequest,
-  SearchFilterKey,
-  LogicalOperator,
 } from "@budibase/types"
-import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
-import { db, context, features } from "@budibase/backend-core"
-import { enrichSearchContext } from "./utils"
-import { isExternalTableID } from "../../../integrations/utils"
+import { context } from "@budibase/backend-core"
 
 export async function searchView(
   ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
@@ -27,58 +22,23 @@ export async function searchView(
 
   const { body } = ctx.request
 
-  // Enrich saved query with ephemeral query params.
-  // We prevent searching on any fields that are saved as part of the query, as
-  // that could let users find rows they should not be allowed to access.
-  let query = dataFilters.buildQuery(view.query || [])
-  if (body.query) {
-    // Delete extraneous search params that cannot be overridden
-    delete body.query.onEmptyFilter
-
-    if (
-      !isExternalTableID(view.tableId) &&
-      !(await features.flags.isEnabled("SQS"))
-    ) {
-      // Extract existing fields
-      const existingFields =
-        view.query
-          ?.filter(filter => filter.field)
-          .map(filter => db.removeKeyNumbering(filter.field)) || []
-
-      // Carry over filters for unused fields
-      Object.keys(body.query).forEach(key => {
-        const operator = key as Exclude<SearchFilterKey, LogicalOperator>
-        Object.keys(body.query[operator] || {}).forEach(field => {
-          if (!existingFields.includes(db.removeKeyNumbering(field))) {
-            query[operator]![field] = body.query[operator]![field]
-          }
-        })
-      })
-    } else {
-      query = {
-        $and: {
-          conditions: [query, body.query],
-        },
-      }
-    }
-  }
-
   await context.ensureSnippetContext(true)
 
-  const enrichedQuery = await enrichSearchContext(query, {
-    user: sdk.users.getUserContextBindings(ctx.user),
-  })
-
-  const result = await sdk.rows.search({
+  const result = await sdk.rows.search(
+    {
       viewId: view.id,
       tableId: view.tableId,
-    query: enrichedQuery,
+      query: body.query,
       ...getSortOptions(body, view),
       limit: body.limit,
       bookmark: body.bookmark,
       paginate: body.paginate,
       countRows: body.countRows,
-  })
+    },
+    {
+      user: sdk.users.getUserContextBindings(ctx.user),
+    }
+  )
 
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result

packages/server/src/api/routes/tests/search.spec.ts

@@ -408,7 +408,6 @@ describe.each([
     })
   })
 
-  // We've decided not to try and support binding for in-memory search just now.
   !isInMemory &&
     describe("bindings", () => {
       let globalUsers: any = []
@@ -528,6 +527,20 @@ describe.each([
         ])
       })
 
+      !isLucene &&
+        it("should return all rows matching the session user firstname when logical operator used", async () => {
+          await expectQuery({
+            $and: {
+              conditions: [{ equal: { name: "{{ [user].firstName }}" } }],
+            },
+          }).toContainExactly([
+            {
+              name: config.getUser().firstName,
+              appointment: future.toISOString(),
+            },
+          ])
+        })
+
       it("should parse the date binding and return all rows after the resolved value", async () => {
         await tk.withFreeze(serverTime, async () => {
           await expectQuery({

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -1738,6 +1738,40 @@ describe.each([
         })
       })
 
+      it("views filters are respected even if the column is hidden", async () => {
+        await config.api.row.save(table._id!, {
+          one: "foo",
+          two: "bar",
+        })
+        const two = await config.api.row.save(table._id!, {
+          one: "foo2",
+          two: "bar2",
+        })
+
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          query: [
+            {
+              operator: BasicOperator.EQUAL,
+              field: "two",
+              value: "bar2",
+            },
+          ],
+          schema: {
+            id: { visible: true },
+            one: { visible: false },
+            two: { visible: false },
+          },
+        })
+
+        const response = await config.api.viewV2.search(view.id)
+        expect(response.rows).toHaveLength(1)
+        expect(response.rows).toEqual([
+          expect.objectContaining({ _id: two._id }),
+        ])
+      })
+
       it("views without data can be returned", async () => {
         const response = await config.api.viewV2.search(view.id)
         expect(response.rows).toHaveLength(0)

packages/server/src/sdk/app/rows/queryUtils.ts

@@ -16,11 +16,11 @@ export const removeInvalidFilters = (
 
   validFields = validFields.map(f => f.toLowerCase())
   for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) {
+    if (isLogicalSearchOperator(filterKey)) {
       const filter = result[filterKey]
       if (!filter || typeof filter !== "object") {
         continue
       }
-    if (isLogicalSearchOperator(filterKey)) {
       const resultingConditions: SearchFilters[] = []
       for (const condition of filter.conditions) {
         const resultingCondition = removeInvalidFilters(condition, validFields)
@@ -36,6 +36,11 @@ export const removeInvalidFilters = (
       continue
     }
 
+    const filter = result[filterKey]
+    if (!filter || typeof filter !== "object") {
+      continue
+    }
+
     for (const columnKey of Object.keys(filter)) {
       const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map(
         c => c.toLowerCase()

packages/server/src/sdk/app/rows/search.ts

@@ -1,7 +1,10 @@
 import {
   EmptyFilterOption,
+  LogicalOperator,
   Row,
   RowSearchParams,
+  SearchFilterKey,
+  SearchFilters,
   SearchResponse,
   SortOrder,
   Table,
@@ -14,9 +17,10 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../index"
 import { searchInputMapping } from "./search/utils"
-import { features } from "@budibase/backend-core"
+import { db, features } from "@budibase/backend-core"
 import tracer from "dd-trace"
 import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
+import { enrichSearchContext } from "../../../api/controllers/row/utils"
 
 export { isValidFilter } from "../../../integrations/utils"
 
@@ -34,7 +38,8 @@ function pickApi(tableId: any) {
 }
 
 export async function search(
-  options: RowSearchParams
+  options: RowSearchParams,
+  context?: Record<string, any>
 ): Promise<SearchResponse<Row>> {
   return await tracer.trace("search", async span => {
     span?.addTags({
@@ -51,7 +56,73 @@ export async function search(
       countRows: options.countRows,
     })
 
-    options.query = dataFilters.cleanupQuery(options.query || {})
+    let source: Table | ViewV2
+    let table: Table
+    if (options.viewId) {
+      source = await sdk.views.get(options.viewId)
+      table = await sdk.views.getTable(source)
+      options = searchInputMapping(table, options)
+    } else if (options.tableId) {
+      source = await sdk.tables.getTable(options.tableId)
+      table = source
+    } else {
+      throw new Error(`Must supply either a view ID or a table ID`)
+    }
+
+    const isExternalTable = isExternalTableID(table._id!)
+
+    if (options.query) {
+      const visibleFields = (
+        options.fields || Object.keys(table.schema)
+      ).filter(field => table.schema[field].visible !== false)
+
+      const queryableFields = await getQueryableFields(table, visibleFields)
+      options.query = removeInvalidFilters(options.query, queryableFields)
+    } else {
+      options.query = {}
+    }
+
+    if (options.viewId) {
+      const view = await sdk.views.get(options.viewId)
+      // Enrich saved query with ephemeral query params.
+      // We prevent searching on any fields that are saved as part of the query, as
+      // that could let users find rows they should not be allowed to access.
+      let viewQuery = dataFilters.buildQuery(view.query || [])
+
+      if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
+        // Lucene does not accept conditional filters, so we need to keep the old logic
+        const query: SearchFilters = viewQuery
+
+        // Extract existing fields
+        const existingFields =
+          view.query
+            ?.filter(filter => filter.field)
+            .map(filter => db.removeKeyNumbering(filter.field)) || []
+
+        // Carry over filters for unused fields
+        Object.keys(options.query || {}).forEach(key => {
+          const operator = key as Exclude<SearchFilterKey, LogicalOperator>
+          Object.keys(options.query[operator] || {}).forEach(field => {
+            if (!existingFields.includes(db.removeKeyNumbering(field))) {
+              query[operator]![field] = options.query[operator]![field]
+            }
+          })
+        })
+        options.query = query
+      } else {
+        options.query = {
+          $and: {
+            conditions: [viewQuery, options.query],
+          },
+        }
+      }
+    }
+
+    if (context) {
+      options.query = await enrichSearchContext(options.query, context)
+    }
+
+    options.query = dataFilters.cleanupQuery(options.query)
     options.query = dataFilters.fixupFilterArrays(options.query)
 
     span.addTags({
@@ -72,30 +143,8 @@ export async function search(
       options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
     }
 
-    let source: Table | ViewV2
-    let table: Table
-    if (options.viewId) {
-      source = await sdk.views.get(options.viewId)
-      table = await sdk.views.getTable(source)
     options = searchInputMapping(table, options)
-    } else if (options.tableId) {
-      source = await sdk.tables.getTable(options.tableId)
-      table = source
-      options = searchInputMapping(table, options)
-    } else {
-      throw new Error(`Must supply either a view ID or a table ID`)
-    }
 
-    if (options.query) {
-      const visibleFields = (
-        options.fields || Object.keys(table.schema)
-      ).filter(field => table.schema[field].visible !== false)
-
-      const queryableFields = await getQueryableFields(table, visibleFields)
-      options.query = removeInvalidFilters(options.query, queryableFields)
-    }
-
-    const isExternalTable = isExternalTableID(table._id!)
     let result: SearchResponse<Row>
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })

packages/server/src/sdk/app/rows/search/utils.ts

@@ -11,7 +11,7 @@ import {
   RowSearchParams,
 } from "@budibase/types"
 import { db as dbCore, context } from "@budibase/backend-core"
-import { utils } from "@budibase/shared-core"
+import { utils, dataFilters } from "@budibase/shared-core"
 
 export async function paginatedSearch(
   query: SearchFilters,
@@ -31,13 +31,13 @@ export async function fullSearch(
 
 function findColumnInQueries(
   column: string,
-  options: RowSearchParams,
+  filters: SearchFilters,
   callback: (filter: any) => any
 ) {
-  if (!options.query) {
+  if (!filters) {
     return
   }
-  for (let filterBlock of Object.values(options.query)) {
+  for (let filterBlock of Object.values(filters)) {
     if (typeof filterBlock !== "object") {
       continue
     }
@@ -49,8 +49,8 @@ function findColumnInQueries(
   }
 }
 
-function userColumnMapping(column: string, options: RowSearchParams) {
-  findColumnInQueries(column, options, (filterValue: any): any => {
+function userColumnMapping(column: string, filters: SearchFilters) {
+  findColumnInQueries(column, filters, (filterValue: any): any => {
     const isArray = Array.isArray(filterValue),
       isString = typeof filterValue === "string"
     if (!isString && !isArray) {
@@ -83,13 +83,15 @@ function userColumnMapping(column: string, options: RowSearchParams) {
 // maps through the search parameters to check if any of the inputs are invalid
 // based on the table schema, converts them to something that is valid.
 export function searchInputMapping(table: Table, options: RowSearchParams) {
+  // need an internal function to loop over filters, because this takes the full options
+  function checkFilters(filters: SearchFilters) {
     for (let [key, column] of Object.entries(table.schema || {})) {
       switch (column.type) {
         case FieldType.BB_REFERENCE_SINGLE: {
           const subtype = column.subtype
           switch (subtype) {
             case BBReferenceFieldSubType.USER:
-            userColumnMapping(key, options)
+              userColumnMapping(key, filters)
               break
 
             default:
@@ -98,11 +100,14 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
           break
         }
         case FieldType.BB_REFERENCE: {
-        userColumnMapping(key, options)
+          userColumnMapping(key, filters)
           break
         }
       }
     }
+    return dataFilters.recurseLogicalOperators(filters, checkFilters)
+  }
+  options.query = checkFilters(options.query)
   return options
 }
 

packages/server/src/sdk/users/utils.ts

@@ -130,6 +130,26 @@ export function getUserContextBindings(user: ContextUser) {
     return {}
   }
   // Current user context for bindable search
-  const { _id, _rev, firstName, lastName, email, status, roleId } = user
-  return { _id, _rev, firstName, lastName, email, status, roleId }
+  const {
+    _id,
+    _rev,
+    firstName,
+    lastName,
+    email,
+    status,
+    roleId,
+    globalId,
+    userId,
+  } = user
+  return {
+    _id,
+    _rev,
+    firstName,
+    lastName,
+    email,
+    status,
+    roleId,
+    globalId,
+    userId,
+  }
 }

packages/shared-core/src/filters.ts

@@ -124,7 +124,7 @@ export function recurseLogicalOperators(
   fn: (f: SearchFilters) => SearchFilters
 ) {
   for (const logical of LOGICAL_OPERATORS) {
-    if (filters[logical]) {
+    if (filters?.[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => fn(condition)
       )

packages/types/src/sdk/search.ts

@@ -68,6 +68,8 @@ type RangeFilter = Record<
   [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never
 }
 
+type LogicalFilter = { conditions: SearchFilters[] }
+
 export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter
 
 export interface SearchFilters {
@@ -92,12 +94,8 @@ export interface SearchFilters {
   // specific document type (such as just rows)
   documentType?: DocumentType
 
-  [LogicalOperator.AND]?: {
-    conditions: SearchFilters[]
-  }
-  [LogicalOperator.OR]?: {
-    conditions: SearchFilters[]
-  }
+  [LogicalOperator.AND]?: LogicalFilter
+  [LogicalOperator.OR]?: LogicalFilter
 }
 
 export type SearchFilterKey = keyof Omit<