Merge branch 'master' into test-oracle

.github/workflows/budibase_ci.yml

@@ -147,7 +147,7 @@ jobs:
           fi
 
   test-server:
-    runs-on: budi-tubby-tornado-quad-core-150gb
+    runs-on: budi-tubby-tornado-quad-core-300gb
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4

README.md

@@ -116,7 +116,7 @@ As with anything that we build in Budibase, our new public API is simple to use,
 You can learn more about the Budibase API at the following places:
 
 - [General documentation](https://docs.budibase.com/docs/public-api): Learn how to get your API key, how to use spec, and how to use Postman
-- [Interactive API documentation](https://docs.budibase.com/reference/post_applications) : Learn how to interact with the API
+- [Interactive API documentation](https://docs.budibase.com/reference/appcreate) : Learn how to interact with the API
 
 <br /><br />
 

i18n/README.es.md

@@ -144,7 +144,7 @@ del sistema. Budibase API ofrece:
 
 Puedes aprender mas acerca de Budibase API en los siguientes documentos:
 - [Documentacion general](https://docs.budibase.com/docs/public-api) : Como optener tu clave para la API, usar Insomnia y Postman
-- [API Interactiva](https://docs.budibase.com/reference/post_applications) : Aprende como trabajar con la API
+- [API Interactiva](https://docs.budibase.com/reference/appcreate) : Aprende como trabajar con la API
 
 #### Guias
 

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.29.28",
+  "version": "2.29.30",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/features/index.ts

@@ -1,79 +1,108 @@
 import env from "../environment"
 import * as context from "../context"
+import { cloneDeep } from "lodash"
 
-export * from "./installation"
+class Flag<T> {
+  static withDefault<T>(value: T) {
+    return new Flag(value)
+  }
+
+  private constructor(public defaultValue: T) {}
+}
+
+// This is the primary source of truth for feature flags. If you want to add a
+// new flag, add it here and use the `fetch` and `get` functions to access it.
+// All of the machinery in this file is to make sure that flags have their
+// default values set correctly and their types flow through the system.
+const FLAGS = {
+  LICENSING: Flag.withDefault(false),
+  GOOGLE_SHEETS: Flag.withDefault(false),
+  USER_GROUPS: Flag.withDefault(false),
+  ONBOARDING_TOUR: Flag.withDefault(false),
+}
+
+const DEFAULTS = Object.keys(FLAGS).reduce((acc, key) => {
+  const typedKey = key as keyof typeof FLAGS
+  // @ts-ignore
+  acc[typedKey] = FLAGS[typedKey].defaultValue
+  return acc
+}, {} as Flags)
+
+type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
+export type Flags = {
+  [K in keyof typeof FLAGS]: UnwrapFlag<(typeof FLAGS)[K]>
+}
+
+// Exported for use in tests, should not be used outside of this file.
+export function defaultFlags(): Flags {
+  return cloneDeep(DEFAULTS)
+}
+
+function isFlagName(name: string): name is keyof Flags {
+  return FLAGS[name as keyof typeof FLAGS] !== undefined
+}
 
 /**
- * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
+ * Reads the TENANT_FEATURE_FLAGS environment variable and returns a Flags object
- * The env var is formatted as:
+ * populated with the flags for the current tenant, filling in the default values
- *  tenant1:feature1:feature2,tenant2:feature1
+ * if the flag is not set.
+ *
+ * Check the tests for examples of how TENANT_FEATURE_FLAGS should be formatted.
+ *
+ * In future we plan to add more ways of setting feature flags, e.g. PostHog, and
+ * they will be accessed through this function as well.
  */
-export function buildFeatureFlags() {
+export async function fetch(): Promise<Flags> {
-  if (!env.TENANT_FEATURE_FLAGS) {
+  const currentTenantId = context.getTenantId()
-    return
+  const flags = defaultFlags()
+
+  const split = (env.TENANT_FEATURE_FLAGS || "")
+    .split(",")
+    .map(x => x.split(":"))
+  for (const [tenantId, ...features] of split) {
+    if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
+      continue
     }
 
-  const tenantFeatureFlags: Record<string, string[]> = {}
+    for (let feature of features) {
-
+      let value = true
-  env.TENANT_FEATURE_FLAGS.split(",").forEach(tenantToFeatures => {
+      if (feature.startsWith("!")) {
-    const [tenantId, ...features] = tenantToFeatures.split(":")
+        feature = feature.slice(1)
-
+        value = false
-    features.forEach(feature => {
-      if (!tenantFeatureFlags[tenantId]) {
-        tenantFeatureFlags[tenantId] = []
-      }
-      tenantFeatureFlags[tenantId].push(feature)
-    })
-  })
-
-  return tenantFeatureFlags
-}
-
-export function isEnabled(featureFlag: string) {
-  const tenantId = context.getTenantId()
-  const flags = getTenantFeatureFlags(tenantId)
-  return flags.includes(featureFlag)
-}
-
-export function getTenantFeatureFlags(tenantId: string) {
-  let flags: string[] = []
-  const envFlags = buildFeatureFlags()
-  if (envFlags) {
-    const globalFlags = envFlags["*"]
-    const tenantFlags = envFlags[tenantId] || []
-
-    // Explicitly exclude tenants from global features if required.
-    // Prefix the tenant flag with '!'
-    const tenantOverrides = tenantFlags.reduce(
-      (acc: string[], flag: string) => {
-        if (flag.startsWith("!")) {
-          let stripped = flag.substring(1)
-          acc.push(stripped)
-        }
-        return acc
-      },
-      []
-    )
-
-    if (globalFlags) {
-      flags.push(...globalFlags)
-    }
-    if (tenantFlags.length) {
-      flags.push(...tenantFlags)
       }
 
-    // Purge any tenant specific overrides
+      if (!isFlagName(feature)) {
-    flags = flags.filter(flag => {
+        throw new Error(`Feature: ${feature} is not an allowed option`)
-      return tenantOverrides.indexOf(flag) == -1 && !flag.startsWith("!")
+      }
-    })
+
+      if (typeof flags[feature] !== "boolean") {
+        throw new Error(`Feature: ${feature} is not a boolean`)
+      }
+
+      // @ts-ignore
+      flags[feature] = value
+    }
   }
 
   return flags
 }
 
-export enum TenantFeatureFlag {
+// Gets a single feature flag value. This is a convenience function for
-  LICENSING = "LICENSING",
+// `fetch().then(flags => flags[name])`.
-  GOOGLE_SHEETS = "GOOGLE_SHEETS",
+export async function get<K extends keyof Flags>(name: K): Promise<Flags[K]> {
-  USER_GROUPS = "USER_GROUPS",
+  const flags = await fetch()
-  ONBOARDING_TOUR = "ONBOARDING_TOUR",
+  return flags[name]
+}
+
+type BooleanFlags = {
+  [K in keyof typeof FLAGS]: (typeof FLAGS)[K] extends Flag<boolean> ? K : never
+}[keyof typeof FLAGS]
+
+// Convenience function for boolean flag values. This makes callsites more
+// readable for boolean flags.
+export async function isEnabled<K extends BooleanFlags>(
+  name: K
+): Promise<boolean> {
+  const flags = await fetch()
+  return flags[name]
 }

packages/backend-core/src/features/installation.ts

@@ -1,17 +0,0 @@
-export function processFeatureEnvVar<T>(
-  fullList: string[],
-  featureList?: string
-) {
-  let list
-  if (!featureList) {
-    list = fullList
-  } else {
-    list = featureList.split(",")
-  }
-  for (let feature of list) {
-    if (!fullList.includes(feature)) {
-      throw new Error(`Feature: ${feature} is not an allowed option`)
-    }
-  }
-  return list as unknown as T[]
-}

packages/backend-core/src/features/tests/featureFlags.spec.ts

@@ -1,85 +0,0 @@
-import {
-  TenantFeatureFlag,
-  buildFeatureFlags,
-  getTenantFeatureFlags,
-} from "../"
-import env from "../../environment"
-
-const { ONBOARDING_TOUR, LICENSING, USER_GROUPS } = TenantFeatureFlag
-
-describe("featureFlags", () => {
-  beforeEach(() => {
-    env._set("TENANT_FEATURE_FLAGS", "")
-  })
-
-  it("Should return no flags when the TENANT_FEATURE_FLAG is empty", async () => {
-    let features = buildFeatureFlags()
-    expect(features).toBeUndefined()
-  })
-
-  it("Should generate a map of global and named tenant feature flags from the env value", async () => {
-    env._set(
-      "TENANT_FEATURE_FLAGS",
-      `*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR},tenant2:${USER_GROUPS},tenant1:${LICENSING}`
-    )
-
-    const parsedFlags: Record<string, string[]> = {
-      "*": [ONBOARDING_TOUR],
-      tenant1: [`!${ONBOARDING_TOUR}`, LICENSING],
-      tenant2: [USER_GROUPS],
-    }
-
-    let features = buildFeatureFlags()
-
-    expect(features).toBeDefined()
-    expect(features).toEqual(parsedFlags)
-  })
-
-  it("Should add feature flag flag only to explicitly configured tenant", async () => {
-    env._set(
-      "TENANT_FEATURE_FLAGS",
-      `*:${LICENSING},*:${USER_GROUPS},tenant1:${ONBOARDING_TOUR}`
-    )
-
-    let tenant1Flags = getTenantFeatureFlags("tenant1")
-    let tenant2Flags = getTenantFeatureFlags("tenant2")
-
-    expect(tenant1Flags).toBeDefined()
-    expect(tenant1Flags).toEqual([LICENSING, USER_GROUPS, ONBOARDING_TOUR])
-
-    expect(tenant2Flags).toBeDefined()
-    expect(tenant2Flags).toEqual([LICENSING, USER_GROUPS])
-  })
-})
-
-it("Should exclude tenant1 from global feature flag", async () => {
-  env._set(
-    "TENANT_FEATURE_FLAGS",
-    `*:${LICENSING},*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR}`
-  )
-
-  let tenant1Flags = getTenantFeatureFlags("tenant1")
-  let tenant2Flags = getTenantFeatureFlags("tenant2")
-
-  expect(tenant1Flags).toBeDefined()
-  expect(tenant1Flags).toEqual([LICENSING])
-
-  expect(tenant2Flags).toBeDefined()
-  expect(tenant2Flags).toEqual([LICENSING, ONBOARDING_TOUR])
-})
-
-it("Should explicitly add flags to configured tenants only", async () => {
-  env._set(
-    "TENANT_FEATURE_FLAGS",
-    `tenant1:${ONBOARDING_TOUR},tenant1:${LICENSING},tenant2:${LICENSING}`
-  )
-
-  let tenant1Flags = getTenantFeatureFlags("tenant1")
-  let tenant2Flags = getTenantFeatureFlags("tenant2")
-
-  expect(tenant1Flags).toBeDefined()
-  expect(tenant1Flags).toEqual([ONBOARDING_TOUR, LICENSING])
-
-  expect(tenant2Flags).toBeDefined()
-  expect(tenant2Flags).toEqual([LICENSING])
-})

packages/backend-core/src/features/tests/features.spec.ts

@@ -0,0 +1,86 @@
+import { defaultFlags, fetch, get, Flags } from "../"
+import { context } from "../.."
+import env from "../../environment"
+
+async function withFlags<T>(flags: string, f: () => T): Promise<T> {
+  const oldFlags = env.TENANT_FEATURE_FLAGS
+  env._set("TENANT_FEATURE_FLAGS", flags)
+  try {
+    return await f()
+  } finally {
+    env._set("TENANT_FEATURE_FLAGS", oldFlags)
+  }
+}
+
+describe("feature flags", () => {
+  interface TestCase {
+    tenant: string
+    flags: string
+    expected: Partial<Flags>
+  }
+
+  it.each<TestCase>([
+    {
+      tenant: "tenant1",
+      flags: "tenant1:ONBOARDING_TOUR",
+      expected: { ONBOARDING_TOUR: true },
+    },
+    {
+      tenant: "tenant1",
+      flags: "tenant1:!ONBOARDING_TOUR",
+      expected: { ONBOARDING_TOUR: false },
+    },
+    {
+      tenant: "tenant1",
+      flags: "*:ONBOARDING_TOUR",
+      expected: { ONBOARDING_TOUR: true },
+    },
+    {
+      tenant: "tenant1",
+      flags: "tenant2:ONBOARDING_TOUR",
+      expected: { ONBOARDING_TOUR: false },
+    },
+    {
+      tenant: "tenant1",
+      flags: "",
+      expected: defaultFlags(),
+    },
+  ])(
+    'should find flags $expected for $tenant with string "$flags"',
+    ({ tenant, flags, expected }) =>
+      context.doInTenant(tenant, () =>
+        withFlags(flags, async () => {
+          const flags = await fetch()
+          expect(flags).toMatchObject(expected)
+
+          for (const [key, expectedValue] of Object.entries(expected)) {
+            const value = await get(key as keyof Flags)
+            expect(value).toBe(expectedValue)
+          }
+        })
+      )
+  )
+
+  interface FailedTestCase {
+    tenant: string
+    flags: string
+    expected: string | RegExp
+  }
+
+  it.each<FailedTestCase>([
+    {
+      tenant: "tenant1",
+      flags: "tenant1:ONBOARDING_TOUR,tenant1:FOO",
+      expected: "Feature: FOO is not an allowed option",
+    },
+  ])(
+    "should fail with message \"$expected\" for $tenant with string '$flags'",
+    async ({ tenant, flags, expected }) => {
+      context.doInTenant(tenant, () =>
+        withFlags(flags, async () => {
+          await expect(fetch()).rejects.toThrow(expected)
+        })
+      )
+    }
+  )
+})

packages/backend-core/src/index.ts

@@ -7,8 +7,7 @@ export * as roles from "./security/roles"
 export * as permissions from "./security/permissions"
 export * as accounts from "./accounts"
 export * as installation from "./installation"
-export * as featureFlags from "./features"
+export * as features from "./features"
-export * as features from "./features/installation"
 export * as sessions from "./security/sessions"
 export * as platform from "./platform"
 export * as auth from "./auth"

packages/backend-core/src/sql/sql.ts

@@ -463,6 +463,24 @@ class InternalBuilder {
       }
     }
 
+    if (filters.$and) {
+      const { $and } = filters
+      query = query.where(x => {
+        for (const condition of $and.conditions) {
+          x = this.addFilters(x, condition, opts)
+        }
+      })
+    }
+
+    if (filters.$or) {
+      const { $or } = filters
+      query = query.where(x => {
+        for (const condition of $or.conditions) {
+          x = this.addFilters(x, { ...condition, allOr: true }, opts)
+        }
+      })
+    }
+
     if (filters.oneOf) {
       const fnc = allOr ? "orWhereIn" : "whereIn"
       iterate(

packages/builder/src/helpers/featureFlags.js

@@ -5,9 +5,10 @@ export const TENANT_FEATURE_FLAGS = {
   LICENSING: "LICENSING",
   USER_GROUPS: "USER_GROUPS",
   ONBOARDING_TOUR: "ONBOARDING_TOUR",
+  GOOGLE_SHEETS: "GOOGLE_SHEETS",
 }
 
 export const isEnabled = featureFlag => {
   const user = get(auth).user
-  return !!user?.featureFlags?.includes(featureFlag)
+  return !!user?.flags?.[featureFlag]
 }

packages/pro

@@ -1 +1 @@
-Subproject commit 7dbe323aec724ae6336b13c06aaefa4a89837edf
+Subproject commit 62ef0e2d6e83522b6732fb3c61338de303f06ff0

packages/server/package.json

@@ -80,7 +80,7 @@
     "dotenv": "8.2.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
-    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.2",
+    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3",
     "ioredis": "5.3.2",
     "isolated-vm": "^4.7.2",
     "jimp": "0.22.12",

packages/server/src/api/controllers/row/views.ts

@@ -6,11 +6,13 @@ import {
   RequiredKeys,
   RowSearchParams,
   SearchFilterKey,
+  LogicalOperator,
 } from "@budibase/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
 import { db, context } from "@budibase/backend-core"
 import { enrichSearchContext } from "./utils"
+import { isExternalTableID } from "../../../integrations/utils"
 
 export async function searchView(
   ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
@@ -35,25 +37,33 @@ export async function searchView(
   // that could let users find rows they should not be allowed to access.
   let query = dataFilters.buildQuery(view.query || [])
   if (body.query) {
+    // Delete extraneous search params that cannot be overridden
+    delete body.query.allOr
+    delete body.query.onEmptyFilter
+
+    if (!isExternalTableID(view.tableId) && !db.isSqsEnabledForTenant()) {
       // Extract existing fields
       const existingFields =
         view.query
           ?.filter(filter => filter.field)
           .map(filter => db.removeKeyNumbering(filter.field)) || []
 
-    // Delete extraneous search params that cannot be overridden
-    delete body.query.allOr
-    delete body.query.onEmptyFilter
-
       // Carry over filters for unused fields
       Object.keys(body.query).forEach(key => {
-      const operator = key as SearchFilterKey
+        const operator = key as Exclude<SearchFilterKey, LogicalOperator>
         Object.keys(body.query[operator] || {}).forEach(field => {
           if (!existingFields.includes(db.removeKeyNumbering(field))) {
             query[operator]![field] = body.query[operator]![field]
           }
         })
       })
+    } else {
+      query = {
+        $and: {
+          conditions: [query, body.query],
+        },
+      }
+    }
   }
 
   await context.ensureSnippetContext(true)

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -17,9 +17,14 @@ import {
   SupportedSqlTypes,
   JsonFieldSubType,
 } from "@budibase/types"
-import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+import {
+  DatabaseName,
+  getDatasource,
+  knexClient,
+} from "../../../integrations/tests/utils"
 import { tableForDatasource } from "../../../tests/utilities/structures"
 import nock from "nock"
+import { Knex } from "knex"
 
 describe("/datasources", () => {
   const config = setup.getConfig()
@@ -164,11 +169,15 @@ describe("/datasources", () => {
     [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
     [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
     [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+    [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
   ])("%s", (_, dsProvider) => {
     let rawDatasource: Datasource
+    let client: Knex
+
     beforeEach(async () => {
       rawDatasource = await dsProvider
       datasource = await config.api.datasource.create(rawDatasource)
+      client = await knexClient(rawDatasource)
     })
 
     describe("get", () => {
@@ -285,9 +294,6 @@ describe("/datasources", () => {
           [FieldType.STRING]: {
             name: stringName,
             type: FieldType.STRING,
-            constraints: {
-              presence: true,
-            },
           },
           [FieldType.LONGFORM]: {
             name: "longform",
@@ -381,10 +387,6 @@ describe("/datasources", () => {
                   ),
                   schema: Object.entries(table.schema).reduce<TableSchema>(
                     (acc, [fieldName, field]) => {
-                      // the constraint will be unset - as the DB doesn't recognise it as not null
-                      if (fieldName === stringName) {
-                        field.constraints = {}
-                      }
                       acc[fieldName] = expect.objectContaining({
                         ...field,
                       })
@@ -441,20 +443,49 @@ describe("/datasources", () => {
     })
 
     describe("info", () => {
-      it("should fetch information about postgres datasource", async () => {
+      it("should fetch information about a datasource with a single table", async () => {
-        const table = await config.api.table.save(
+        const existingTableNames = (
-          tableForDatasource(datasource, {
+          await config.api.datasource.info(datasource)
-            schema: {
+        ).tableNames
-              name: {
+
-                name: "name",
+        const tableName = generator.guid()
-                type: FieldType.STRING,
+        await client.schema.createTable(tableName, table => {
-              },
+          table.increments("id").primary()
-            },
+          table.string("name")
         })
-        )
 
         const info = await config.api.datasource.info(datasource)
-        expect(info.tableNames).toContain(table.name)
+        expect(info.tableNames).toEqual(
+          expect.arrayContaining([tableName, ...existingTableNames])
+        )
+        expect(info.tableNames).toHaveLength(existingTableNames.length + 1)
+      })
+
+      it("should fetch information about a datasource with multiple tables", async () => {
+        const existingTableNames = (
+          await config.api.datasource.info(datasource)
+        ).tableNames
+
+        const tableNames = [
+          generator.guid(),
+          generator.guid(),
+          generator.guid(),
+          generator.guid(),
+        ]
+        for (const tableName of tableNames) {
+          await client.schema.createTable(tableName, table => {
+            table.increments("id").primary()
+            table.string("name")
+          })
+        }
+
+        const info = await config.api.datasource.info(datasource)
+        expect(info.tableNames).toEqual(
+          expect.arrayContaining([...tableNames, ...existingTableNames])
+        )
+        expect(info.tableNames).toHaveLength(
+          existingTableNames.length + tableNames.length
+        )
       })
     })
   })

packages/server/src/api/routes/tests/search.spec.ts

@@ -2693,4 +2693,239 @@ describe.each([
         )
       })
     })
+
+  !isLucene &&
+    describe("$and", () => {
+      beforeAll(async () => {
+        table = await createTable({
+          age: { name: "age", type: FieldType.NUMBER },
+          name: { name: "name", type: FieldType.STRING },
+        })
+        await createRows([
+          { age: 1, name: "Jane" },
+          { age: 10, name: "Jack" },
+          { age: 7, name: "Hanna" },
+          { age: 8, name: "Jan" },
+        ])
+      })
+
+      it("successfully finds a row for one level condition", async () => {
+        await expectQuery({
+          $and: {
+            conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }],
+          },
+        }).toContainExactly([{ age: 10, name: "Jack" }])
+      })
+
+      it("successfully finds a row for one level with multiple conditions", async () => {
+        await expectQuery({
+          $and: {
+            conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }],
+          },
+        }).toContainExactly([{ age: 10, name: "Jack" }])
+      })
+
+      it("successfully finds multiple rows for one level with multiple conditions", async () => {
+        await expectQuery({
+          $and: {
+            conditions: [
+              { range: { age: { low: 1, high: 9 } } },
+              { string: { name: "Ja" } },
+            ],
+          },
+        }).toContainExactly([
+          { age: 1, name: "Jane" },
+          { age: 8, name: "Jan" },
+        ])
+      })
+
+      it("successfully finds rows for nested filters", async () => {
+        await expectQuery({
+          $and: {
+            conditions: [
+              {
+                $and: {
+                  conditions: [
+                    {
+                      range: { age: { low: 1, high: 10 } },
+                    },
+                    { string: { name: "Ja" } },
+                  ],
+                },
+                equal: { name: "Jane" },
+              },
+            ],
+          },
+        }).toContainExactly([{ age: 1, name: "Jane" }])
+      })
+
+      it("returns nothing when filtering out all data", async () => {
+        await expectQuery({
+          $and: {
+            conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
+          },
+        }).toFindNothing()
+      })
+
+      !isInMemory &&
+        it("validates conditions that are not objects", async () => {
+          await expect(
+            expectQuery({
+              $and: {
+                conditions: [{ equal: { age: 10 } }, "invalidCondition" as any],
+              },
+            }).toFindNothing()
+          ).rejects.toThrow(
+            'Invalid body - "query.$and.conditions[1]" must be of type object'
+          )
+        })
+
+      !isInMemory &&
+        it("validates $and without conditions", async () => {
+          await expect(
+            expectQuery({
+              $and: {
+                conditions: [
+                  { equal: { age: 10 } },
+                  {
+                    $and: {
+                      conditions: undefined as any,
+                    },
+                  },
+                ],
+              },
+            }).toFindNothing()
+          ).rejects.toThrow(
+            'Invalid body - "query.$and.conditions[1].$and.conditions" is required'
+          )
+        })
+    })
+
+  !isLucene &&
+    describe("$or", () => {
+      beforeAll(async () => {
+        table = await createTable({
+          age: { name: "age", type: FieldType.NUMBER },
+          name: { name: "name", type: FieldType.STRING },
+        })
+        await createRows([
+          { age: 1, name: "Jane" },
+          { age: 10, name: "Jack" },
+          { age: 7, name: "Hanna" },
+          { age: 8, name: "Jan" },
+        ])
+      })
+
+      it("successfully finds a row for one level condition", async () => {
+        await expectQuery({
+          $or: {
+            conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
+          },
+        }).toContainExactly([
+          { age: 10, name: "Jack" },
+          { age: 7, name: "Hanna" },
+        ])
+      })
+
+      it("successfully finds a row for one level with multiple conditions", async () => {
+        await expectQuery({
+          $or: {
+            conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
+          },
+        }).toContainExactly([
+          { age: 10, name: "Jack" },
+          { age: 7, name: "Hanna" },
+        ])
+      })
+
+      it("successfully finds multiple rows for one level with multiple conditions", async () => {
+        await expectQuery({
+          $or: {
+            conditions: [
+              { range: { age: { low: 1, high: 9 } } },
+              { string: { name: "Jan" } },
+            ],
+          },
+        }).toContainExactly([
+          { age: 1, name: "Jane" },
+          { age: 7, name: "Hanna" },
+          { age: 8, name: "Jan" },
+        ])
+      })
+
+      it("successfully finds rows for nested filters", async () => {
+        await expectQuery({
+          $or: {
+            conditions: [
+              {
+                $or: {
+                  conditions: [
+                    {
+                      range: { age: { low: 1, high: 7 } },
+                    },
+                    { string: { name: "Jan" } },
+                  ],
+                },
+                equal: { name: "Jane" },
+              },
+            ],
+          },
+        }).toContainExactly([
+          { age: 1, name: "Jane" },
+          { age: 7, name: "Hanna" },
+          { age: 8, name: "Jan" },
+        ])
+      })
+
+      it("returns nothing when filtering out all data", async () => {
+        await expectQuery({
+          $or: {
+            conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }],
+          },
+        }).toFindNothing()
+      })
+
+      it("can nest $and under $or filters", async () => {
+        await expectQuery({
+          $or: {
+            conditions: [
+              {
+                $and: {
+                  conditions: [
+                    {
+                      range: { age: { low: 1, high: 8 } },
+                    },
+                    { equal: { name: "Jan" } },
+                  ],
+                },
+                equal: { name: "Jane" },
+              },
+            ],
+          },
+        }).toContainExactly([
+          { age: 1, name: "Jane" },
+          { age: 8, name: "Jan" },
+        ])
+      })
+
+      it("can nest $or under $and filters", async () => {
+        await expectQuery({
+          $and: {
+            conditions: [
+              {
+                $or: {
+                  conditions: [
+                    {
+                      range: { age: { low: 1, high: 8 } },
+                    },
+                    { equal: { name: "Jan" } },
+                  ],
+                },
+                equal: { name: "Jane" },
+              },
+            ],
+          },
+        }).toContainExactly([{ age: 1, name: "Jane" }])
+      })
+    })
 })

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -1485,6 +1485,119 @@ describe.each([
           }
         )
       })
+
+      isLucene &&
+        it("in lucene, cannot override a view filter", async () => {
+          await config.api.row.save(table._id!, {
+            one: "foo",
+            two: "bar",
+          })
+          const two = await config.api.row.save(table._id!, {
+            one: "foo2",
+            two: "bar2",
+          })
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            query: [
+              {
+                operator: BasicOperator.EQUAL,
+                field: "two",
+                value: "bar2",
+              },
+            ],
+            schema: {
+              id: { visible: true },
+              one: { visible: false },
+              two: { visible: true },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id, {
+            query: {
+              equal: {
+                two: "bar",
+              },
+            },
+          })
+          expect(response.rows).toHaveLength(1)
+          expect(response.rows).toEqual([
+            expect.objectContaining({ _id: two._id }),
+          ])
+        })
+
+      !isLucene &&
+        it("can filter a view without a view filter", async () => {
+          const one = await config.api.row.save(table._id!, {
+            one: "foo",
+            two: "bar",
+          })
+          await config.api.row.save(table._id!, {
+            one: "foo2",
+            two: "bar2",
+          })
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              id: { visible: true },
+              one: { visible: false },
+              two: { visible: true },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id, {
+            query: {
+              equal: {
+                two: "bar",
+              },
+            },
+          })
+          expect(response.rows).toHaveLength(1)
+          expect(response.rows).toEqual([
+            expect.objectContaining({ _id: one._id }),
+          ])
+        })
+
+      !isLucene &&
+        it("cannot bypass a view filter", async () => {
+          await config.api.row.save(table._id!, {
+            one: "foo",
+            two: "bar",
+          })
+          await config.api.row.save(table._id!, {
+            one: "foo2",
+            two: "bar2",
+          })
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            query: [
+              {
+                operator: BasicOperator.EQUAL,
+                field: "two",
+                value: "bar2",
+              },
+            ],
+            schema: {
+              id: { visible: true },
+              one: { visible: false },
+              two: { visible: true },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id, {
+            query: {
+              equal: {
+                two: "bar",
+              },
+            },
+          })
+          expect(response.rows).toHaveLength(0)
+        })
     })
 
     describe("permissions", () => {

packages/server/src/api/routes/utils/validators.ts

@@ -1,6 +1,11 @@
 import { auth, permissions } from "@budibase/backend-core"
 import { DataSourceOperation } from "../../../constants"
-import { Table, WebhookActionType } from "@budibase/types"
+import {
+  EmptyFilterOption,
+  SearchFilters,
+  Table,
+  WebhookActionType,
+} from "@budibase/types"
 import Joi, { CustomValidator } from "joi"
 import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
 import sdk from "../../../sdk"
@@ -84,7 +89,12 @@ export function datasourceValidator() {
 }
 
 function filterObject() {
-  return Joi.object({
+  const conditionalFilteringObject = () =>
+    Joi.object({
+      conditions: Joi.array().items(Joi.link("#schema")).required(),
+    })
+
+  const filtersValidators: Record<keyof SearchFilters, any> = {
     string: Joi.object().optional(),
     fuzzy: Joi.object().optional(),
     range: Joi.object().optional(),
@@ -95,8 +105,17 @@ function filterObject() {
     oneOf: Joi.object().optional(),
     contains: Joi.object().optional(),
     notContains: Joi.object().optional(),
+    containsAny: Joi.object().optional(),
     allOr: Joi.boolean().optional(),
-  }).unknown(true)
+    onEmptyFilter: Joi.string()
+      .optional()
+      .valid(...Object.values(EmptyFilterOption)),
+    $and: conditionalFilteringObject(),
+    $or: conditionalFilteringObject(),
+    fuzzyOr: Joi.forbidden(),
+    documentType: Joi.forbidden(),
+  }
+  return Joi.object(filtersValidators).unknown(true).id("schema")
 }
 
 export function internalSearchValidator() {

packages/server/src/automations/steps/queryRows.ts

@@ -11,13 +11,10 @@ import {
   AutomationStepSchema,
   AutomationStepType,
   EmptyFilterOption,
-  SearchFilters,
-  Table,
   SortOrder,
   QueryRowsStepInputs,
   QueryRowsStepOutputs,
 } from "@budibase/types"
-import { db as dbCore } from "@budibase/backend-core"
 
 const SortOrderPretty = {
   [SortOrder.ASCENDING]: "Ascending",
@@ -95,38 +92,6 @@ async function getTable(appId: string, tableId: string) {
   return ctx.body
 }
 
-function typeCoercion(filters: SearchFilters, table: Table) {
-  if (!filters || !table) {
-    return filters
-  }
-  for (let key of Object.keys(filters)) {
-    const searchParam = filters[key as keyof SearchFilters]
-    if (typeof searchParam === "object") {
-      for (let [property, value] of Object.entries(searchParam)) {
-        // We need to strip numerical prefixes here, so that we can look up
-        // the correct field name in the schema
-        const columnName = dbCore.removeKeyNumbering(property)
-        const column = table.schema[columnName]
-
-        // convert string inputs
-        if (!column || typeof value !== "string") {
-          continue
-        }
-        if (column.type === FieldType.NUMBER) {
-          if (key === "oneOf") {
-            searchParam[property] = value
-              .split(",")
-              .map(item => parseFloat(item))
-          } else {
-            searchParam[property] = parseFloat(value)
-          }
-        }
-      }
-    }
-  }
-  return filters
-}
-
 function hasNullFilters(filters: any[]) {
   return (
     filters.length === 0 ||
@@ -157,7 +122,7 @@ export async function run({
     sortType =
       fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING
   }
-  const ctx: any = buildCtx(appId, null, {
+  const ctx = buildCtx(appId, null, {
     params: {
       tableId,
     },
@@ -165,7 +130,7 @@ export async function run({
       sortType,
       limit,
       sort: sortColumn,
-      query: typeCoercion(filters || {}, table),
+      query: filters || {},
       // default to ascending, like data tab
       sortOrder: sortOrder || SortOrder.ASCENDING,
     },

packages/server/src/features.ts

@@ -1,4 +1,3 @@
-import { features } from "@budibase/backend-core"
 import env from "./environment"
 
 enum AppFeature {
@@ -6,7 +5,25 @@ enum AppFeature {
   AUTOMATIONS = "automations",
 }
 
-const featureList = features.processFeatureEnvVar<AppFeature>(
+export function processFeatureEnvVar<T>(
+  fullList: string[],
+  featureList?: string
+) {
+  let list
+  if (!featureList) {
+    list = fullList
+  } else {
+    list = featureList.split(",")
+  }
+  for (let feature of list) {
+    if (!fullList.includes(feature)) {
+      throw new Error(`Feature: ${feature} is not an allowed option`)
+    }
+  }
+  return list as unknown as T[]
+}
+
+const featureList = processFeatureEnvVar<AppFeature>(
   Object.values(AppFeature),
   env.APP_FEATURES
 )

packages/server/src/integrations/oracle.ts

@@ -400,7 +400,9 @@ class OracleIntegration extends Sql implements DatasourcePlus {
             if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
               table.primary!.push(columnName)
             } else if (
-              oracleConstraint.type === OracleContraintTypes.NOT_NULL_OR_CHECK
+              oracleConstraint.type ===
+                OracleContraintTypes.NOT_NULL_OR_CHECK &&
+              oracleConstraint.searchCondition?.endsWith("IS NOT NULL")
             ) {
               table.schema[columnName].constraints = {
                 presence: true,
@@ -421,7 +423,11 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
       sql: OracleIntegration.COLUMNS_SQL,
     })
-    return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
+    const tableNames = new Set<string>()
+    for (const row of columnsResponse.rows || []) {
+      tableNames.add(row.TABLE_NAME)
+    }
+    return Array.from(tableNames)
   }
 
   async testConnection() {

packages/server/src/sdk/app/rows/search.ts

@@ -13,6 +13,7 @@ import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../index"
 import { searchInputMapping } from "./search/utils"
 import { db as dbCore } from "@budibase/backend-core"
+import tracer from "dd-trace"
 
 export { isValidFilter } from "../../../integrations/utils"
 
@@ -32,13 +33,34 @@ function pickApi(tableId: any) {
 export async function search(
   options: RowSearchParams
 ): Promise<SearchResponse<Row>> {
+  return await tracer.trace("search", async span => {
+    span?.addTags({
+      tableId: options.tableId,
+      query: options.query,
+      sort: options.sort,
+      sortOrder: options.sortOrder,
+      sortType: options.sortType,
+      limit: options.limit,
+      bookmark: options.bookmark,
+      paginate: options.paginate,
+      fields: options.fields,
+      countRows: options.countRows,
+    })
+
     const isExternalTable = isExternalTableID(options.tableId)
     options.query = dataFilters.cleanupQuery(options.query || {})
     options.query = dataFilters.fixupFilterArrays(options.query)
+
+    span?.addTags({
+      cleanedQuery: options.query,
+      isExternalTable,
+    })
+
     if (
       !dataFilters.hasFilters(options.query) &&
       options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
     ) {
+      span?.addTags({ emptyQuery: true })
       return {
         rows: [],
       }
@@ -51,13 +73,25 @@ export async function search(
     const table = await sdk.tables.getTable(options.tableId)
     options = searchInputMapping(table, options)
 
+    let result: SearchResponse<Row>
     if (isExternalTable) {
-    return external.search(options, table)
+      span?.addTags({ searchType: "external" })
+      result = await external.search(options, table)
     } else if (dbCore.isSqsEnabledForTenant()) {
-    return internal.sqs.search(options, table)
+      span?.addTags({ searchType: "sqs" })
+      result = await internal.sqs.search(options, table)
     } else {
-    return internal.lucene.search(options, table)
+      span?.addTags({ searchType: "lucene" })
+      result = await internal.lucene.search(options, table)
     }
+
+    span?.addTags({
+      foundRows: result.rows.length,
+      totalRows: result.totalRows,
+    })
+
+    return result
+  })
 }
 
 export async function exportRows(

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -2,6 +2,7 @@ import {
   Datasource,
   DocumentType,
   FieldType,
+  isLogicalSearchOperator,
   Operation,
   QueryJson,
   RelationshipFieldMetadata,
@@ -137,20 +138,33 @@ function cleanupFilters(
     allTables.some(table => table.schema[key])
 
   const splitter = new dataFilters.ColumnSplitter(allTables)
-  for (const filter of Object.values(filters)) {
+
+  const prefixFilters = (filters: SearchFilters) => {
+    for (const filterKey of Object.keys(filters) as (keyof SearchFilters)[]) {
+      if (isLogicalSearchOperator(filterKey)) {
+        for (const condition of filters[filterKey]!.conditions) {
+          prefixFilters(condition)
+        }
+      } else {
+        const filter = filters[filterKey]!
+        if (typeof filter !== "object") {
+          continue
+        }
         for (const key of Object.keys(filter)) {
           const { numberPrefix, relationshipPrefix, column } = splitter.run(key)
           if (keyInAnyTable(column)) {
             filter[
-          `${numberPrefix || ""}${relationshipPrefix || ""}${mapToUserColumn(
+              `${numberPrefix || ""}${
-            column
+                relationshipPrefix || ""
-          )}`
+              }${mapToUserColumn(column)}`
             ] = filter[key]
             delete filter[key]
           }
         }
       }
-
+    }
+  }
+  prefixFilters(filters)
   return filters
 }
 

packages/shared-core/src/filters.ts

@@ -17,6 +17,8 @@ import {
   Table,
   BasicOperator,
   RangeOperator,
+  LogicalOperator,
+  isLogicalSearchOperator,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
@@ -358,6 +360,8 @@ export const buildQuery = (filter: SearchFilter[]) => {
           high: value,
         }
       }
+    } else if (isLogicalSearchOperator(queryOperator)) {
+      // TODO
     } else if (query[queryOperator] && operator !== "onEmptyFilter") {
       if (type === "boolean") {
         // Transform boolean filters to cope with null.
@@ -458,14 +462,17 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
     ) =>
     (doc: Record<string, any>) => {
       for (const [key, testValue] of Object.entries(query[type] || {})) {
-        const result = test(deepGet(doc, removeKeyNumbering(key)), testValue)
+        const valueToCheck = isLogicalSearchOperator(type)
+          ? doc
+          : deepGet(doc, removeKeyNumbering(key))
+        const result = test(valueToCheck, testValue)
         if (query.allOr && result) {
           return true
         } else if (!query.allOr && !result) {
           return false
         }
       }
-      return true
+      return !query.allOr
     }
 
   const stringMatch = match(
@@ -666,8 +673,45 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
   )
   const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some"))
 
+  const and = match(
+    LogicalOperator.AND,
+    (docValue: Record<string, any>, conditions: SearchFilters[]) => {
+      if (!conditions.length) {
+        return false
+      }
+      for (const condition of conditions) {
+        const matchesCondition = runQuery([docValue], condition)
+        if (!matchesCondition.length) {
+          return false
+        }
+      }
+      return true
+    }
+  )
+  const or = match(
+    LogicalOperator.OR,
+    (docValue: Record<string, any>, conditions: SearchFilters[]) => {
+      if (!conditions.length) {
+        return false
+      }
+      for (const condition of conditions) {
+        const matchesCondition = runQuery([docValue], {
+          ...condition,
+          allOr: true,
+        })
+        if (matchesCondition.length) {
+          return true
+        }
+      }
+      return false
+    }
+  )
+
   const docMatch = (doc: Record<string, any>) => {
-    const filterFunctions = {
+    const filterFunctions: Record<
+      SearchFilterOperator,
+      (doc: Record<string, any>) => boolean
+    > = {
       string: stringMatch,
       fuzzy: fuzzyMatch,
       range: rangeMatch,
@@ -679,6 +723,8 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
       contains: contains,
       containsAny: containsAny,
       notContains: notContains,
+      [LogicalOperator.AND]: and,
+      [LogicalOperator.OR]: or,
     }
 
     const results = Object.entries(query || {})

packages/shared-core/tsconfig.build.json

@@ -18,6 +18,6 @@
     },
     "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
   },
-  "include": ["src/**/*"],
+  "include": ["src/**/*.ts"],
   "exclude": ["**/*.spec.ts", "**/*.spec.js", "__mocks__", "src/tests"]
 }

packages/shared-core/tsconfig.json

@@ -1,9 +1,6 @@
 {
   "extends": "./tsconfig.build.json",
   "compilerOptions": {
-    "baseUrl": "..",
-    "rootDir": "src",
-    "composite": true,
     "types": ["node", "jest"]
   },
   "exclude": ["node_modules", "dist"]

packages/types/src/sdk/search.ts

@@ -23,7 +23,22 @@ export enum RangeOperator {
   RANGE = "range",
 }
 
-export type SearchFilterOperator = BasicOperator | ArrayOperator | RangeOperator
+export enum LogicalOperator {
+  AND = "$and",
+  OR = "$or",
+}
+
+export function isLogicalSearchOperator(
+  value: string
+): value is LogicalOperator {
+  return value === LogicalOperator.AND || value === LogicalOperator.OR
+}
+
+export type SearchFilterOperator =
+  | BasicOperator
+  | ArrayOperator
+  | RangeOperator
+  | LogicalOperator
 
 export enum InternalSearchFilterOperator {
   COMPLEX_ID_OPERATOR = "_complexIdOperator",
@@ -75,6 +90,13 @@ export interface SearchFilters {
   // to make sure the documents returned are always filtered down to a
   // specific document type (such as just rows)
   documentType?: DocumentType
+
+  [LogicalOperator.AND]?: {
+    conditions: SearchFilters[]
+  }
+  [LogicalOperator.OR]?: {
+    conditions: SearchFilters[]
+  }
 }
 
 export type SearchFilterKey = keyof Omit<

packages/worker/src/api/controllers/global/self.ts

@@ -1,6 +1,6 @@
 import * as userSdk from "../../../sdk/users"
 import {
-  featureFlags,
+  features,
   tenancy,
   db as dbCore,
   utils,
@@ -104,8 +104,8 @@ export async function getSelf(ctx: any) {
   ctx.body = await groups.enrichUserRolesFromGroups(user)
 
   // add the feature flags for this tenant
-  const tenantId = tenancy.getTenantId()
+  const flags = await features.fetch()
-  ctx.body.featureFlags = featureFlags.getTenantFeatureFlags(tenantId)
+  ctx.body.flags = flags
 
   addSessionAttributesToUser(ctx)
 }

packages/worker/src/environment.ts

@@ -19,8 +19,6 @@ function parseIntSafe(number: any) {
 }
 
 const environment = {
-  // features
-  WORKER_FEATURES: process.env.WORKER_FEATURES,
   // auth
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,

packages/worker/src/features.ts

@@ -1,13 +0,0 @@
-import { features } from "@budibase/backend-core"
-import env from "./environment"
-
-enum WorkerFeature {}
-
-const featureList: WorkerFeature[] = features.processFeatureEnvVar(
-  Object.values(WorkerFeature),
-  env.WORKER_FEATURES
-)
-
-export function isFeatureEnabled(feature: WorkerFeature) {
-  return featureList.includes(feature)
-}

yarn.lock

@@ -12072,10 +12072,10 @@ google-p12-pem@^4.0.0:
   dependencies:
     node-forge "^1.3.1"
 
-"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.2":
+"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3":
-  version "4.1.2"
+  version "4.1.3"
-  resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.2.tgz#90548ccba2284b3042b08d2974ef3caeaf772ad9"
+  resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad"
-  integrity sha512-dxoY3rQGGnuNeZiXhNc9oYPduzU8xnIjWujFwNvaRRv3zWeUV7mj6HE2o/OJOeekPGt7o44B+w6DfkiaoteZgg==
+  integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w==
   dependencies:
     axios "^1.4.0"
     lodash "^4.17.21"