diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 288a0462e7..d63596f08f 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -108,7 +108,7 @@ jobs: - name: Pull testcontainers images run: | docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb:v3.2.1-sql & + docker pull budibase/couchdb:v3.2.1-sqs & docker pull redis & wait $(jobs -p) diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 906a95e1db..4362bd2c94 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -206,3 +206,21 @@ export function pagination( nextPage, } } + +export function isSqsEnabledForTenant(): boolean { + const tenantId = getTenantId() + if (!env.SQS_SEARCH_ENABLE) { + return false + } + + // This is to guard against the situation in tests where tests pass because + // we're not actually using SQS, we're using Lucene and the tests pass due to + // parity. + if (env.isTest() && env.SQS_SEARCH_ENABLE_TENANTS.length === 0) { + throw new Error( + "to enable SQS you must specify a list of tenants in the SQS_SEARCH_ENABLE_TENANTS env var" + ) + } + + return env.SQS_SEARCH_ENABLE_TENANTS.includes(tenantId) +} diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index e06d51f918..8ed96b32c3 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -116,6 +116,9 @@ const environment = { COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006", SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, + SQS_SEARCH_ENABLE_TENANTS: + process.env.SQS_SEARCH_ENABLE_TENANTS?.split(",") || [], + SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE, COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 7d62a6ef39..0c994d8287 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -18,9 +18,10 @@ import { CouchFindOptions, DatabaseQueryOpts, SearchFilters, - SearchFilterOperator, SearchUsersRequest, User, + BasicOperator, + ArrayOperator, } from "@budibase/types" import * as context from "../context" import { getGlobalDB } from "../context" @@ -46,9 +47,9 @@ function removeUserPassword(users: User | User[]) { export function isSupportedUserSearch(query: SearchFilters) { const allowed = [ - { op: SearchFilterOperator.STRING, key: "email" }, - { op: SearchFilterOperator.EQUAL, key: "_id" }, - { op: SearchFilterOperator.ONE_OF, key: "_id" }, + { op: BasicOperator.STRING, key: "email" }, + { op: BasicOperator.EQUAL, key: "_id" }, + { op: ArrayOperator.ONE_OF, key: "_id" }, ] for (let [key, operation] of Object.entries(query)) { if (typeof operation !== "object") { diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index c0bc328a4e..5b6b6b4c86 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -11,7 +11,7 @@ Label, Multiselect, } from "@budibase/bbui" - import { FieldType, SearchFilterOperator } from "@budibase/types" + import { ArrayOperator, FieldType } from "@budibase/types" import { generate } from "shortid" import { QueryUtils, Constants } from "@budibase/frontend-core" import { getContext } from "svelte" @@ -268,7 +268,7 @@ {:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)} - {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchFilterOperator.ONE_OF)} + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} { + await config.withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init()) if (isSqs) { - envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + envCleanup = config.setCoreEnv({ + SQS_SEARCH_ENABLE: "true", + SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + }) } - await config.init() if (config.app?.appId) { config.app = await config.api.application.update(config.app?.appId, { @@ -780,6 +783,32 @@ describe.each([ it("fails to find nonexistent row", async () => { await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing() }) + + it("can have multiple values for same column", async () => { + await expectQuery({ + oneOf: { + name: ["foo", "bar"], + }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) + + it("splits comma separated strings", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + name: "foo,bar", + }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) + + it("trims whitespace", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + name: "foo, bar", + }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("fuzzy", () => { @@ -1002,6 +1031,32 @@ describe.each([ it("fails to find nonexistent row", async () => { await expectQuery({ oneOf: { age: [2] } }).toFindNothing() }) + + // I couldn't find a way to make this work in Lucene and given that + // we're getting rid of Lucene soon I wasn't inclined to spend time on + // it. + !isLucene && + it("can convert from a string", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + age: "1", + }, + }).toContainExactly([{ age: 1 }]) + }) + + // I couldn't find a way to make this work in Lucene and given that + // we're getting rid of Lucene soon I wasn't inclined to spend time on + // it. + !isLucene && + it("can find multiple values for same column", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + age: "1,10", + }, + }).toContainExactly([{ age: 1 }, { age: 10 }]) + }) }) describe("range", () => { diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 680ddb39d7..81c615487d 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -86,9 +86,10 @@ describe("/templates", () => { async source => { const env = { SQS_SEARCH_ENABLE: source === "sqs" ? "true" : "false", + SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], } - await config.withEnv(env, async () => { + await config.withCoreEnv(env, async () => { const name = generator.guid().replaceAll("-", "") const url = `/${name}` diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 43a6d39172..ba044acf81 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -9,7 +9,6 @@ import { QuotaUsageType, Row, SaveTableRequest, - SearchFilterOperator, SortOrder, SortType, StaticQuotaName, @@ -19,6 +18,7 @@ import { ViewUIFieldMetadata, ViewV2, SearchResponse, + BasicOperator, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -88,10 +88,16 @@ describe.each([ } beforeAll(async () => { + await config.withCoreEnv( + { SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, + () => config.init() + ) if (isSqs) { - envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + envCleanup = config.setCoreEnv({ + SQS_SEARCH_ENABLE: "true", + SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + }) } - await config.init() if (dsProvider) { datasource = await config.createDatasource({ @@ -149,7 +155,7 @@ describe.each([ primaryDisplay: "id", query: [ { - operator: SearchFilterOperator.EQUAL, + operator: BasicOperator.EQUAL, field: "field", value: "value", }, @@ -561,7 +567,7 @@ describe.each([ ...view, query: [ { - operator: SearchFilterOperator.EQUAL, + operator: BasicOperator.EQUAL, field: "newField", value: "thatValue", }, @@ -589,7 +595,7 @@ describe.each([ primaryDisplay: "Price", query: [ { - operator: SearchFilterOperator.EQUAL, + operator: BasicOperator.EQUAL, field: generator.word(), value: generator.word(), }, @@ -673,7 +679,7 @@ describe.each([ tableId: generator.guid(), query: [ { - operator: SearchFilterOperator.EQUAL, + operator: BasicOperator.EQUAL, field: "newField", value: "thatValue", }, @@ -1194,7 +1200,7 @@ describe.each([ name: generator.guid(), query: [ { - operator: SearchFilterOperator.EQUAL, + operator: BasicOperator.EQUAL, field: "two", value: "bar2", }, diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts index 7437753ada..247b74ebfc 100644 --- a/packages/server/src/appMigrations/migrations.ts +++ b/packages/server/src/appMigrations/migrations.ts @@ -1,6 +1,6 @@ // This file should never be manually modified, use `yarn add-app-migration` in order to add a new one -import env from "../environment" +import { env } from "@budibase/backend-core" import { AppMigration } from "." import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs" diff --git a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts index da435705c1..ec7fba99e9 100644 --- a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts +++ b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts @@ -1,8 +1,7 @@ -import { context } from "@budibase/backend-core" +import { context, env } from "@budibase/backend-core" import { allLinkDocs } from "../../db/utils" import LinkDocumentImpl from "../../db/linkedRows/LinkDocument" import sdk from "../../sdk" -import env from "../../environment" const migration = async () => { const linkDocs = await allLinkDocs() diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index 0a34fb2bb4..f337958a5b 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -69,11 +69,14 @@ function oldLinkDocument(): Omit { type SQSEnvVar = "SQS_MIGRATION_ENABLE" | "SQS_SEARCH_ENABLE" async function sqsDisabled(envVar: SQSEnvVar, cb: () => Promise) { - await config.withEnv({ [envVar]: "" }, cb) + await config.withCoreEnv({ [envVar]: "", SQS_SEARCH_ENABLE_TENANTS: [] }, cb) } async function sqsEnabled(envVar: SQSEnvVar, cb: () => Promise) { - await config.withEnv({ [envVar]: "1" }, cb) + await config.withCoreEnv( + { [envVar]: "1", SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()] }, + cb + ) } describe.each(["SQS_MIGRATION_ENABLE", "SQS_SEARCH_ENABLE"] as SQSEnvVar[])( diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index b643bd50b0..28673dcaa8 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -87,8 +87,6 @@ const environment = { SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE, SQL_ALIASING_DISABLE: process.env.SQL_ALIASING_DISABLE, - SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, - SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE, // flags ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 286a88054c..763fbb46ef 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -2,7 +2,6 @@ import { EmptyFilterOption, Row, RowSearchParams, - SearchFilterOperator, SearchFilters, SearchResponse, SortOrder, @@ -12,11 +11,11 @@ import * as internal from "./search/internal" import * as external from "./search/external" import { NoEmptyFilterStrings } from "../../../constants" import * as sqs from "./search/sqs" -import env from "../../../environment" import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" +import { db as dbCore } from "@budibase/backend-core" export { isValidFilter } from "../../../integrations/utils" @@ -66,37 +65,12 @@ export function removeEmptyFilters(filters: SearchFilters) { return filters } -// The frontend can send single values for array fields sometimes, so to handle -// this we convert them to arrays at the controller level so that nothing below -// this has to worry about the non-array values. -function fixupFilterArrays(filters: SearchFilters) { - const arrayFields = [ - SearchFilterOperator.ONE_OF, - SearchFilterOperator.CONTAINS, - SearchFilterOperator.NOT_CONTAINS, - SearchFilterOperator.CONTAINS_ANY, - ] - for (const searchField of arrayFields) { - const field = filters[searchField] - if (field == null) { - continue - } - - for (const key of Object.keys(field)) { - if (!Array.isArray(field[key])) { - field[key] = [field[key]] - } - } - } - return filters -} - export async function search( options: RowSearchParams ): Promise> { const isExternalTable = isExternalTableID(options.tableId) options.query = removeEmptyFilters(options.query || {}) - options.query = fixupFilterArrays(options.query) + options.query = dataFilters.fixupFilterArrays(options.query) if ( !dataFilters.hasFilters(options.query) && options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE @@ -115,7 +89,7 @@ export async function search( if (isExternalTable) { return external.search(options, table) - } else if (env.SQS_SEARCH_ENABLE) { + } else if (dbCore.isSqsEnabledForTenant()) { return sqs.search(options, table) } else { return internal.search(options, table) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 2f347475f4..4b80d5bcd3 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -31,10 +31,17 @@ describe.each([ let rows: Row[] beforeAll(async () => { + await config.withCoreEnv( + { SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, + () => config.init() + ) + if (isSqs) { - envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + envCleanup = config.setCoreEnv({ + SQS_SEARCH_ENABLE: "true", + SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + }) } - await config.init() if (dsProvider) { datasource = await config.createDatasource({ diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 738e57eff8..725c4e5cd2 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,4 +1,4 @@ -import { context } from "@budibase/backend-core" +import { context, db as dbCore, env } from "@budibase/backend-core" import { getTableParams } from "../../../db/utils" import { breakExternalTableId, @@ -15,7 +15,6 @@ import { } from "@budibase/types" import datasources from "../datasources" import sdk from "../../../sdk" -import env from "../../../environment" export function processTable(table: Table): Table { if (!table) { @@ -34,7 +33,7 @@ export function processTable(table: Table): Table { sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID, sourceType: TableSourceType.INTERNAL, } - if (env.SQS_SEARCH_ENABLE) { + if (dbCore.isSqsEnabledForTenant()) { processed.sql = !!env.SQS_SEARCH_ENABLE } return processed diff --git a/packages/server/src/sdk/tests/attachments.spec.ts b/packages/server/src/sdk/tests/attachments.spec.ts index 12c808d3a5..5656a1b608 100644 --- a/packages/server/src/sdk/tests/attachments.spec.ts +++ b/packages/server/src/sdk/tests/attachments.spec.ts @@ -39,7 +39,9 @@ describe("should be able to re-write attachment URLs", () => { } const db = dbCore.getDB(config.getAppId()) - await sdk.backups.updateAttachmentColumns(db.name, db) + await config.doInContext(config.getAppId(), () => + sdk.backups.updateAttachmentColumns(db.name, db) + ) return { db, diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 793bfa8c6a..828b389add 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -245,10 +245,10 @@ export default class TestConfiguration { } } - async withEnv(newEnvVars: Partial, f: () => Promise) { + async withEnv(newEnvVars: Partial, f: () => Promise) { let cleanup = this.setEnv(newEnvVars) try { - await f() + return await f() } finally { cleanup() } @@ -273,13 +273,13 @@ export default class TestConfiguration { } } - async withCoreEnv( + async withCoreEnv( newEnvVars: Partial, - f: () => Promise + f: () => Promise ) { let cleanup = this.setCoreEnv(newEnvVars) try { - await f() + return await f() } finally { cleanup() } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 28f0b28425..d3fcae11e2 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -6,6 +6,7 @@ import { SearchFilter, SearchFilters, SearchQueryFields, + ArrayOperator, SearchFilterOperator, SortType, FieldConstraints, @@ -14,11 +15,13 @@ import { EmptyFilterOption, SearchResponse, Table, + BasicOperator, + RangeOperator, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" import { deepGet, schema } from "./helpers" -import _ from "lodash" +import { isPlainObject, isEmpty } from "lodash" const HBS_REGEX = /{{([^{].*?)}}/g @@ -323,6 +326,32 @@ export const buildQuery = (filter: SearchFilter[]) => { return query } +// The frontend can send single values for array fields sometimes, so to handle +// this we convert them to arrays at the controller level so that nothing below +// this has to worry about the non-array values. +export function fixupFilterArrays(filters: SearchFilters) { + for (const searchField of Object.values(ArrayOperator)) { + const field = filters[searchField] + if (field == null || !isPlainObject(field)) { + continue + } + + for (const key of Object.keys(field)) { + if (Array.isArray(field[key])) { + continue + } + + const value = field[key] as any + if (typeof value === "string") { + field[key] = value.split(",").map((x: string) => x.trim()) + } else { + field[key] = [value] + } + } + } + return filters +} + export const search = ( docs: Record[], query: RowSearchParams @@ -356,6 +385,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { } query = cleanupQuery(query) + query = fixupFilterArrays(query) if ( !hasFilters(query) && @@ -382,7 +412,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { } const stringMatch = match( - SearchFilterOperator.STRING, + BasicOperator.STRING, (docValue: any, testValue: any) => { if (!(typeof docValue === "string")) { return false @@ -395,7 +425,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { ) const fuzzyMatch = match( - SearchFilterOperator.FUZZY, + BasicOperator.FUZZY, (docValue: any, testValue: any) => { if (!(typeof docValue === "string")) { return false @@ -408,17 +438,17 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { ) const rangeMatch = match( - SearchFilterOperator.RANGE, + RangeOperator.RANGE, (docValue: any, testValue: any) => { if (docValue == null || docValue === "") { return false } - if (_.isObject(testValue.low) && _.isEmpty(testValue.low)) { + if (isPlainObject(testValue.low) && isEmpty(testValue.low)) { testValue.low = undefined } - if (_.isObject(testValue.high) && _.isEmpty(testValue.high)) { + if (isPlainObject(testValue.high) && isEmpty(testValue.high)) { testValue.high = undefined } @@ -497,11 +527,8 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { (...args: T): boolean => !f(...args) - const equalMatch = match(SearchFilterOperator.EQUAL, _valueMatches) - const notEqualMatch = match( - SearchFilterOperator.NOT_EQUAL, - not(_valueMatches) - ) + const equalMatch = match(BasicOperator.EQUAL, _valueMatches) + const notEqualMatch = match(BasicOperator.NOT_EQUAL, not(_valueMatches)) const _empty = (docValue: any) => { if (typeof docValue === "string") { @@ -516,26 +543,24 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { return docValue == null } - const emptyMatch = match(SearchFilterOperator.EMPTY, _empty) - const notEmptyMatch = match(SearchFilterOperator.NOT_EMPTY, not(_empty)) + const emptyMatch = match(BasicOperator.EMPTY, _empty) + const notEmptyMatch = match(BasicOperator.NOT_EMPTY, not(_empty)) - const oneOf = match( - SearchFilterOperator.ONE_OF, - (docValue: any, testValue: any) => { - if (typeof testValue === "string") { - testValue = testValue.split(",") - if (typeof docValue === "number") { - testValue = testValue.map((item: string) => parseFloat(item)) - } - } - - if (!Array.isArray(testValue)) { - return false - } - - return testValue.some(item => _valueMatches(docValue, item)) + const oneOf = match(ArrayOperator.ONE_OF, (docValue: any, testValue: any) => { + if (typeof testValue === "string") { + testValue = testValue.split(",") } - ) + + if (typeof docValue === "number") { + testValue = testValue.map((item: string) => parseFloat(item)) + } + + if (!Array.isArray(testValue)) { + return false + } + + return testValue.some(item => _valueMatches(docValue, item)) + }) const _contains = (f: "some" | "every") => (docValue: any, testValue: any) => { @@ -562,7 +587,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { } const contains = match( - SearchFilterOperator.CONTAINS, + ArrayOperator.CONTAINS, (docValue: any, testValue: any) => { if (Array.isArray(testValue) && testValue.length === 0) { return true @@ -571,7 +596,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { } ) const notContains = match( - SearchFilterOperator.NOT_CONTAINS, + ArrayOperator.NOT_CONTAINS, (docValue: any, testValue: any) => { // Not sure if this is logically correct, but at the time this code was // written the search endpoint behaved this way and we wanted to make this @@ -582,10 +607,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { return not(_contains("every"))(docValue, testValue) } ) - const containsAny = match( - SearchFilterOperator.CONTAINS_ANY, - _contains("some") - ) + const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some")) const docMatch = (doc: Record) => { const filterFunctions = { diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index ccb73a7fba..5607efece8 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -3,20 +3,28 @@ import { Row, Table, DocumentType } from "../documents" import { SortOrder, SortType } from "../api" import { Knex } from "knex" -export enum SearchFilterOperator { - STRING = "string", - FUZZY = "fuzzy", - RANGE = "range", +export enum BasicOperator { EQUAL = "equal", NOT_EQUAL = "notEqual", EMPTY = "empty", NOT_EMPTY = "notEmpty", - ONE_OF = "oneOf", + FUZZY = "fuzzy", + STRING = "string", +} + +export enum ArrayOperator { CONTAINS = "contains", NOT_CONTAINS = "notContains", CONTAINS_ANY = "containsAny", + ONE_OF = "oneOf", } +export enum RangeOperator { + RANGE = "range", +} + +export type SearchFilterOperator = BasicOperator | ArrayOperator | RangeOperator + export enum InternalSearchFilterOperator { COMPLEX_ID_OPERATOR = "_complexIdOperator", } @@ -52,17 +60,17 @@ export interface SearchFilters { // allows just fuzzy to be or - all the fuzzy/like parameters fuzzyOr?: boolean onEmptyFilter?: EmptyFilterOption - [SearchFilterOperator.STRING]?: BasicFilter - [SearchFilterOperator.FUZZY]?: BasicFilter - [SearchFilterOperator.RANGE]?: RangeFilter - [SearchFilterOperator.EQUAL]?: BasicFilter - [SearchFilterOperator.NOT_EQUAL]?: BasicFilter - [SearchFilterOperator.EMPTY]?: BasicFilter - [SearchFilterOperator.NOT_EMPTY]?: BasicFilter - [SearchFilterOperator.ONE_OF]?: ArrayFilter - [SearchFilterOperator.CONTAINS]?: ArrayFilter - [SearchFilterOperator.NOT_CONTAINS]?: ArrayFilter - [SearchFilterOperator.CONTAINS_ANY]?: ArrayFilter + [BasicOperator.STRING]?: BasicFilter + [BasicOperator.FUZZY]?: BasicFilter + [RangeOperator.RANGE]?: RangeFilter + [BasicOperator.EQUAL]?: BasicFilter + [BasicOperator.NOT_EQUAL]?: BasicFilter + [BasicOperator.EMPTY]?: BasicFilter + [BasicOperator.NOT_EMPTY]?: BasicFilter + [ArrayOperator.ONE_OF]?: ArrayFilter + [ArrayOperator.CONTAINS]?: ArrayFilter + [ArrayOperator.NOT_CONTAINS]?: ArrayFilter + [ArrayOperator.CONTAINS_ANY]?: ArrayFilter // specific to SQS/SQLite search on internal tables this can be used // to make sure the documents returned are always filtered down to a // specific document type (such as just rows) diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index 4deca5df6e..788b45f25d 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -24,7 +24,7 @@ async function isSqsAvailable() { } async function isSqsMissing() { - return env.SQS_SEARCH_ENABLE && !(await isSqsAvailable()) + return coreEnv.SQS_SEARCH_ENABLE && !(await isSqsAvailable()) } export const fetch = async (ctx: Ctx) => { diff --git a/packages/worker/src/api/index.ts b/packages/worker/src/api/index.ts index 08c65b98d4..8ab093a359 100644 --- a/packages/worker/src/api/index.ts +++ b/packages/worker/src/api/index.ts @@ -5,8 +5,7 @@ const compress = require("koa-compress") import zlib from "zlib" import { routes } from "./routes" import { middleware as pro, sdk } from "@budibase/pro" -import { auth, middleware } from "@budibase/backend-core" -import env from "../environment" +import { auth, middleware, env } from "@budibase/backend-core" if (env.SQS_SEARCH_ENABLE) { sdk.auditLogs.useSQLSearch() diff --git a/packages/worker/src/environment.ts b/packages/worker/src/environment.ts index d642d50846..9f7baf9e9b 100644 --- a/packages/worker/src/environment.ts +++ b/packages/worker/src/environment.ts @@ -46,7 +46,6 @@ const environment = { DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, - SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, // smtp SMTP_USER: process.env.SMTP_USER,