diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 6cdfba068b..3e4b2221d2 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -194,7 +194,8 @@ jobs: node-version: 18.x cache: "yarn" - run: yarn --frozen-lockfile - - run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client + - name: Build packages + run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client --scope @budibase/backend-core - name: Run tests run: | cd qa-core diff --git a/.vscode/launch.json b/.vscode/launch.json index 6c0089bb6b..cfd8d7b155 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,3 +1,4 @@ + { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. @@ -8,7 +9,6 @@ "name": "Budibase Server", "type": "node", "request": "launch", - "runtimeVersion": "14.20.1", "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], "args": ["${workspaceFolder}/packages/server/src/index.ts"], "cwd": "${workspaceFolder}/packages/server" @@ -17,7 +17,6 @@ "name": "Budibase Worker", "type": "node", "request": "launch", - "runtimeVersion": "14.20.1", "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], "args": ["${workspaceFolder}/packages/worker/src/index.ts"], "cwd": "${workspaceFolder}/packages/worker" diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index e5ce4f53fd..e5f1eabb53 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -137,7 +137,6 @@ services: path: /health port: 10000 scheme: HTTP - enabled: true periodSeconds: 3 failureThreshold: 1 livenessProbe: @@ -170,7 +169,6 @@ services: path: /health port: 4002 scheme: HTTP - enabled: true periodSeconds: 3 failureThreshold: 1 livenessProbe: @@ -204,7 +202,6 @@ services: path: /health port: 4003 scheme: HTTP - enabled: true periodSeconds: 3 failureThreshold: 1 livenessProbe: @@ -411,14 +408,12 @@ couchdb: ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes # FOR COUCHDB livenessProbe: - enabled: true failureThreshold: 3 initialDelaySeconds: 0 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 readinessProbe: - enabled: true failureThreshold: 3 initialDelaySeconds: 0 periodSeconds: 10 diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index bad34a20ea..b3887c15fa 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,6 +27,7 @@ services: BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL} BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} + OFFLINE_MODE: ${OFFLINE_MODE} depends_on: - worker-service - redis-service @@ -54,6 +55,7 @@ services: INTERNAL_API_KEY: ${INTERNAL_API_KEY} REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} + OFFLINE_MODE: ${OFFLINE_MODE} depends_on: - redis-service - minio-service diff --git a/lerna.json b/lerna.json index d00419f904..9cade6054d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.9.26-alpha.2", + "version": "2.9.30-alpha.7", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index a491451a62..7451d581b5 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -1,7 +1,6 @@ import fetch from "node-fetch" import { getCouchInfo } from "./couch" -import { SearchFilters, Row } from "@budibase/types" -import { createUserIndex } from "./searchIndexes/searchIndexes" +import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types" const QUERY_START_REGEX = /\d[0-9]*:/g @@ -65,6 +64,7 @@ export class QueryBuilder { this.#index = index this.#query = { allOr: false, + onEmptyFilter: EmptyFilterOption.RETURN_ALL, string: {}, fuzzy: {}, range: {}, @@ -218,6 +218,10 @@ export class QueryBuilder { this.#query.allOr = true } + setOnEmptyFilter(value: EmptyFilterOption) { + this.#query.onEmptyFilter = value + } + handleSpaces(input: string) { if (this.#noEscaping) { return input @@ -289,8 +293,9 @@ export class QueryBuilder { const builder = this let allOr = this.#query && this.#query.allOr let query = allOr ? "" : "*:*" + let allFiltersEmpty = true const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } - let tableId + let tableId: string = "" if (this.#query.equal!.tableId) { tableId = this.#query.equal!.tableId delete this.#query.equal!.tableId @@ -305,7 +310,7 @@ export class QueryBuilder { } const contains = (key: string, value: any, mode = "AND") => { - if (Array.isArray(value) && value.length === 0) { + if (!value || (Array.isArray(value) && value.length === 0)) { return null } if (!Array.isArray(value)) { @@ -384,6 +389,12 @@ export class QueryBuilder { built += ` ${mode} ` } built += expression + if ( + (typeof value !== "string" && value != null) || + (typeof value === "string" && value !== tableId && value !== "") + ) { + allFiltersEmpty = false + } } if (opts?.returnBuilt) { return built @@ -463,6 +474,13 @@ export class QueryBuilder { allOr = false build({ tableId }, equal) } + if (allFiltersEmpty) { + if (this.#query.onEmptyFilter === EmptyFilterOption.RETURN_NONE) { + return "" + } else if (this.#query?.allOr) { + return query.replace("()", "(*:*)") + } + } return query } diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index a82828d8f2..7716661d88 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -1,6 +1,6 @@ import { newid } from "../../docIds/newid" import { getDB } from "../db" -import { Database } from "@budibase/types" +import { Database, EmptyFilterOption } from "@budibase/types" import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene" const INDEX_NAME = "main" @@ -156,6 +156,76 @@ describe("lucene", () => { expect(resp.rows.length).toBe(2) }) + describe("empty filters behaviour", () => { + it("should return all rows by default", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addEqual("property", "") + builder.addEqual("number", null) + builder.addString("property", "") + builder.addFuzzy("property", "") + builder.addNotEqual("number", undefined) + builder.addOneOf("number", null) + builder.addContains("array", undefined) + builder.addNotContains("array", null) + builder.addContainsAny("array", null) + + const resp = await builder.run() + expect(resp.rows.length).toBe(3) + }) + + it("should return all rows when onEmptyFilter is ALL", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.setOnEmptyFilter(EmptyFilterOption.RETURN_ALL) + builder.setAllOr() + builder.addEqual("property", "") + builder.addEqual("number", null) + builder.addString("property", "") + builder.addFuzzy("property", "") + builder.addNotEqual("number", undefined) + builder.addOneOf("number", null) + builder.addContains("array", undefined) + builder.addNotContains("array", null) + builder.addContainsAny("array", null) + + const resp = await builder.run() + expect(resp.rows.length).toBe(3) + }) + + it("should return no rows when onEmptyFilter is NONE", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE) + builder.addEqual("property", "") + builder.addEqual("number", null) + builder.addString("property", "") + builder.addFuzzy("property", "") + builder.addNotEqual("number", undefined) + builder.addOneOf("number", null) + builder.addContains("array", undefined) + builder.addNotContains("array", null) + builder.addContainsAny("array", null) + + const resp = await builder.run() + expect(resp.rows.length).toBe(0) + }) + + it("should return all matching rows when onEmptyFilter is NONE, but a filter value is provided", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE) + builder.addEqual("property", "") + builder.addEqual("number", 1) + builder.addString("property", "") + builder.addFuzzy("property", "") + builder.addNotEqual("number", undefined) + builder.addOneOf("number", null) + builder.addContains("array", undefined) + builder.addNotContains("array", null) + builder.addContainsAny("array", null) + + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + }) + describe("skip", () => { const skipDbName = `db-${newid()}` let docs: { diff --git a/packages/backend-core/src/featureFlags/index.ts b/packages/backend-core/src/features/index.ts similarity index 98% rename from packages/backend-core/src/featureFlags/index.ts rename to packages/backend-core/src/features/index.ts index 877cd60e1a..8f5c903e05 100644 --- a/packages/backend-core/src/featureFlags/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -1,5 +1,6 @@ import env from "../environment" import * as context from "../context" +export * from "./installation" /** * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant. diff --git a/packages/backend-core/src/features/installation.ts b/packages/backend-core/src/features/installation.ts new file mode 100644 index 0000000000..defc8bf987 --- /dev/null +++ b/packages/backend-core/src/features/installation.ts @@ -0,0 +1,17 @@ +export function processFeatureEnvVar( + fullList: string[], + featureList?: string +) { + let list + if (!featureList) { + list = fullList + } else { + list = featureList.split(",") + } + for (let feature of list) { + if (!fullList.includes(feature)) { + throw new Error(`Feature: ${feature} is not an allowed option`) + } + } + return list as unknown as T[] +} diff --git a/packages/backend-core/src/featureFlags/tests/featureFlags.spec.ts b/packages/backend-core/src/features/tests/featureFlags.spec.ts similarity index 100% rename from packages/backend-core/src/featureFlags/tests/featureFlags.spec.ts rename to packages/backend-core/src/features/tests/featureFlags.spec.ts diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 7b98674788..ffffd8240a 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -6,7 +6,8 @@ export * as roles from "./security/roles" export * as permissions from "./security/permissions" export * as accounts from "./accounts" export * as installation from "./installation" -export * as featureFlags from "./featureFlags" +export * as featureFlags from "./features" +export * as features from "./features/installation" export * as sessions from "./security/sessions" export * as platform from "./platform" export * as auth from "./auth" diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 55cc97bb1c..14140cba81 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -1,30 +1,30 @@ import env from "../environment" import * as eventHelpers from "./events" import * as accounts from "../accounts" +import * as accountSdk from "../accounts" import * as cache from "../cache" -import { getIdentity, getTenantId, getGlobalDB } from "../context" +import { getGlobalDB, getIdentity, getTenantId } from "../context" import * as dbUtils from "../db" import { EmailUnavailableError, HTTPError } from "../errors" import * as platform from "../platform" import * as sessions from "../security/sessions" import * as usersCore from "./users" import { + Account, AllDocsResponse, BulkUserCreated, BulkUserDeleted, + isSSOAccount, + isSSOUser, RowResponse, SaveUserOpts, User, - Account, - isSSOUser, - isSSOAccount, UserStatus, } from "@budibase/types" -import * as accountSdk from "../accounts" import { - validateUniqueUser, getAccountHolderFromUserIds, isAdmin, + validateUniqueUser, } from "./utils" import { searchExistingEmails } from "./lookup" import { hash } from "../utils" @@ -179,6 +179,14 @@ export class UserDB { return user } + static async bulkGet(userIds: string[]) { + return await usersCore.bulkGetGlobalUsersById(userIds) + } + + static async bulkUpdate(users: User[]) { + return await usersCore.bulkUpdateGlobalUsers(users) + } + static async save(user: User, opts: SaveUserOpts = {}): Promise { // default booleans to true if (opts.hashPassword == null) { diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index 6747282040..14a1f1f4d3 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -86,6 +86,10 @@ export const useAuditLogs = () => { return useFeature(Feature.AUDIT_LOGS) } +export const usePublicApiUserRoles = () => { + return useFeature(Feature.USER_ROLE_PUBLIC_API) +} + export const useScimIntegration = () => { return useFeature(Feature.SCIM) } diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index e9f6e5629f..ef8699824e 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -35,22 +35,28 @@ { value: "and", label: "Match all filters" }, { value: "or", label: "Match any filter" }, ] + const onEmptyOptions = [ + { value: "all", label: "Return all table rows" }, + { value: "none", label: "Return no rows" }, + ] let rawFilters let matchAny = false + let onEmptyFilter = "all" $: parseFilters(filters) - $: dispatch("change", enrichFilters(rawFilters, matchAny)) + $: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter)) $: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true }) $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] - // Remove field key prefixes and determine whether to use the "match all" - // or "match any" behaviour + // Remove field key prefixes and determine which behaviours to use const parseFilters = filters => { matchAny = filters?.find(filter => filter.operator === "allOr") != null + onEmptyFilter = + filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" rawFilters = (filters || []) - .filter(filter => filter.operator !== "allOr") + .filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter) .map(filter => { const { field } = filter let newFilter = { ...filter } @@ -74,8 +80,8 @@ }) // Add field key prefixes and a special metadata filter object to indicate - // whether to use the "match all" or "match any" behaviour - const enrichFilters = (rawFilters, matchAny) => { + // how to handle filter behaviour + const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => { let count = 1 return rawFilters .filter(filter => filter.field) @@ -84,6 +90,7 @@ field: `${count++}:${filter.field}`, })) .concat(matchAny ? [{ operator: "allOr" }] : []) + .concat([{ onEmptyFilter }]) } const addFilter = () => { @@ -195,6 +202,17 @@ on:change={e => (matchAny = e.detail === "or")} placeholder={null} /> + {#if datasource?.type === "table"} +