diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index c279babb71..701b5bb329 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -5,6 +5,8 @@ import { GoogleInnerConfig, OIDCConfig, OIDCInnerConfig, + SCIMConfig, + SCIMInnerConfig, SettingsConfig, SettingsInnerConfig, SMTPConfig, @@ -241,3 +243,10 @@ export async function getSMTPConfig( } } } + +// SCIM + +export async function getSCIMConfig(): Promise { + const config = await getConfig(ConfigType.SCIM) + return config?.config +} diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 15cec7a6b9..2bb8f815cf 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -22,6 +22,7 @@ export enum Header { TOKEN = "x-budibase-token", CSRF_TOKEN = "x-csrf-token", CORRELATION_ID = "x-budibase-correlation-id", + AUTHORIZATION = "authorization", } export enum GlobalRole { @@ -38,6 +39,7 @@ export enum Config { GOOGLE = "google", OIDC = "oidc", OIDC_LOGOS = "logos_oidc", + SCIM = "scim", } export const MIN_VALID_DATE = new Date(-2147483647000) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 02ba16aa8c..e1bd535b78 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -214,6 +214,13 @@ export function doInEnvironmentContext( return newContext(updates, task) } +export function doInScimContext(task: any) { + const updates: ContextMap = { + isScim: true, + } + return newContext(updates, task) +} + export function getEnvironmentVariables() { const context = Context.get() if (!context.environmentVariables) { @@ -270,3 +277,9 @@ export function getDevAppDB(opts?: any): Database { } return getDB(conversions.getDevelopmentAppID(appId), opts) } + +export function isScim(): boolean { + const context = Context.get() + const scimCall = context?.isScim + return !!scimCall +} diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.ts index 5c8ce6fc19..3f760d4a49 100644 --- a/packages/backend-core/src/context/tests/index.spec.ts +++ b/packages/backend-core/src/context/tests/index.spec.ts @@ -1,6 +1,6 @@ import { testEnv } from "../../../tests" -const context = require("../") -const { DEFAULT_TENANT_ID } = require("../../constants") +import * as context from "../" +import { DEFAULT_TENANT_ID } from "../../constants" describe("context", () => { describe("doInTenant", () => { @@ -131,4 +131,17 @@ describe("context", () => { }) }) }) + + describe("doInScimContext", () => { + it("returns true when set", () => { + context.doInScimContext(() => { + const isScim = context.isScim() + expect(isScim).toBe(true) + }) + }) + it("returns false when not set", () => { + const isScim = context.isScim() + expect(isScim).toBe(false) + }) + }) }) diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index 78197ed528..727dad80bc 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -6,4 +6,5 @@ export type ContextMap = { appId?: string identity?: IdentityContext environmentVariables?: Record + isScim?: boolean } diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index a569b17b36..ea93b91d14 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -8,3 +8,4 @@ export { default as Replication } from "./Replication" export * from "../constants/db" export { getGlobalDBName, baseGlobalDBName } from "../context" export * from "./lucene" +export * as searchIndexes from "./searchIndexes" diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 71ce4ba9ac..6f2f4fc991 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -1,12 +1,14 @@ import fetch from "node-fetch" import { getCouchInfo } from "./couch" import { SearchFilters, Row } from "@budibase/types" +import { createUserIndex } from "./searchIndexes/searchIndexes" const QUERY_START_REGEX = /\d[0-9]*:/g interface SearchResponse { rows: T[] | any[] - bookmark: string + bookmark?: string + totalRows: number } interface PaginatedSearchResponse extends SearchResponse { @@ -42,23 +44,26 @@ export function removeKeyNumbering(key: any): string { * Optionally takes a base lucene query object. */ export class QueryBuilder { - dbName: string - index: string - query: SearchFilters - limit: number - sort?: string - bookmark?: string - sortOrder: string - sortType: string - includeDocs: boolean - version?: string - indexBuilder?: () => Promise - noEscaping = false + #dbName: string + #index: string + #query: SearchFilters + #limit: number + #sort?: string + #bookmark?: string + #sortOrder: string + #sortType: string + #includeDocs: boolean + #version?: string + #indexBuilder?: () => Promise + #noEscaping = false + #skip?: number + + static readonly maxLimit = 200 constructor(dbName: string, index: string, base?: SearchFilters) { - this.dbName = dbName - this.index = index - this.query = { + this.#dbName = dbName + this.#index = index + this.#query = { allOr: false, string: {}, fuzzy: {}, @@ -73,86 +78,96 @@ export class QueryBuilder { containsAny: {}, ...base, } - this.limit = 50 - this.sortOrder = "ascending" - this.sortType = "string" - this.includeDocs = true + this.#limit = 50 + this.#sortOrder = "ascending" + this.#sortType = "string" + this.#includeDocs = true } disableEscaping() { - this.noEscaping = true + this.#noEscaping = true return this } setIndexBuilder(builderFn: () => Promise) { - this.indexBuilder = builderFn + this.#indexBuilder = builderFn return this } setVersion(version?: string) { if (version != null) { - this.version = version + this.#version = version } return this } setTable(tableId: string) { - this.query.equal!.tableId = tableId + this.#query.equal!.tableId = tableId return this } setLimit(limit?: number) { if (limit != null) { - this.limit = limit + this.#limit = limit } return this } setSort(sort?: string) { if (sort != null) { - this.sort = sort + this.#sort = sort } return this } setSortOrder(sortOrder?: string) { if (sortOrder != null) { - this.sortOrder = sortOrder + this.#sortOrder = sortOrder } return this } setSortType(sortType?: string) { if (sortType != null) { - this.sortType = sortType + this.#sortType = sortType } return this } setBookmark(bookmark?: string) { if (bookmark != null) { - this.bookmark = bookmark + this.#bookmark = bookmark } return this } + setSkip(skip: number | undefined) { + this.#skip = skip + return this + } + excludeDocs() { - this.includeDocs = false + this.#includeDocs = false + return this + } + + includeDocs() { + this.#includeDocs = true return this } addString(key: string, partial: string) { - this.query.string![key] = partial + this.#query.string![key] = partial return this } addFuzzy(key: string, fuzzy: string) { - this.query.fuzzy![key] = fuzzy + this.#query.fuzzy![key] = fuzzy return this } addRange(key: string, low: string | number, high: string | number) { - this.query.range![key] = { + this.#query.range![key] = { low, high, } @@ -160,51 +175,51 @@ export class QueryBuilder { } addEqual(key: string, value: any) { - this.query.equal![key] = value + this.#query.equal![key] = value return this } addNotEqual(key: string, value: any) { - this.query.notEqual![key] = value + this.#query.notEqual![key] = value return this } addEmpty(key: string, value: any) { - this.query.empty![key] = value + this.#query.empty![key] = value return this } addNotEmpty(key: string, value: any) { - this.query.notEmpty![key] = value + this.#query.notEmpty![key] = value return this } addOneOf(key: string, value: any) { - this.query.oneOf![key] = value + this.#query.oneOf![key] = value return this } addContains(key: string, value: any) { - this.query.contains![key] = value + this.#query.contains![key] = value return this } addNotContains(key: string, value: any) { - this.query.notContains![key] = value + this.#query.notContains![key] = value return this } addContainsAny(key: string, value: any) { - this.query.containsAny![key] = value + this.#query.containsAny![key] = value return this } setAllOr() { - this.query.allOr = true + this.#query.allOr = true } handleSpaces(input: string) { - if (this.noEscaping) { + if (this.#noEscaping) { return input } else { return input.replace(/ /g, "_") @@ -219,7 +234,7 @@ export class QueryBuilder { * @returns {string|*} */ preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) { - const hasVersion = !!this.version + const hasVersion = !!this.#version // Determine if type needs wrapped const originalType = typeof value // Convert to lowercase @@ -227,7 +242,7 @@ export class QueryBuilder { value = value.toLowerCase ? value.toLowerCase() : value } // Escape characters - if (!this.noEscaping && escape && originalType === "string") { + if (!this.#noEscaping && escape && originalType === "string") { value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&") } @@ -242,7 +257,7 @@ export class QueryBuilder { isMultiCondition() { let count = 0 - for (let filters of Object.values(this.query)) { + for (let filters of Object.values(this.#query)) { // not contains is one massive filter in allOr mode if (typeof filters === "object") { count += Object.keys(filters).length @@ -272,13 +287,13 @@ export class QueryBuilder { buildSearchQuery() { const builder = this - let allOr = this.query && this.query.allOr + let allOr = this.#query && this.#query.allOr let query = allOr ? "" : "*:*" const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } let tableId - if (this.query.equal!.tableId) { - tableId = this.query.equal!.tableId - delete this.query.equal!.tableId + if (this.#query.equal!.tableId) { + tableId = this.#query.equal!.tableId + delete this.#query.equal!.tableId } const equal = (key: string, value: any) => { @@ -363,8 +378,8 @@ export class QueryBuilder { } // Construct the actual lucene search query string from JSON structure - if (this.query.string) { - build(this.query.string, (key: string, value: any) => { + if (this.#query.string) { + build(this.#query.string, (key: string, value: any) => { if (!value) { return null } @@ -376,8 +391,8 @@ export class QueryBuilder { return `${key}:${value}*` }) } - if (this.query.range) { - build(this.query.range, (key: string, value: any) => { + if (this.#query.range) { + build(this.#query.range, (key: string, value: any) => { if (!value) { return null } @@ -392,8 +407,8 @@ export class QueryBuilder { return `${key}:[${low} TO ${high}]` }) } - if (this.query.fuzzy) { - build(this.query.fuzzy, (key: string, value: any) => { + if (this.#query.fuzzy) { + build(this.#query.fuzzy, (key: string, value: any) => { if (!value) { return null } @@ -405,34 +420,34 @@ export class QueryBuilder { return `${key}:${value}~` }) } - if (this.query.equal) { - build(this.query.equal, equal) + if (this.#query.equal) { + build(this.#query.equal, equal) } - if (this.query.notEqual) { - build(this.query.notEqual, (key: string, value: any) => { + if (this.#query.notEqual) { + build(this.#query.notEqual, (key: string, value: any) => { if (!value) { return null } return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}` }) } - if (this.query.empty) { - build(this.query.empty, (key: string) => `!${key}:["" TO *]`) + if (this.#query.empty) { + build(this.#query.empty, (key: string) => `!${key}:["" TO *]`) } - if (this.query.notEmpty) { - build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`) + if (this.#query.notEmpty) { + build(this.#query.notEmpty, (key: string) => `${key}:["" TO *]`) } - if (this.query.oneOf) { - build(this.query.oneOf, oneOf) + if (this.#query.oneOf) { + build(this.#query.oneOf, oneOf) } - if (this.query.contains) { - build(this.query.contains, contains) + if (this.#query.contains) { + build(this.#query.contains, contains) } - if (this.query.notContains) { - build(this.compressFilters(this.query.notContains), notContains) + if (this.#query.notContains) { + build(this.compressFilters(this.#query.notContains), notContains) } - if (this.query.containsAny) { - build(this.query.containsAny, containsAny) + if (this.#query.containsAny) { + build(this.#query.containsAny, containsAny) } // make sure table ID is always added as an AND if (tableId) { @@ -446,29 +461,65 @@ export class QueryBuilder { buildSearchBody() { let body: any = { q: this.buildSearchQuery(), - limit: Math.min(this.limit, 200), - include_docs: this.includeDocs, + limit: Math.min(this.#limit, QueryBuilder.maxLimit), + include_docs: this.#includeDocs, } - if (this.bookmark) { - body.bookmark = this.bookmark + if (this.#bookmark) { + body.bookmark = this.#bookmark } - if (this.sort) { - const order = this.sortOrder === "descending" ? "-" : "" - const type = `<${this.sortType}>` - body.sort = `${order}${this.handleSpaces(this.sort)}${type}` + if (this.#sort) { + const order = this.#sortOrder === "descending" ? "-" : "" + const type = `<${this.#sortType}>` + body.sort = `${order}${this.handleSpaces(this.#sort)}${type}` } return body } async run() { + if (this.#skip) { + await this.#skipItems(this.#skip) + } + return await this.#execute() + } + + /** + * Lucene queries do not support pagination and use bookmarks instead. + * For the given builder, walk through pages using bookmarks until the desired + * page has been met. + */ + async #skipItems(skip: number) { + // Lucene does not support pagination. + // Handle pagination by finding the right bookmark + const prevIncludeDocs = this.#includeDocs + const prevLimit = this.#limit + + this.excludeDocs() + let skipRemaining = skip + let iterationFetched = 0 + do { + const toSkip = Math.min(QueryBuilder.maxLimit, skipRemaining) + this.setLimit(toSkip) + const { bookmark, rows } = await this.#execute() + this.setBookmark(bookmark) + iterationFetched = rows.length + skipRemaining -= rows.length + } while (skipRemaining > 0 && iterationFetched > 0) + + this.#includeDocs = prevIncludeDocs + this.#limit = prevLimit + } + + async #execute() { const { url, cookie } = getCouchInfo() - const fullPath = `${url}/${this.dbName}/_design/database/_search/${this.index}` + const fullPath = `${url}/${this.#dbName}/_design/database/_search/${ + this.#index + }` const body = this.buildSearchBody() try { return await runQuery(fullPath, body, cookie) } catch (err: any) { - if (err.status === 404 && this.indexBuilder) { - await this.indexBuilder() + if (err.status === 404 && this.#indexBuilder) { + await this.#indexBuilder() return await runQuery(fullPath, body, cookie) } else { throw err @@ -502,8 +553,9 @@ async function runQuery( } const json = await response.json() - let output: any = { + let output: SearchResponse = { rows: [], + totalRows: 0, } if (json.rows != null && json.rows.length > 0) { output.rows = json.rows.map((row: any) => row.doc) @@ -511,6 +563,9 @@ async function runQuery( if (json.bookmark) { output.bookmark = json.bookmark } + if (json.total_rows) { + output.totalRows = json.total_rows + } return output } @@ -543,8 +598,8 @@ async function recursiveSearch( if (rows.length >= params.limit) { return rows } - let pageSize = 200 - if (rows.length > params.limit - 200) { + let pageSize = QueryBuilder.maxLimit + if (rows.length > params.limit - QueryBuilder.maxLimit) { pageSize = params.limit - rows.length } const page = await new QueryBuilder(dbName, index, query) @@ -559,7 +614,7 @@ async function recursiveSearch( if (!page.rows.length) { return rows } - if (page.rows.length < 200) { + if (page.rows.length < QueryBuilder.maxLimit) { return [...rows, ...page.rows] } const newParams = { @@ -597,7 +652,7 @@ export async function paginatedSearch( if (limit == null || isNaN(limit) || limit < 0) { limit = 50 } - limit = Math.min(limit, 200) + limit = Math.min(limit, QueryBuilder.maxLimit) const search = new QueryBuilder(dbName, index, query) if (params.version) { search.setVersion(params.version) diff --git a/packages/backend-core/src/db/searchIndexes/index.ts b/packages/backend-core/src/db/searchIndexes/index.ts new file mode 100644 index 0000000000..d3054e90c0 --- /dev/null +++ b/packages/backend-core/src/db/searchIndexes/index.ts @@ -0,0 +1 @@ +export * from "./searchIndexes" diff --git a/packages/backend-core/src/db/searchIndexes/searchIndexes.ts b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts new file mode 100644 index 0000000000..f03259b47f --- /dev/null +++ b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts @@ -0,0 +1,62 @@ +import { User, SearchIndex } from "@budibase/types" +import { getGlobalDB } from "../../context" + +export async function createUserIndex() { + const db = getGlobalDB() + let designDoc + try { + designDoc = await db.get("_design/database") + } catch (err: any) { + if (err.status === 404) { + designDoc = { _id: "_design/database" } + } + } + + const fn = function (user: User) { + if (user._id && !user._id.startsWith("us_")) { + return + } + const ignoredFields = [ + "_id", + "_rev", + "password", + "account", + "license", + "budibaseAccess", + "accountPortalAccess", + "csrfToken", + ] + + function idx(input: Record, prev?: string) { + for (let key of Object.keys(input)) { + if (ignoredFields.includes(key)) { + continue + } + let idxKey = prev != null ? `${prev}.${key}` : key + if (typeof input[key] === "string") { + // eslint-disable-next-line no-undef + // @ts-ignore + index(idxKey, input[key].toLowerCase(), { facet: true }) + } else if (typeof input[key] !== "object") { + // eslint-disable-next-line no-undef + // @ts-ignore + index(idxKey, input[key], { facet: true }) + } else { + idx(input[key], idxKey) + } + } + } + idx(user) + } + + designDoc.indexes = { + [SearchIndex.USER]: { + index: fn.toString(), + analyzer: { + default: "keyword", + name: "perfield", + }, + }, + } + await db.put(designDoc) +} diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index 52017cc94c..26ce316a9d 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -136,6 +136,106 @@ describe("lucene", () => { const resp = await builder.run() expect(resp.rows.length).toBe(2) }) + + describe("skip", () => { + const skipDbName = `db-${newid()}` + let docs: { + _id: string + property: string + array: string[] + }[] + + beforeAll(async () => { + const db = getDB(skipDbName) + + docs = Array(QueryBuilder.maxLimit * 2.5) + .fill(0) + .map((_, i) => ({ + _id: i.toString().padStart(3, "0"), + property: `value_${i.toString().padStart(3, "0")}`, + array: [], + })) + await db.bulkDocs(docs) + + await db.put({ + _id: "_design/database", + indexes: { + [INDEX_NAME]: { + index: index, + analyzer: "standard", + }, + }, + }) + }) + + it("should be able to apply skip", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + const firstResponse = await builder.run() + builder.setSkip(40) + const secondResponse = await builder.run() + + // Return the default limit + expect(firstResponse.rows.length).toBe(50) + expect(secondResponse.rows.length).toBe(50) + + // Should have the expected overlap + expect(firstResponse.rows.slice(40)).toEqual( + secondResponse.rows.slice(0, 10) + ) + }) + + it("should handle limits", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + builder.setLimit(10) + builder.setSkip(50) + builder.setSort("_id") + + const resp = await builder.run() + expect(resp.rows.length).toBe(10) + expect(resp.rows).toEqual( + docs.slice(50, 60).map(expect.objectContaining) + ) + }) + + it("should be able to skip searching through multiple responses", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + // Skipping 2 max limits plus a little bit more + const skip = QueryBuilder.maxLimit * 2 + 37 + builder.setSkip(skip) + builder.setSort("_id") + const resp = await builder.run() + + expect(resp.rows.length).toBe(50) + expect(resp.rows).toEqual( + docs.slice(skip, skip + resp.rows.length).map(expect.objectContaining) + ) + }) + + it("should not return results if skipping all docs", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + // Skipping 2 max limits plus a little bit more + const skip = docs.length + 1 + builder.setSkip(skip) + + const resp = await builder.run() + + expect(resp.rows.length).toBe(0) + }) + + it("skip should respect with filters", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + builder.setLimit(10) + builder.setSkip(50) + builder.addString("property", "value_1") + builder.setSort("property") + + const resp = await builder.run() + expect(resp.rows.length).toBe(10) + expect(resp.rows).toEqual( + docs.slice(150, 160).map(expect.objectContaining) + ) + }) + }) }) describe("paginated search", () => { diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 76c52d08ad..441c118235 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -434,8 +434,8 @@ export const getPluginParams = (pluginId?: string | null, otherProps = {}) => { return getDocParams(DocumentType.PLUGIN, pluginId, otherProps) } -export function pagination( - data: any[], +export function pagination( + data: T[], pageSize: number, { paginate, @@ -444,7 +444,7 @@ export function pagination( }: { paginate: boolean property: string - getKey?: (doc: any) => string | undefined + getKey?: (doc: T) => string | undefined } = { paginate: true, property: "_id", diff --git a/packages/backend-core/src/events/publishers/group.ts b/packages/backend-core/src/events/publishers/group.ts index a000b880a2..6870ceb350 100644 --- a/packages/backend-core/src/events/publishers/group.ts +++ b/packages/backend-core/src/events/publishers/group.ts @@ -9,12 +9,13 @@ import { GroupUsersDeletedEvent, GroupAddedOnboardingEvent, GroupPermissionsEditedEvent, - UserGroupRoles, } from "@budibase/types" +import { isScim } from "../../context" async function created(group: UserGroup, timestamp?: number) { const properties: GroupCreatedEvent = { groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, @@ -25,6 +26,7 @@ async function created(group: UserGroup, timestamp?: number) { async function updated(group: UserGroup) { const properties: GroupUpdatedEvent = { groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, @@ -35,6 +37,7 @@ async function updated(group: UserGroup) { async function deleted(group: UserGroup) { const properties: GroupDeletedEvent = { groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, @@ -46,6 +49,7 @@ async function usersAdded(count: number, group: UserGroup) { const properties: GroupUsersAddedEvent = { count, groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, @@ -57,6 +61,7 @@ async function usersDeleted(count: number, group: UserGroup) { const properties: GroupUsersDeletedEvent = { count, groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, diff --git a/packages/backend-core/src/events/publishers/user.ts b/packages/backend-core/src/events/publishers/user.ts index 8dbc494d1e..0d08c0a759 100644 --- a/packages/backend-core/src/events/publishers/user.ts +++ b/packages/backend-core/src/events/publishers/user.ts @@ -15,10 +15,12 @@ import { UserUpdatedEvent, UserOnboardingEvent, } from "@budibase/types" +import { isScim } from "../../context" async function created(user: User, timestamp?: number) { const properties: UserCreatedEvent = { userId: user._id as string, + viaScim: isScim(), audited: { email: user.email, }, @@ -29,6 +31,7 @@ async function created(user: User, timestamp?: number) { async function updated(user: User) { const properties: UserUpdatedEvent = { userId: user._id as string, + viaScim: isScim(), audited: { email: user.email, }, @@ -39,6 +42,7 @@ async function updated(user: User) { async function deleted(user: User) { const properties: UserDeletedEvent = { userId: user._id as string, + viaScim: isScim(), audited: { email: user.email, }, diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 8a97319586..f877985ee0 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -96,9 +96,15 @@ export default function ( } try { // check the actual user is authenticated first, try header or cookie - const headerToken = ctx.request.headers[Header.TOKEN] + let headerToken = ctx.request.headers[Header.TOKEN] + const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken) - const apiKey = ctx.request.headers[Header.API_KEY] + let apiKey = ctx.request.headers[Header.API_KEY] + + if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) { + apiKey = ctx.request.headers[Header.AUTHORIZATION].split(" ")[1] + } + const tenantId = ctx.request.headers[Header.TENANT_ID] let authenticated = false, user = null, diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index dfc544c3ed..c7d8a94e95 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -8,8 +8,10 @@ import { DocumentType, SEPARATOR, directCouchFind, + getGlobalUserParams, + pagination, } from "./db" -import { BulkDocsResponse, User } from "@budibase/types" +import { BulkDocsResponse, SearchUsersRequest, User } from "@budibase/types" import { getGlobalDB } from "./context" import * as context from "./context" @@ -199,3 +201,41 @@ export const searchGlobalUsersByEmail = async ( } return users } + +const PAGE_LIMIT = 8 +export const paginatedUsers = async ({ + page, + email, + appId, +}: SearchUsersRequest = {}) => { + const db = getGlobalDB() + // get one extra document, to have the next page + const opts: any = { + include_docs: true, + limit: PAGE_LIMIT + 1, + } + // add a startkey if the page was specified (anchor) + if (page) { + opts.startkey = page + } + // property specifies what to use for the page/anchor + let userList: User[], + property = "_id", + getKey + if (appId) { + userList = await searchGlobalUsersByApp(appId, opts) + getKey = (doc: any) => getGlobalUserByAppPage(appId, doc) + } else if (email) { + userList = await searchGlobalUsersByEmail(email, opts) + property = "email" + } else { + // no search, query allDocs + const response = await db.allDocs(getGlobalUserParams(null, opts)) + userList = response.rows.map((row: any) => row.doc) + } + return pagination(userList, PAGE_LIMIT, { + paginate: true, + property, + getKey, + }) +} diff --git a/packages/backend-core/tests/utilities/mocks/licenses.ts b/packages/backend-core/tests/utilities/mocks/licenses.ts index 2ca41616e4..839b22e5f9 100644 --- a/packages/backend-core/tests/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/utilities/mocks/licenses.ts @@ -86,6 +86,10 @@ export const useAuditLogs = () => { return useFeature(Feature.AUDIT_LOGS) } +export const useScimIntegration = () => { + return useFeature(Feature.SCIM) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index ff2e5b147f..5592a7e1f9 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -10,3 +10,4 @@ export * as tenant from "./tenants" export * as users from "./users" export * as userGroups from "./userGroups" export { generator } from "./generator" +export * as scim from "./scim" diff --git a/packages/backend-core/tests/utilities/structures/scim.ts b/packages/backend-core/tests/utilities/structures/scim.ts new file mode 100644 index 0000000000..6657bb90b5 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/scim.ts @@ -0,0 +1,69 @@ +import { ScimCreateGroupRequest, ScimCreateUserRequest } from "@budibase/types" +import { uuid } from "./common" +import { generator } from "./generator" + +export function createUserRequest(userData?: { + externalId?: string + email?: string + firstName?: string + lastName?: string + username?: string +}) { + const { + externalId = uuid(), + email = generator.email(), + firstName = generator.first(), + lastName = generator.last(), + username = generator.name(), + } = userData || {} + + const user: ScimCreateUserRequest = { + schemas: [ + "urn:ietf:params:scim:schemas:core:2.0:User", + "urn:ietf:params:scim:schemas:extension:enterprise:2.0:User", + ], + externalId, + userName: username, + active: true, + emails: [ + { + primary: true, + type: "work", + value: email, + }, + ], + meta: { + resourceType: "User", + }, + name: { + formatted: generator.name(), + familyName: lastName, + givenName: firstName, + }, + roles: [], + } + return user +} + +export function createGroupRequest(groupData?: { + externalId?: string + displayName?: string +}) { + const { externalId = uuid(), displayName = generator.word() } = + groupData || {} + + const group: ScimCreateGroupRequest = { + schemas: [ + "urn:ietf:params:scim:schemas:core:2.0:Group", + "http://schemas.microsoft.com/2006/11/ResourceManagement/ADSCIM/2.0/Group", + ], + externalId: externalId, + displayName: displayName, + meta: { + resourceType: "Group", + created: new Date(), + lastModified: new Date(), + }, + } + return group +} diff --git a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte index 2d4dc7ee46..60f84049a3 100644 --- a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte @@ -27,6 +27,7 @@ import { onMount } from "svelte" import { API } from "api" import { organisation, admin, licensing } from "stores/portal" + import Scim from "./scim.svelte" const ConfigTypes = { Google: "google", @@ -606,12 +607,17 @@ +
{/if} + {#if $licensing.scimEnabled} + + + {/if} diff --git a/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte b/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte new file mode 100644 index 0000000000..2d02214800 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte @@ -0,0 +1,15 @@ + + +
+ + Users are synced from your AD +
+ + diff --git a/packages/builder/src/pages/builder/portal/users/_layout.svelte b/packages/builder/src/pages/builder/portal/users/_layout.svelte index db598c4e43..d53da89613 100644 --- a/packages/builder/src/pages/builder/portal/users/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/users/_layout.svelte @@ -1,12 +1,23 @@ diff --git a/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte b/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte index a45414d307..bfb60f1c72 100644 --- a/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte +++ b/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte @@ -14,7 +14,7 @@ } from "@budibase/bbui" import UserGroupPicker from "components/settings/UserGroupPicker.svelte" import { createPaginationStore } from "helpers/pagination" - import { users, apps, groups, auth } from "stores/portal" + import { users, apps, groups, auth, features } from "stores/portal" import { onMount, setContext } from "svelte" import { roles } from "stores/backend" import ConfirmDialog from "components/common/ConfirmDialog.svelte" @@ -24,18 +24,23 @@ import AppNameTableRenderer from "../users/_components/AppNameTableRenderer.svelte" import RemoveUserTableRenderer from "./_components/RemoveUserTableRenderer.svelte" import AppRoleTableRenderer from "../users/_components/AppRoleTableRenderer.svelte" + import ScimBanner from "../_components/SCIMBanner.svelte" export let groupId - const userSchema = { + $: userSchema = { email: { width: "1fr", }, - _id: { - displayName: "", - width: "auto", - borderLeft: true, - }, + ...(readonly + ? {} + : { + _id: { + displayName: "", + width: "auto", + borderLeft: true, + }, + }), } const appSchema = { name: { @@ -70,7 +75,9 @@ let loaded = false let editModal, deleteModal - $: readonly = !$auth.isAdmin + const scimEnabled = $features.isScimEnabled + + $: readonly = !$auth.isAdmin || scimEnabled $: page = $pageInfo.page $: fetchUsers(page, searchTerm) $: group = $groups.find(x => x._id === groupId) @@ -182,11 +189,15 @@
Users -
- -
+ {#if !scimEnabled} +
+ +
+ {:else} + + {/if} {#if $licensing.groupsEnabled} - - + {#if !$features.isScimEnabled} + + + {:else} + + {/if} {:else}