From 635af0f76aec1c8da2139701ecd3ab99f606a38e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 6 Nov 2023 19:03:37 +0000 Subject: [PATCH 01/12] Getting external rows to be more consistent with the internal API - the formulas should be processed using the outputProcessing as they were being processed, but was causing an overwrite. The problem was that the external system internally processed the formulas/relationships, then squashed the relationships. Once it got to the external API, it would go through normal output processing, which would run over the squashed rows, causing an inconsistent behaviour. --- .../src/api/controllers/deploy/index.ts | 1 - .../api/controllers/row/ExternalRequest.ts | 33 ++++---- .../src/api/controllers/row/external.ts | 14 +++- .../src/api/controllers/row/staticFormula.ts | 14 ++-- packages/server/src/db/linkedRows/index.ts | 1 - .../server/src/db/linkedRows/linkUtils.ts | 4 +- .../src/sdk/app/rows/search/external.ts | 5 +- .../src/utilities/rowProcessor/index.ts | 2 +- .../src/utilities/rowProcessor/utils.ts | 76 ++++++++++--------- 9 files changed, 79 insertions(+), 71 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 66439d3411..2cf3da3dda 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -106,7 +106,6 @@ export async function fetchDeployments(ctx: any) { } ctx.body = Object.values(deployments.history).reverse() } catch (err) { - console.error(err) ctx.body = [] } } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index c3c5468840..7c98fecb9b 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -23,7 +23,6 @@ import { breakRowIdField, convertRowId, generateRowIdField, - getPrimaryDisplay, isRowId, isSQL, } from "../../../integrations/utils" @@ -237,7 +236,7 @@ function basicProcessing({ thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" - return processFormulas(table, thisRow) + return thisRow } function fixArrayTypes(row: Row, table: Table) { @@ -392,7 +391,7 @@ export class ExternalRequest { return { row: newRow, manyRelationships } } - squashRelationshipColumns( + processRelationshipFields( table: Table, row: Row, relationships: RelationshipsJson[] @@ -402,7 +401,6 @@ export class ExternalRequest { if (!linkedTable || !row[relationship.column]) { continue } - const display = linkedTable.primaryDisplay for (let key of Object.keys(row[relationship.column])) { let relatedRow: Row = row[relationship.column][key] // add this row as context for the relationship @@ -411,15 +409,10 @@ export class ExternalRequest { relatedRow[col.name] = [row] } } + // process additional types + relatedRow = processDates(table, relatedRow) relatedRow = processFormulas(linkedTable, relatedRow) - let relatedDisplay - if (display) { - relatedDisplay = getPrimaryDisplay(relatedRow[display]) - } - row[relationship.column][key] = { - primaryDisplay: relatedDisplay || "Invalid display column", - _id: relatedRow._id, - } + row[relationship.column][key] = relatedRow } } return row @@ -521,14 +514,14 @@ export class ExternalRequest { ) } - // Process some additional data types - let finalRowArray = Object.values(finalRows) - finalRowArray = processDates(table, finalRowArray) - finalRowArray = processFormulas(table, finalRowArray) as Row[] - - return finalRowArray.map((row: Row) => - this.squashRelationshipColumns(table, row, relationships) + // make sure all related rows are correct + let finalRowArray = Object.values(finalRows).map(row => + this.processRelationshipFields(table, row, relationships) ) + + // process some additional types + finalRowArray = processDates(table, finalRowArray) + return finalRowArray } /** @@ -663,7 +656,7 @@ export class ExternalRequest { linkPrimary, linkSecondary, }: { - row: { [key: string]: any } + row: Row linkPrimary: string linkSecondary?: string }) { diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 0515b6b97e..287b2ae6aa 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -76,6 +76,7 @@ export async function patch(ctx: UserCtx) { relationships: true, }) const enrichedRow = await outputProcessing(table, row, { + squash: true, preserveLinks: true, }) return { @@ -119,7 +120,10 @@ export async function save(ctx: UserCtx) { }) return { ...response, - row: await outputProcessing(table, row, { preserveLinks: true }), + row: await outputProcessing(table, row, { + preserveLinks: true, + squash: true, + }), } } else { return response @@ -140,7 +144,7 @@ export async function find(ctx: UserCtx): Promise { const table = await sdk.tables.getTable(tableId) // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case return await outputProcessing(table, row, { - squash: false, + squash: true, preserveLinks: true, }) } @@ -207,7 +211,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { // don't support composite keys right now const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0]) const primaryLink = linkedTable.primary?.[0] as string - row[fieldName] = await handleRequest(Operation.READ, linkedTableId!, { + const relatedRows = await handleRequest(Operation.READ, linkedTableId!, { tables, filters: { oneOf: { @@ -216,6 +220,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) { }, includeSqlRelationships: IncludeRelationship.INCLUDE, }) + row[fieldName] = await outputProcessing(linkedTable, relatedRows, { + squash: true, + preserveLinks: true, + }) } return row } diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 6f426c6fa0..87cbdad23f 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) { const db = context.getAppDB() // start by getting the raw rows (which will be written back to DB after update) let rows = ( - await db.allDocs( + await db.allDocs( getRowParams(table._id, null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc as Row) // now enrich the rows, note the clone so that we have the base state of the // rows so that we don't write any of the enriched information back let enrichedRows = await outputProcessing(table, cloneDeep(rows), { @@ -101,12 +101,12 @@ export async function updateAllFormulasInTable(table: Table) { for (let row of rows) { // find the enriched row, if found process the formulas const enrichedRow = enrichedRows.find( - (enriched: any) => enriched._id === row._id + (enriched: Row) => enriched._id === row._id ) if (enrichedRow) { const processed = processFormulas(table, cloneDeep(row), { dynamic: false, - contextRows: enrichedRow, + contextRows: [enrichedRow], }) // values have changed, need to add to bulk docs to update if (!isEqual(processed, row)) { @@ -139,7 +139,7 @@ export async function finaliseRow( // use enriched row to generate formulas for saving, specifically only use as context row = processFormulas(table, row, { dynamic: false, - contextRows: enrichedRow, + contextRows: [enrichedRow], }) // don't worry about rev, tables handle rev/lastID updates // if another row has been written since processing this will @@ -163,7 +163,9 @@ export async function finaliseRow( const response = await db.put(row) // for response, calculate the formulas for the enriched row enrichedRow._rev = response.rev - enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false }) + enrichedRow = processFormulas(table, enrichedRow, { + dynamic: false, + }) // this updates the related formulas in other rows based on the relations to this row if (updateFormula) { await updateRelatedFormula(table, enrichedRow) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7a7a06551e..0dba773ad7 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -2,7 +2,6 @@ import LinkController from "./LinkController" import { IncludeDocs, getLinkDocuments, - createLinkView, getUniqueByProp, getRelatedTableForField, getLinkedTableIDs, diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index db9a0dc7d5..5942e7e5a1 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -8,6 +8,7 @@ import { LinkDocumentValue, Table, } from "@budibase/types" +import sdk from "../../sdk" export { createLinkView } from "../views/staticViews" @@ -110,12 +111,11 @@ export function getLinkedTableIDs(table: Table): string[] { } export async function getLinkedTable(id: string, tables: Table[]) { - const db = context.getAppDB() let linkedTable = tables.find(table => table._id === id) if (linkedTable) { return linkedTable } - linkedTable = await db.get(id) + linkedTable = await sdk.tables.getTable(id) if (linkedTable) { tables.push(linkedTable) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 981ae1bf8d..71c12f0a79 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -80,7 +80,10 @@ export async function search(options: SearchParams) { rows = rows.map((r: any) => pick(r, fields)) } - rows = await outputProcessing(table, rows, { preserveLinks: true }) + rows = await outputProcessing(table, rows, { + preserveLinks: true, + squash: true, + }) // need wrapper object for bookmarks etc when paginating return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 098962c646..6070d40f7d 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -258,7 +258,7 @@ export async function outputProcessing( } // process formulas after the complex types had been processed - enriched = processFormulas(table, enriched, { dynamic: true }) as Row[] + enriched = processFormulas(table, enriched, { dynamic: true }) if (opts.squash) { enriched = (await linkRows.squashLinksToPrimaryDisplay( diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 48697af6a9..8d4af29a6b 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -12,6 +12,11 @@ import { Table, } from "@budibase/types" +interface FormulaOpts { + dynamic?: boolean + contextRows?: Row[] +} + /** * If the subtype has been lost for any reason this works out what * subtype the auto column should be. @@ -40,52 +45,50 @@ export function fixAutoColumnSubType( /** * Looks through the rows provided and finds formulas - which it then processes. */ -export function processFormulas( +export function processFormulas( table: Table, - rows: Row[] | Row, - { dynamic, contextRows }: any = { dynamic: true } -) { - const single = !Array.isArray(rows) - let rowArray: Row[] - if (single) { - rowArray = [rows] - contextRows = contextRows ? [contextRows] : contextRows - } else { - rowArray = rows - } - for (let [column, schema] of Object.entries(table.schema)) { - if (schema.type !== FieldTypes.FORMULA) { - continue - } + inputRows: T, + { dynamic, contextRows }: FormulaOpts = { dynamic: true } +): Promise { + const rows = Array.isArray(inputRows) ? inputRows : [inputRows] + if (rows) + for (let [column, schema] of Object.entries(table.schema)) { + if (schema.type !== FieldTypes.FORMULA) { + continue + } - const isStatic = schema.formulaType === FormulaTypes.STATIC + const isStatic = schema.formulaType === FormulaTypes.STATIC - if ( - schema.formula == null || - (dynamic && isStatic) || - (!dynamic && !isStatic) - ) { - continue - } - // iterate through rows and process formula - for (let i = 0; i < rowArray.length; i++) { - let row = rowArray[i] - let context = contextRows ? contextRows[i] : row - rowArray[i] = { - ...row, - [column]: processStringSync(schema.formula, context), + if ( + schema.formula == null || + (dynamic && isStatic) || + (!dynamic && !isStatic) + ) { + continue + } + // iterate through rows and process formula + for (let i = 0; i < rows.length; i++) { + let row = rows[i] + let context = contextRows ? contextRows[i] : row + rows[i] = { + ...row, + [column]: processStringSync(schema.formula, context), + } } } - } - return single ? rowArray[0] : rowArray + return Array.isArray(inputRows) ? rows : rows[0] } /** * Processes any date columns and ensures that those without the ignoreTimezones * flag set are parsed as UTC rather than local time. */ -export function processDates(table: Table, rows: Row[]) { - let datesWithTZ = [] +export function processDates( + table: Table, + inputRows: T +): T { + let rows = Array.isArray(inputRows) ? inputRows : [inputRows] + let datesWithTZ: string[] = [] for (let [column, schema] of Object.entries(table.schema)) { if (schema.type !== FieldTypes.DATETIME) { continue @@ -102,5 +105,6 @@ export function processDates(table: Table, rows: Row[]) { } } } - return rows + + return Array.isArray(inputRows) ? rows : rows[0] } From 49b00ee7eeb23717b5f075064b021e0aa74e6429 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 7 Nov 2023 11:07:25 +0000 Subject: [PATCH 02/12] Fixing build issue. --- packages/server/src/utilities/rowProcessor/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 8d4af29a6b..9eb725dd7c 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -49,7 +49,7 @@ export function processFormulas( table: Table, inputRows: T, { dynamic, contextRows }: FormulaOpts = { dynamic: true } -): Promise { +): T { const rows = Array.isArray(inputRows) ? inputRows : [inputRows] if (rows) for (let [column, schema] of Object.entries(table.schema)) { From fdae7ab913ec22e7ce22ff01e3356c13e1d53d05 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 7 Nov 2023 13:37:45 +0000 Subject: [PATCH 03/12] Fixing issues discovered by tests. --- packages/server/src/integration-test/postgres.spec.ts | 2 -- packages/server/src/sdk/app/rows/search/external.ts | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 60416853b3..8dc49a9489 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -923,7 +923,6 @@ describe("postgres integrations", () => { [m2mFieldName]: [ { _id: row._id, - primaryDisplay: "Invalid display column", }, ], }) @@ -932,7 +931,6 @@ describe("postgres integrations", () => { [m2mFieldName]: [ { _id: row._id, - primaryDisplay: "Invalid display column", }, ], }) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 71c12f0a79..1edd6b30f2 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -186,6 +186,7 @@ export async function fetch(tableId: string): Promise { const table = await sdk.tables.getTable(tableId) return await outputProcessing(table, response, { preserveLinks: true, + squash: true, }) } From 312415ca7d1e99f71d3442f0b9674b1f73d8b349 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 7 Nov 2023 13:47:21 +0000 Subject: [PATCH 04/12] Enforcing squash for rows which contain circular structures. --- packages/backend-core/src/utils/utils.ts | 14 +++++++++++++ .../src/utilities/rowProcessor/index.ts | 20 +++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index b92471a7a4..1c1ca8473b 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -237,3 +237,17 @@ export function timeout(timeMs: number) { export function isAudited(event: Event) { return !!AuditedEventFriendlyName[event] } + +export function hasCircularStructure(json: any) { + if (typeof json !== "object") { + return false + } + try { + JSON.stringify(json) + } catch (err) { + if (err instanceof Error && err?.message.includes("circular structure")) { + return true + } + } + return false +} diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 6070d40f7d..0e53422a4f 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -2,16 +2,15 @@ import * as linkRows from "../../db/linkedRows" import { FieldTypes, AutoFieldSubTypes } from "../../constants" import { processFormulas, fixAutoColumnSubType } from "./utils" import { ObjectStoreBuckets } from "../../constants" -import { context, db as dbCore, objectStore } from "@budibase/backend-core" +import { + context, + db as dbCore, + objectStore, + utils, +} from "@budibase/backend-core" import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" -import { - AutoColumnFieldMetadata, - FieldSubtype, - Row, - RowAttachment, - Table, -} from "@budibase/types" +import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types" import { cloneDeep } from "lodash/fp" import { processInputBBReferences, @@ -233,6 +232,11 @@ export async function outputProcessing( }) : safeRows + // make sure squash is enabled if needed + if (!opts.squash && utils.hasCircularStructure(rows)) { + opts.squash = true + } + // process complex types: attachements, bb references... for (let [property, column] of Object.entries(table.schema)) { if (column.type === FieldTypes.ATTACHMENT) { From 8d35453f015b53d4f9a159ea6a494bfb227cd702 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 7 Nov 2023 13:56:42 +0000 Subject: [PATCH 05/12] Adding test case for circular detection. --- packages/backend-core/src/utils/tests/utils.spec.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/backend-core/src/utils/tests/utils.spec.ts b/packages/backend-core/src/utils/tests/utils.spec.ts index 5a0ac4f283..7b411e801c 100644 --- a/packages/backend-core/src/utils/tests/utils.spec.ts +++ b/packages/backend-core/src/utils/tests/utils.spec.ts @@ -188,4 +188,17 @@ describe("utils", () => { expectResult(false) }) }) + + describe("hasCircularStructure", () => { + it("should detect a circular structure", () => { + const a: any = { b: "b" } + const b = { a } + a.b = b + expect(utils.hasCircularStructure(b)).toBe(true) + }) + + it("should allow none circular structures", () => { + expect(utils.hasCircularStructure({ a: "b" })).toBe(false) + }) + }) }) From 68a3e030c2df46f81975ea81af9c3919bb6de0f6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 7 Nov 2023 18:34:51 +0000 Subject: [PATCH 06/12] Adding test case to check that relationships can be used in external table formulas. --- .../server/src/api/routes/tests/row.spec.ts | 47 +++++++++++++++++++ .../server/src/tests/utilities/api/row.ts | 3 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 48f7ab4f09..97b06f33c8 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -10,6 +10,7 @@ import { FieldSchema, FieldType, FieldTypeSubtypes, + FormulaTypes, INTERNAL_TABLE_SOURCE_ID, MonthlyQuotaName, PermissionLevel, @@ -1998,4 +1999,50 @@ describe.each([ }) }) }) + + describe("Formula fields", () => { + let relationshipTable: Table, tableId: string, relatedRow: Row + + beforeAll(async () => { + const otherTableId = config.table!._id! + const cfg = generateTableConfig() + relationshipTable = await config.createLinkedTable( + RelationshipType.ONE_TO_MANY, + ["links"], + { + ...cfg, + schema: { + ...cfg.schema, + formula: { + name: "formula", + type: FieldType.FORMULA, + formula: "{{ links.0.name }}", + formulaType: FormulaTypes.DYNAMIC, + }, + }, + } + ) + + tableId = relationshipTable._id! + + relatedRow = await config.api.row.save(otherTableId, { + name: generator.word(), + description: generator.paragraph(), + }) + await config.api.row.save(tableId, { + name: generator.word(), + description: generator.paragraph(), + tableId, + links: [relatedRow._id], + }) + }) + + it("should be able to search for rows containing formulas", async () => { + const { rows } = await config.api.row.search(tableId) + expect(rows.length).toBe(1) + expect(rows[0].links.length).toBe(1) + const row = rows[0] + expect(row.formula).toBe(relatedRow.name) + }) + }) }) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 20b1d6f9ee..3d4cf6c82c 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -6,6 +6,7 @@ import { ExportRowsRequest, BulkImportRequest, BulkImportResponse, + SearchRowResponse, } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" @@ -154,7 +155,7 @@ export class RowAPI extends TestAPI { search = async ( sourceId: string, { expectStatus } = { expectStatus: 200 } - ): Promise => { + ): Promise => { const request = this.request .post(`/api/${sourceId}/search`) .set(this.config.defaultHeaders()) From 0633a3de65922beeea097d933706406bc125e642 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 8 Nov 2023 10:03:22 +0000 Subject: [PATCH 07/12] WIP --- packages/server/src/api/controllers/auth.ts | 2 +- packages/server/src/api/controllers/user.ts | 6 +-- .../tests/{user.spec.js => user.spec.ts} | 27 ++++++----- .../server/src/tests/utilities/api/index.ts | 3 ++ .../server/src/tests/utilities/api/user.ts | 48 +++++++++++++++++++ packages/server/src/utilities/users.ts | 12 +++-- packages/types/src/api/web/app/index.ts | 1 + packages/types/src/api/web/app/user.ts | 3 ++ 8 files changed, 81 insertions(+), 21 deletions(-) rename packages/server/src/api/routes/tests/{user.spec.js => user.spec.ts} (89%) create mode 100644 packages/server/src/tests/utilities/api/user.ts create mode 100644 packages/types/src/api/web/app/user.ts diff --git a/packages/server/src/api/controllers/auth.ts b/packages/server/src/api/controllers/auth.ts index eabfe10bab..9b1b78ed9e 100644 --- a/packages/server/src/api/controllers/auth.ts +++ b/packages/server/src/api/controllers/auth.ts @@ -26,7 +26,7 @@ export async function fetchSelf(ctx: UserCtx) { } const appId = context.getAppId() - let user: ContextUser = await getFullUser(ctx, userId) + let user: ContextUser = await getFullUser(userId) // this shouldn't be returned by the app self delete user.roles // forward the csrf token from the session diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index b6c3e7c6bd..bfe206e98a 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -1,10 +1,10 @@ import { generateUserFlagID, InternalTables } from "../../db/utils" import { getFullUser } from "../../utilities/users" import { context } from "@budibase/backend-core" -import { Ctx, UserCtx } from "@budibase/types" +import { Ctx, FetchUserMetadataResponse, UserCtx } from "@budibase/types" import sdk from "../../sdk" -export async function fetchMetadata(ctx: Ctx) { +export async function fetchMetadata(ctx: Ctx) { ctx.body = await sdk.users.fetchMetadata() } @@ -44,7 +44,7 @@ export async function destroyMetadata(ctx: UserCtx) { } export async function findMetadata(ctx: UserCtx) { - ctx.body = await getFullUser(ctx, ctx.params.id) + ctx.body = await getFullUser(ctx.params.id) } export async function setFlag(ctx: UserCtx) { diff --git a/packages/server/src/api/routes/tests/user.spec.js b/packages/server/src/api/routes/tests/user.spec.ts similarity index 89% rename from packages/server/src/api/routes/tests/user.spec.js rename to packages/server/src/api/routes/tests/user.spec.ts index e8ffd8df2b..caae54bc68 100644 --- a/packages/server/src/api/routes/tests/user.spec.js +++ b/packages/server/src/api/routes/tests/user.spec.ts @@ -1,7 +1,6 @@ -const { roles, utils } = require("@budibase/backend-core") -const { checkPermissionsEndpoint } = require("./utilities/TestFunctions") -const setup = require("./utilities") -const { BUILTIN_ROLE_IDS } = roles +import { roles, utils } from "@budibase/backend-core" +import { checkPermissionsEndpoint } from "./utilities/TestFunctions" +import * as setup from "./utilities" jest.setTimeout(30000) @@ -47,8 +46,8 @@ describe("/users", () => { request, method: "GET", url: `/api/users/metadata`, - passRole: BUILTIN_ROLE_IDS.ADMIN, - failRole: BUILTIN_ROLE_IDS.PUBLIC, + passRole: roles.BUILTIN_ROLE_IDS.ADMIN, + failRole: roles.BUILTIN_ROLE_IDS.PUBLIC, }) }) }) @@ -56,7 +55,7 @@ describe("/users", () => { describe("update", () => { it("should be able to update the user", async () => { const user = await config.createUser({ id: `us_update${utils.newid()}` }) - user.roleId = BUILTIN_ROLE_IDS.BASIC + user.roleId = roles.BUILTIN_ROLE_IDS.BASIC delete user._rev const res = await request .put(`/api/users/metadata`) @@ -74,14 +73,18 @@ describe("/users", () => { const res1 = await request .put(`/api/users/metadata`) .set(config.defaultHeaders()) - .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC }) + .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.BASIC }) .expect(200) .expect("Content-Type", /json/) const res = await request .put(`/api/users/metadata`) .set(config.defaultHeaders()) - .send({ ...user, _rev: res1.body.rev, roleId: BUILTIN_ROLE_IDS.POWER }) + .send({ + ...user, + _rev: res1.body.rev, + roleId: roles.BUILTIN_ROLE_IDS.POWER, + }) .expect(200) .expect("Content-Type", /json/) @@ -95,14 +98,14 @@ describe("/users", () => { await request .put(`/api/users/metadata`) .set(config.defaultHeaders()) - .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC }) + .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.BASIC }) .expect(200) .expect("Content-Type", /json/) await request .put(`/api/users/metadata`) .set(config.defaultHeaders()) - .send({ ...user, roleId: BUILTIN_ROLE_IDS.POWER }) + .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER }) .expect(409) .expect("Content-Type", /json/) }) @@ -129,7 +132,7 @@ describe("/users", () => { .expect(200) .expect("Content-Type", /json/) expect(res.body._id).toEqual(user._id) - expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.ADMIN) + expect(res.body.roleId).toEqual(roles.BUILTIN_ROLE_IDS.ADMIN) expect(res.body.tableId).toBeDefined() }) }) diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index c553e7b8f4..20b96f7a99 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -9,6 +9,7 @@ import { ScreenAPI } from "./screen" import { ApplicationAPI } from "./application" import { BackupAPI } from "./backup" import { AttachmentAPI } from "./attachment" +import { UserAPI } from "./user" export default class API { table: TableAPI @@ -21,6 +22,7 @@ export default class API { application: ApplicationAPI backup: BackupAPI attachment: AttachmentAPI + user: UserAPI constructor(config: TestConfiguration) { this.table = new TableAPI(config) @@ -33,5 +35,6 @@ export default class API { this.application = new ApplicationAPI(config) this.backup = new BackupAPI(config) this.attachment = new AttachmentAPI(config) + this.user = new UserAPI(config) } } diff --git a/packages/server/src/tests/utilities/api/user.ts b/packages/server/src/tests/utilities/api/user.ts new file mode 100644 index 0000000000..2066315778 --- /dev/null +++ b/packages/server/src/tests/utilities/api/user.ts @@ -0,0 +1,48 @@ +import { FetchUserMetadataResponse } from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" + +export class UserAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + fetch = async ( + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .get(`/api/users/metadata`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body + } + + get = async ( + id: string, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .get(`/api/users/metadata/${id}`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body + } +} diff --git a/packages/server/src/utilities/users.ts b/packages/server/src/utilities/users.ts index bbc1370355..73b2f48b15 100644 --- a/packages/server/src/utilities/users.ts +++ b/packages/server/src/utilities/users.ts @@ -1,11 +1,13 @@ import { InternalTables } from "../db/utils" import { getGlobalUser } from "./global" import { context, roles } from "@budibase/backend-core" -import { UserCtx } from "@budibase/types" +import { ContextUserMetadata, UserCtx, UserMetadata } from "@budibase/types" -export async function getFullUser(ctx: UserCtx, userId: string) { +export async function getFullUser( + userId: string +): Promise { const global = await getGlobalUser(userId) - let metadata: any = {} + let metadata: UserMetadata | undefined = undefined // always prefer the user metadata _id and _rev delete global._id @@ -14,11 +16,11 @@ export async function getFullUser(ctx: UserCtx, userId: string) { try { // this will throw an error if the db doesn't exist, or there is no appId const db = context.getAppDB() - metadata = await db.get(userId) + metadata = await db.get(userId) + delete metadata.csrfToken } catch (err) { // it is fine if there is no user metadata yet } - delete metadata.csrfToken return { ...metadata, ...global, diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index f5b876009b..cb1cea2b08 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -6,3 +6,4 @@ export * from "./rows" export * from "./table" export * from "./permission" export * from "./attachment" +export * from "./user" diff --git a/packages/types/src/api/web/app/user.ts b/packages/types/src/api/web/app/user.ts new file mode 100644 index 0000000000..2e0eeee7f8 --- /dev/null +++ b/packages/types/src/api/web/app/user.ts @@ -0,0 +1,3 @@ +import { ContextUserMetadata } from "src/documents" + +export type FetchUserMetadataResponse = ContextUserMetadata[] From d146df5f7367959bccc6d3ce3b02c9098e8e4e4e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 8 Nov 2023 11:53:00 +0000 Subject: [PATCH 08/12] Convert user.spec.js to user.spec.ts --- .../src/api/controllers/row/internal.ts | 3 +- packages/server/src/api/controllers/user.ts | 48 ++++-- .../server/src/api/routes/tests/user.spec.ts | 149 +++++------------- .../src/tests/utilities/TestConfiguration.ts | 2 +- .../server/src/tests/utilities/api/user.ts | 115 +++++++++++++- packages/types/src/api/web/app/user.ts | 8 +- packages/types/src/documents/account/flag.ts | 5 + packages/types/src/documents/account/index.ts | 1 + 8 files changed, 201 insertions(+), 130 deletions(-) create mode 100644 packages/types/src/documents/account/flag.ts diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index d53345a239..fe7d94547a 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -2,7 +2,6 @@ import * as linkRows from "../../../db/linkedRows" import { generateRowID, getMultiIDParams, - getTableIDFromRowID, InternalTables, } from "../../../db/utils" import * as userController from "../user" @@ -89,7 +88,7 @@ export async function patch(ctx: UserCtx) { if (isUserTable) { // the row has been updated, need to put it into the ctx ctx.request.body = row as any - await userController.updateMetadata(ctx) + await userController.updateMetadata(ctx as any) return { row: ctx.body as Row, table } } diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index bfe206e98a..108e29fd3d 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -1,14 +1,26 @@ import { generateUserFlagID, InternalTables } from "../../db/utils" import { getFullUser } from "../../utilities/users" import { context } from "@budibase/backend-core" -import { Ctx, FetchUserMetadataResponse, UserCtx } from "@budibase/types" +import { + ContextUserMetadata, + Ctx, + FetchUserMetadataResponse, + FindUserMetadataResponse, + Flags, + SetFlagRequest, + UserCtx, + UserMetadata, +} from "@budibase/types" import sdk from "../../sdk" +import { DocumentInsertResponse } from "@budibase/nano" -export async function fetchMetadata(ctx: Ctx) { +export async function fetchMetadata(ctx: Ctx) { ctx.body = await sdk.users.fetchMetadata() } -export async function updateSelfMetadata(ctx: UserCtx) { +export async function updateSelfMetadata( + ctx: UserCtx +) { // overwrite the ID with current users ctx.request.body._id = ctx.user?._id // make sure no stale rev @@ -18,19 +30,21 @@ export async function updateSelfMetadata(ctx: UserCtx) { await updateMetadata(ctx) } -export async function updateMetadata(ctx: UserCtx) { +export async function updateMetadata( + ctx: UserCtx +) { const db = context.getAppDB() const user = ctx.request.body - // this isn't applicable to the user - delete user.roles - const metadata = { + const metadata: ContextUserMetadata = { tableId: InternalTables.USER_METADATA, ...user, } + // this isn't applicable to the user + delete metadata.roles ctx.body = await db.put(metadata) } -export async function destroyMetadata(ctx: UserCtx) { +export async function destroyMetadata(ctx: UserCtx) { const db = context.getAppDB() try { const dbUser = await sdk.users.get(ctx.params.id) @@ -43,11 +57,15 @@ export async function destroyMetadata(ctx: UserCtx) { } } -export async function findMetadata(ctx: UserCtx) { +export async function findMetadata( + ctx: UserCtx +) { ctx.body = await getFullUser(ctx.params.id) } -export async function setFlag(ctx: UserCtx) { +export async function setFlag( + ctx: UserCtx +) { const userId = ctx.user?._id const { flag, value } = ctx.request.body if (!flag) { @@ -55,9 +73,9 @@ export async function setFlag(ctx: UserCtx) { } const flagDocId = generateUserFlagID(userId!) const db = context.getAppDB() - let doc + let doc: Flags try { - doc = await db.get(flagDocId) + doc = await db.get(flagDocId) } catch (err) { doc = { _id: flagDocId } } @@ -66,13 +84,13 @@ export async function setFlag(ctx: UserCtx) { ctx.body = { message: "Flag set successfully" } } -export async function getFlags(ctx: UserCtx) { +export async function getFlags(ctx: UserCtx) { const userId = ctx.user?._id const docId = generateUserFlagID(userId!) const db = context.getAppDB() - let doc + let doc: Flags try { - doc = await db.get(docId) + doc = await db.get(docId) } catch (err) { doc = { _id: docId } } diff --git a/packages/server/src/api/routes/tests/user.spec.ts b/packages/server/src/api/routes/tests/user.spec.ts index caae54bc68..e6349099d7 100644 --- a/packages/server/src/api/routes/tests/user.spec.ts +++ b/packages/server/src/api/routes/tests/user.spec.ts @@ -1,6 +1,7 @@ import { roles, utils } from "@budibase/backend-core" import { checkPermissionsEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" +import { UserMetadata } from "@budibase/types" jest.setTimeout(30000) @@ -28,15 +29,13 @@ describe("/users", () => { it("returns a list of users from an instance db", async () => { await config.createUser({ id: "uuidx" }) await config.createUser({ id: "uuidy" }) - const res = await request - .get(`/api/users/metadata`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body.length).toBe(3) - expect(res.body.find(u => u._id === `ro_ta_users_us_uuidx`)).toBeDefined() - expect(res.body.find(u => u._id === `ro_ta_users_us_uuidy`)).toBeDefined() + const res = await config.api.user.fetch() + expect(res.length).toBe(3) + + const ids = res.map(u => u._id) + expect(ids).toContain(`ro_ta_users_us_uuidx`) + expect(ids).toContain(`ro_ta_users_us_uuidy`) }) it("should apply authorization to endpoint", async () => { @@ -54,86 +53,61 @@ describe("/users", () => { describe("update", () => { it("should be able to update the user", async () => { - const user = await config.createUser({ id: `us_update${utils.newid()}` }) + const user: UserMetadata = await config.createUser({ + id: `us_update${utils.newid()}`, + }) user.roleId = roles.BUILTIN_ROLE_IDS.BASIC delete user._rev - const res = await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send(user) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.ok).toEqual(true) + const res = await config.api.user.update(user) + expect(res.ok).toEqual(true) }) it("should be able to update the user multiple times", async () => { const user = await config.createUser() delete user._rev - const res1 = await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.BASIC }) - .expect(200) - .expect("Content-Type", /json/) - - const res = await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send({ - ...user, - _rev: res1.body.rev, - roleId: roles.BUILTIN_ROLE_IDS.POWER, - }) - .expect(200) - .expect("Content-Type", /json/) - - expect(res.body.ok).toEqual(true) + const res1 = await config.api.user.update({ + ...user, + roleId: roles.BUILTIN_ROLE_IDS.BASIC, + }) + const res2 = await config.api.user.update({ + ...user, + _rev: res1.rev, + roleId: roles.BUILTIN_ROLE_IDS.POWER, + }) + expect(res2.ok).toEqual(true) }) it("should require the _rev field for multiple updates", async () => { const user = await config.createUser() delete user._rev - await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.BASIC }) - .expect(200) - .expect("Content-Type", /json/) - - await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER }) - .expect(409) - .expect("Content-Type", /json/) + await config.api.user.update({ + ...user, + roleId: roles.BUILTIN_ROLE_IDS.BASIC, + }) + await config.api.user.update( + { ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER }, + { expectStatus: 409 } + ) }) }) describe("destroy", () => { it("should be able to delete the user", async () => { const user = await config.createUser() - const res = await request - .delete(`/api/users/metadata/${user._id}`) - .set(config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.message).toBeDefined() + const res = await config.api.user.destroy(user._id!) + expect(res.message).toBeDefined() }) }) describe("find", () => { it("should be able to find the user", async () => { const user = await config.createUser() - const res = await request - .get(`/api/users/metadata/${user._id}`) - .set(config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body._id).toEqual(user._id) - expect(res.body.roleId).toEqual(roles.BUILTIN_ROLE_IDS.ADMIN) - expect(res.body.tableId).toBeDefined() + const res = await config.api.user.find(user._id!) + expect(res._id).toEqual(user._id) + expect(res.roleId).toEqual(roles.BUILTIN_ROLE_IDS.ADMIN) + expect(res.tableId).toBeDefined() }) }) @@ -153,59 +127,18 @@ describe("/users", () => { it("should be able to set a flag on the user", async () => { await config.createUser() - const res = await request - .post(`/api/users/flags`) - .set(config.defaultHeaders()) - .send({ value: "test", flag: "test" }) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.message).toEqual("Flag set successfully") + const res = await config.api.user.setFlag("test", true) + expect(res.message).toEqual("Flag set successfully") }) }) describe("getFlags", () => { it("should get flags for a specific user", async () => { - let flagData = { value: "test", flag: "test" } await config.createUser() - await request - .post(`/api/users/flags`) - .set(config.defaultHeaders()) - .send(flagData) - .expect(200) - .expect("Content-Type", /json/) + await config.api.user.setFlag("test", "test") - const res = await request - .get(`/api/users/flags`) - .set(config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body[flagData.value]).toEqual(flagData.flag) - }) - }) - - describe("setFlag", () => { - it("should throw an error if a flag is not provided", async () => { - await config.createUser() - const res = await request - .post(`/api/users/flags`) - .set(config.defaultHeaders()) - .send({ value: "test" }) - .expect(400) - .expect("Content-Type", /json/) - expect(res.body.message).toEqual( - "Must supply a 'flag' field in request body." - ) - }) - - it("should be able to set a flag on the user", async () => { - await config.createUser() - const res = await request - .post(`/api/users/flags`) - .set(config.defaultHeaders()) - .send({ value: "test", flag: "test" }) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.message).toEqual("Flag set successfully") + const res = await config.api.user.getFlags() + expect(res.test).toEqual("test") }) }) }) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 04c0552457..6877561fcb 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -264,7 +264,7 @@ class TestConfiguration { admin = false, email = this.defaultUserValues.email, roles, - }: any = {}) { + }: any = {}): Promise { const db = tenancy.getTenantDB(this.getTenantId()) let existing try { diff --git a/packages/server/src/tests/utilities/api/user.ts b/packages/server/src/tests/utilities/api/user.ts index 2066315778..2ed23c0461 100644 --- a/packages/server/src/tests/utilities/api/user.ts +++ b/packages/server/src/tests/utilities/api/user.ts @@ -1,6 +1,12 @@ -import { FetchUserMetadataResponse } from "@budibase/types" +import { + FetchUserMetadataResponse, + FindUserMetadataResponse, + Flags, + UserMetadata, +} from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" +import { DocumentInsertResponse } from "@budibase/nano" export class UserAPI extends TestAPI { constructor(config: TestConfiguration) { @@ -26,10 +32,10 @@ export class UserAPI extends TestAPI { return res.body } - get = async ( + find = async ( id: string, { expectStatus } = { expectStatus: 200 } - ): Promise => { + ): Promise => { const res = await this.request .get(`/api/users/metadata/${id}`) .set(this.config.defaultHeaders()) @@ -45,4 +51,107 @@ export class UserAPI extends TestAPI { return res.body } + + update = async ( + user: UserMetadata, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .put(`/api/users/metadata`) + .set(this.config.defaultHeaders()) + .send(user) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as DocumentInsertResponse + } + + updateSelf = async ( + user: UserMetadata, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .post(`/api/users/metadata/self`) + .set(this.config.defaultHeaders()) + .send(user) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as DocumentInsertResponse + } + + destroy = async ( + id: string, + { expectStatus } = { expectStatus: 200 } + ): Promise<{ message: string }> => { + const res = await this.request + .delete(`/api/users/metadata/${id}`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as { message: string } + } + + setFlag = async ( + flag: string, + value: any, + { expectStatus } = { expectStatus: 200 } + ): Promise<{ message: string }> => { + const res = await this.request + .post(`/api/users/flags`) + .set(this.config.defaultHeaders()) + .send({ flag, value }) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as { message: string } + } + + getFlags = async ( + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .get(`/api/users/flags`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as Flags + } } diff --git a/packages/types/src/api/web/app/user.ts b/packages/types/src/api/web/app/user.ts index 2e0eeee7f8..7faec83e9c 100644 --- a/packages/types/src/api/web/app/user.ts +++ b/packages/types/src/api/web/app/user.ts @@ -1,3 +1,9 @@ -import { ContextUserMetadata } from "src/documents" +import { ContextUserMetadata } from "../../../" export type FetchUserMetadataResponse = ContextUserMetadata[] +export type FindUserMetadataResponse = ContextUserMetadata + +export interface SetFlagRequest { + flag: string + value: any +} diff --git a/packages/types/src/documents/account/flag.ts b/packages/types/src/documents/account/flag.ts new file mode 100644 index 0000000000..a214348fe7 --- /dev/null +++ b/packages/types/src/documents/account/flag.ts @@ -0,0 +1,5 @@ +import { Document } from "../../" + +export interface Flags extends Document { + [key: string]: any +} diff --git a/packages/types/src/documents/account/index.ts b/packages/types/src/documents/account/index.ts index 663fb91b58..1e0c800f39 100644 --- a/packages/types/src/documents/account/index.ts +++ b/packages/types/src/documents/account/index.ts @@ -1,2 +1,3 @@ export * from "./account" export * from "./user" +export * from "./flag" From b167fd08e1b4c2f3f1abb02f0583ddc1b78ff6cd Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 8 Nov 2023 13:27:25 +0000 Subject: [PATCH 09/12] Fix issues with automation layout styling --- .../SetupPanel/QueryParamSelector.svelte | 8 +- .../automation/SetupPanel/RowSelector.svelte | 84 ++++++++++--------- .../SetupPanel/RowSelectorTypes.svelte | 1 + .../common/LinkedRowSelector.svelte | 4 +- 4 files changed, 50 insertions(+), 47 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte index 6b3433babc..e68ab1feac 100644 --- a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte @@ -41,8 +41,8 @@ {#if parameters.length} -
- {#each parameters as field} + {#each parameters as field} +
- {/each} -
+
+ {/each} {/if} diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index e4751e0f6a..c4a67ab025 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -114,12 +114,33 @@ {#if schemaFields.length} {#each schemaFields as [field, schema]} - {#if !schema.autocolumn} - {#if schema.type !== "attachment"} -
- -
- {#if isTestModal} + {#if !schema.autocolumn && schema.type !== "attachment"} +
+ +
+ {#if isTestModal} + + {:else} + onChange(e, field)} + {bindings} + allowJS={true} + updateOnChange={false} + drawerLeft="260px" + > - {:else} - onChange(e, field)} - {bindings} - allowJS={true} - updateOnChange={false} - drawerLeft="260px" - > - - - {/if} + + {/if} - {#if isUpdateRow && schema.type === "link"} -
- onChangeSetting(e, field)} - /> -
- {/if} -
+ {#if isUpdateRow && schema.type === "link"} +
+ onChangeSetting(e, field)} + /> +
+ {/if}
- {/if} +
{/if} {/each} {/if} From 8d98da6c5773add2372cb070724571e7622a2126 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 8 Nov 2023 14:57:30 +0000 Subject: [PATCH 11/12] Fixing issue with table names. --- packages/server/src/api/routes/tests/row.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 3598376a76..060f6e46c1 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -2013,6 +2013,8 @@ describe.each([ ["links"], { ...cfg, + // needs to be a short name + name: "b", schema: { ...cfg.schema, formula: { From e4bc6a5fc78448b1c447874a321bf2dd021ec647 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 8 Nov 2023 18:41:15 +0000 Subject: [PATCH 12/12] Attempting to fix QA-core inconsistencies in CI - disabling rate limiting during test runs for public API. --- .../server/src/api/routes/public/index.ts | 72 ++++++++++--------- packages/server/src/environment.ts | 1 + qa-core/package.json | 2 +- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index b37ed931fc..f27f3f8857 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -27,51 +27,59 @@ interface KoaRateLimitOptions { } const PREFIX = "/api/public/v1" -// allow a lot more requests when in test -const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10 -function getApiLimitPerSecond(): number { - if (!env.API_REQ_LIMIT_PER_SEC) { - return DEFAULT_API_REQ_LIMIT_PER_SEC - } - return parseInt(env.API_REQ_LIMIT_PER_SEC) -} +// type can't be known - untyped libraries +let limiter: any, rateLimitStore: any +if (!env.DISABLE_RATE_LIMITING) { + // allow a lot more requests when in test + const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10 -let rateLimitStore: any = null -if (!env.isTest()) { - const { password, host, port } = redis.utils.getRedisConnectionDetails() - let options: KoaRateLimitOptions = { - socket: { - host: host, - port: port, - }, + function getApiLimitPerSecond(): number { + if (!env.API_REQ_LIMIT_PER_SEC) { + return DEFAULT_API_REQ_LIMIT_PER_SEC + } + return parseInt(env.API_REQ_LIMIT_PER_SEC) } - if (password) { - options.password = password - } + if (!env.isTest()) { + const { password, host, port } = redis.utils.getRedisConnectionDetails() + let options: KoaRateLimitOptions = { + socket: { + host: host, + port: port, + }, + } - if (!env.REDIS_CLUSTERED) { - // Can't set direct redis db in clustered env - options.database = SelectableDatabase.RATE_LIMITING + if (password) { + options.password = password + } + + if (!env.REDIS_CLUSTERED) { + // Can't set direct redis db in clustered env + options.database = SelectableDatabase.RATE_LIMITING + } + rateLimitStore = new Stores.Redis(options) + RateLimit.defaultOptions({ + store: rateLimitStore, + }) } - rateLimitStore = new Stores.Redis(options) - RateLimit.defaultOptions({ - store: rateLimitStore, + // rate limiting, allows for 2 requests per second + limiter = RateLimit.middleware({ + interval: { sec: 1 }, + // per ip, per interval + max: getApiLimitPerSecond(), }) +} else { + console.log("**** PUBLIC API RATE LIMITING DISABLED ****") } -// rate limiting, allows for 2 requests per second -const limiter = RateLimit.middleware({ - interval: { sec: 1 }, - // per ip, per interval - max: getApiLimitPerSecond(), -}) const publicRouter = new Router({ prefix: PREFIX, }) -publicRouter.use(limiter) +if (limiter) { + publicRouter.use(limiter) +} function addMiddleware( endpoints: any, diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 91424113ac..c126a61c22 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -61,6 +61,7 @@ const environment = { ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS, + DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING, MULTI_TENANCY: process.env.MULTI_TENANCY, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, SELF_HOSTED: process.env.SELF_HOSTED, diff --git a/qa-core/package.json b/qa-core/package.json index cfccd5e650..87172d2ed9 100644 --- a/qa-core/package.json +++ b/qa-core/package.json @@ -20,7 +20,7 @@ "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.", "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci", "serve": "start-server-and-test dev:built http://localhost:4001/health", - "dev:built": "cd ../ && yarn dev:built" + "dev:built": "cd ../ && DISABLE_RATE_LIMITING=1 yarn dev:built" }, "devDependencies": { "@budibase/types": "^2.3.17",