From 7c6c12f325a1bf5dfb47d39a719e6c174849a6d4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 Jul 2024 18:13:27 +0100 Subject: [PATCH 1/2] Adding a small utility to DB layer for bulk removing documents, this is a problem that Mel ran into, the fact it doesn't default to throwing errors, I've updated a few cases and added functionality for it to maintain compatiability with the old way of doing things (errors silenced). --- .../backend-core/src/db/couch/DatabaseImpl.ts | 49 +++++++++++++++---- .../backend-core/src/db/instrumentation.ts | 10 ++++ packages/backend-core/src/platform/users.ts | 13 ++--- .../src/db/linkedRows/LinkController.ts | 32 +++--------- packages/server/src/db/linkedRows/index.ts | 6 +-- .../server/src/db/linkedRows/linkUtils.ts | 11 +++++ packages/types/src/sdk/db.ts | 4 ++ 7 files changed, 76 insertions(+), 49 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index f3c3beeaab..98d15b3e7e 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -56,24 +56,24 @@ class CouchDBError extends Error implements DBError { constructor( message: string, info: { - status: number | undefined - statusCode: number | undefined + status?: number | undefined + statusCode?: number | undefined name: string - errid: string - description: string - reason: string - error: string + errid?: string + description?: string + reason?: string + error?: string } ) { super(message) const statusCode = info.status || info.statusCode || 500 this.status = statusCode this.statusCode = statusCode - this.reason = info.reason + this.reason = info.reason || "Unknown" this.name = info.name - this.errid = info.errid - this.description = info.description - this.error = info.error + this.errid = info.errid || "Unknown" + this.description = info.description || "Unknown" + this.error = info.error || "Not found" } } @@ -246,6 +246,35 @@ export class DatabaseImpl implements Database { }) } + async bulkRemove(documents: Document[], opts?: { silenceErrors?: boolean }) { + const response: Nano.DocumentBulkResponse[] = await this.performCall(db => { + return () => + db.bulk({ + docs: documents.map(doc => ({ + ...doc, + _deleted: true, + })), + }) + }) + if (opts?.silenceErrors) { + return + } + let errorFound = false + let errorMessage: string = "Unable to bulk remove documents: " + for (let res of response) { + if (res.error) { + errorFound = true + errorMessage += res.error + } + } + if (errorFound) { + throw new CouchDBError(errorMessage, { + name: this.name, + status: 400, + }) + } + } + async post(document: AnyDocument, opts?: DatabasePutOpts) { if (!document._id) { document._id = newid() diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts index 4e2b147ef3..7026224564 100644 --- a/packages/backend-core/src/db/instrumentation.ts +++ b/packages/backend-core/src/db/instrumentation.ts @@ -71,6 +71,16 @@ export class DDInstrumentedDatabase implements Database { }) } + bulkRemove( + documents: Document[], + opts?: { silenceErrors?: boolean } + ): Promise { + return tracer.trace("db.bulkRemove", span => { + span?.addTags({ db_name: this.name, num_docs: documents.length }) + return this.db.bulkRemove(documents, opts) + }) + } + put( document: AnyDocument, opts?: DatabasePutOpts | undefined diff --git a/packages/backend-core/src/platform/users.ts b/packages/backend-core/src/platform/users.ts index ccaad76b19..93e9df8c0e 100644 --- a/packages/backend-core/src/platform/users.ts +++ b/packages/backend-core/src/platform/users.ts @@ -113,15 +113,12 @@ export async function addUser( export async function removeUser(user: User) { const db = getPlatformDB() const keys = [user._id!, user.email] - const userDocs = await db.allDocs({ + const userDocs = await db.allDocs({ keys, include_docs: true, }) - const toDelete = userDocs.rows.map((row: any) => { - return { - ...row.doc, - _deleted: true, - } - }) - await db.bulkDocs(toDelete) + await db.bulkRemove( + userDocs.rows.map(row => row.doc!), + { silenceErrors: true } + ) } diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts index 7427c9a44d..85a160713b 100644 --- a/packages/server/src/db/linkedRows/LinkController.ts +++ b/packages/server/src/db/linkedRows/LinkController.ts @@ -6,7 +6,6 @@ import { Database, FieldSchema, FieldType, - LinkDocumentValue, RelationshipFieldMetadata, RelationshipType, Row, @@ -213,11 +212,10 @@ class LinkController { linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY ) { let links = ( - (await getLinkDocuments({ + await getLinkDocuments({ tableId: field.tableId, rowId: linkId, - includeDocs: IncludeDocs.EXCLUDE, - })) as LinkDocumentValue[] + }) ).filter( link => link.id !== row._id && link.fieldName === linkedSchema.name @@ -295,13 +293,7 @@ class LinkController { if (linkDocs.length === 0) { return null } - const toDelete = linkDocs.map(doc => { - return { - ...doc, - _deleted: true, - } - }) - await this._db.bulkDocs(toDelete) + await this._db.bulkRemove(linkDocs, { silenceErrors: true }) return row } @@ -321,14 +313,8 @@ class LinkController { : linkDoc.doc2.fieldName return correctFieldName === fieldName }) - await this._db.bulkDocs( - toDelete.map(doc => { - return { - ...doc, - _deleted: true, - } - }) - ) + await this._db.bulkRemove(toDelete, { silenceErrors: true }) + try { // remove schema from other table, if it exists let linkedTable = await this._db.get(field.tableId) @@ -453,13 +439,7 @@ class LinkController { return null } // get link docs for this table and configure for deletion - const toDelete = linkDocs.map(doc => { - return { - ...doc, - _deleted: true, - } - }) - await this._db.bulkDocs(toDelete) + await this._db.bulkRemove(linkDocs, { silenceErrors: true }) return table } } diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 1eb534a7f8..87f980600a 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -1,6 +1,5 @@ import LinkController from "./LinkController" import { - IncludeDocs, getLinkDocuments, getUniqueByProp, getRelatedTableForField, @@ -56,12 +55,9 @@ async function getLinksForRows(rows: Row[]): Promise { const promises = tableIds.map(tableId => getLinkDocuments({ tableId: tableId, - includeDocs: IncludeDocs.EXCLUDE, }) ) - const responses = flatten( - (await Promise.all(promises)) as LinkDocumentValue[][] - ) + const responses = flatten(await Promise.all(promises)) // have to get unique as the previous table query can // return duplicates, could be querying for both tables in a relation return getUniqueByProp( diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index a999871939..c30d62ef35 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -34,6 +34,17 @@ export const IncludeDocs = { * @returns This will return an array of the linking documents that were found * (if any). */ +export function getLinkDocuments(args: { + tableId?: string + rowId?: string + fieldName?: string + includeDocs: boolean +}): Promise +export function getLinkDocuments(args: { + tableId?: string + rowId?: string + fieldName?: string +}): Promise export async function getLinkDocuments(args: { tableId?: string rowId?: string diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 63c37195b7..a081f4f1a2 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -137,6 +137,10 @@ export interface Database { ): Promise remove(idOrDoc: Document): Promise remove(idOrDoc: string, rev?: string): Promise + bulkRemove( + documents: Document[], + opts?: { silenceErrors?: boolean } + ): Promise put( document: AnyDocument, opts?: DatabasePutOpts From 4e0a84f7f80df0f3305d883468a3ded8c788a458 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 25 Jul 2024 10:26:02 +0100 Subject: [PATCH 2/2] PR comment. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 98d15b3e7e..feeba6061e 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -56,8 +56,8 @@ class CouchDBError extends Error implements DBError { constructor( message: string, info: { - status?: number | undefined - statusCode?: number | undefined + status?: number + statusCode?: number name: string errid?: string description?: string