Merge pull request #14239 from Budibase/bulk-remove-utility

Adding bulkRemove utility for CouchDB

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -56,24 +56,24 @@ class CouchDBError extends Error implements DBError {
   constructor(
     message: string,
     info: {
-      status: number | undefined
-      statusCode: number | undefined
+      status?: number
+      statusCode?: number
       name: string
-      errid: string
-      description: string
-      reason: string
-      error: string
+      errid?: string
+      description?: string
+      reason?: string
+      error?: string
     }
   ) {
     super(message)
     const statusCode = info.status || info.statusCode || 500
     this.status = statusCode
     this.statusCode = statusCode
-    this.reason = info.reason
+    this.reason = info.reason || "Unknown"
     this.name = info.name
-    this.errid = info.errid
-    this.description = info.description
-    this.error = info.error
+    this.errid = info.errid || "Unknown"
+    this.description = info.description || "Unknown"
+    this.error = info.error || "Not found"
   }
 }
 
@@ -246,6 +246,35 @@ export class DatabaseImpl implements Database {
     })
   }
 
+  async bulkRemove(documents: Document[], opts?: { silenceErrors?: boolean }) {
+    const response: Nano.DocumentBulkResponse[] = await this.performCall(db => {
+      return () =>
+        db.bulk({
+          docs: documents.map(doc => ({
+            ...doc,
+            _deleted: true,
+          })),
+        })
+    })
+    if (opts?.silenceErrors) {
+      return
+    }
+    let errorFound = false
+    let errorMessage: string = "Unable to bulk remove documents: "
+    for (let res of response) {
+      if (res.error) {
+        errorFound = true
+        errorMessage += res.error
+      }
+    }
+    if (errorFound) {
+      throw new CouchDBError(errorMessage, {
+        name: this.name,
+        status: 400,
+      })
+    }
+  }
+
   async post(document: AnyDocument, opts?: DatabasePutOpts) {
     if (!document._id) {
       document._id = newid()

packages/backend-core/src/db/instrumentation.ts

@@ -71,6 +71,16 @@ export class DDInstrumentedDatabase implements Database {
     })
   }
 
+  bulkRemove(
+    documents: Document[],
+    opts?: { silenceErrors?: boolean }
+  ): Promise<void> {
+    return tracer.trace("db.bulkRemove", span => {
+      span?.addTags({ db_name: this.name, num_docs: documents.length })
+      return this.db.bulkRemove(documents, opts)
+    })
+  }
+
   put(
     document: AnyDocument,
     opts?: DatabasePutOpts | undefined

packages/backend-core/src/platform/users.ts

@@ -113,15 +113,12 @@ export async function addUser(
 export async function removeUser(user: User) {
   const db = getPlatformDB()
   const keys = [user._id!, user.email]
-  const userDocs = await db.allDocs({
+  const userDocs = await db.allDocs<User>({
     keys,
     include_docs: true,
   })
-  const toDelete = userDocs.rows.map((row: any) => {
-    return {
-      ...row.doc,
-      _deleted: true,
-    }
-  })
-  await db.bulkDocs(toDelete)
+  await db.bulkRemove(
+    userDocs.rows.map(row => row.doc!),
+    { silenceErrors: true }
+  )
 }

packages/server/src/db/linkedRows/LinkController.ts

@@ -6,7 +6,6 @@ import {
   Database,
   FieldSchema,
   FieldType,
-  LinkDocumentValue,
   RelationshipFieldMetadata,
   RelationshipType,
   Row,
@@ -213,11 +212,10 @@ class LinkController {
             linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY
           ) {
             let links = (
-              (await getLinkDocuments({
+              await getLinkDocuments({
                 tableId: field.tableId,
                 rowId: linkId,
-                includeDocs: IncludeDocs.EXCLUDE,
-              })) as LinkDocumentValue[]
+              })
             ).filter(
               link =>
                 link.id !== row._id && link.fieldName === linkedSchema.name
@@ -295,13 +293,7 @@ class LinkController {
     if (linkDocs.length === 0) {
       return null
     }
-    const toDelete = linkDocs.map(doc => {
-      return {
-        ...doc,
-        _deleted: true,
-      }
-    })
-    await this._db.bulkDocs(toDelete)
+    await this._db.bulkRemove(linkDocs, { silenceErrors: true })
     return row
   }
 
@@ -321,14 +313,8 @@ class LinkController {
           : linkDoc.doc2.fieldName
       return correctFieldName === fieldName
     })
-    await this._db.bulkDocs(
-      toDelete.map(doc => {
-        return {
-          ...doc,
-          _deleted: true,
-        }
-      })
-    )
+    await this._db.bulkRemove(toDelete, { silenceErrors: true })
+
     try {
       // remove schema from other table, if it exists
       let linkedTable = await this._db.get<Table>(field.tableId)
@@ -453,13 +439,7 @@ class LinkController {
       return null
     }
     // get link docs for this table and configure for deletion
-    const toDelete = linkDocs.map(doc => {
-      return {
-        ...doc,
-        _deleted: true,
-      }
-    })
-    await this._db.bulkDocs(toDelete)
+    await this._db.bulkRemove(linkDocs, { silenceErrors: true })
     return table
   }
 }

packages/server/src/db/linkedRows/index.ts

@@ -1,6 +1,5 @@
 import LinkController from "./LinkController"
 import {
-  IncludeDocs,
   getLinkDocuments,
   getUniqueByProp,
   getRelatedTableForField,
@@ -56,12 +55,9 @@ async function getLinksForRows(rows: Row[]): Promise<LinkDocumentValue[]> {
   const promises = tableIds.map(tableId =>
     getLinkDocuments({
       tableId: tableId,
-      includeDocs: IncludeDocs.EXCLUDE,
     })
   )
-  const responses = flatten(
-    (await Promise.all(promises)) as LinkDocumentValue[][]
-  )
+  const responses = flatten(await Promise.all(promises))
   // have to get unique as the previous table query can
   // return duplicates, could be querying for both tables in a relation
   return getUniqueByProp(

packages/server/src/db/linkedRows/linkUtils.ts

@@ -34,6 +34,17 @@ export const IncludeDocs = {
  * @returns This will return an array of the linking documents that were found
  * (if any).
  */
+export function getLinkDocuments(args: {
+  tableId?: string
+  rowId?: string
+  fieldName?: string
+  includeDocs: boolean
+}): Promise<LinkDocument[]>
+export function getLinkDocuments(args: {
+  tableId?: string
+  rowId?: string
+  fieldName?: string
+}): Promise<LinkDocumentValue[]>
 export async function getLinkDocuments(args: {
   tableId?: string
   rowId?: string

packages/types/src/sdk/db.ts

@@ -137,6 +137,10 @@ export interface Database {
   ): Promise<T[]>
   remove(idOrDoc: Document): Promise<Nano.DocumentDestroyResponse>
   remove(idOrDoc: string, rev?: string): Promise<Nano.DocumentDestroyResponse>
+  bulkRemove(
+    documents: Document[],
+    opts?: { silenceErrors?: boolean }
+  ): Promise<void>
   put(
     document: AnyDocument,
     opts?: DatabasePutOpts