This improves the performance of views when the enriched relationships feature has been enabled. There was an issue that caused the squashLinks and outputProcessing to loop onto each other until things broke down - this fixes the issue.

packages/server/src/db/linkedRows/index.ts

@@ -10,7 +10,11 @@ import flatten from "lodash/flatten"
 import { USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
-import { outputProcessing, processFormulas } from "../../utilities/rowProcessor"
+import {
+  coreOutputProcessing,
+  outputProcessing,
+  processFormulas,
+} from "../../utilities/rowProcessor"
 import { context, features } from "@budibase/backend-core"
 import {
   ContextUser,
@@ -156,9 +160,6 @@ export async function updateLinks(args: {
 /**
  * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
  * This is required for formula fields, this may only be utilised internally (for now).
- * @param table The table from which the rows originated.
- * @param rows The rows which are to be enriched.
- * @param opts optional - options like passing in a base row to use for enrichment.
  * @return returns the rows with all of the enriched relationships on it.
  */
 export async function attachFullLinkedDocs(
@@ -248,9 +249,6 @@ export type SquashTableFields = Record<string, { visibleFieldNames: string[] }>
 
 /**
  * This function will take the given enriched rows and squash the links to only contain the primary display field.
- * @param table The table from which the rows originated.
- * @param enriched The pre-enriched rows (full docs) which are to be squashed.
- * @param squashFields Per link column (key) define which columns are allowed while squashing.
  * @returns The rows after having their links squashed to only contain the ID and primary display.
  */
 export async function squashLinks<T = Row[] | Row>(
@@ -283,21 +281,18 @@ export async function squashLinks<T = Row[] | Row>(
       if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
         continue
       }
-      const newLinks = []
+      const relatedTable = await getLinkedTable(schema.tableId, linkedTables)
-      for (const link of row[column]) {
+      if (viewSchema[column]?.columns) {
-        const linkTblId =
+        row[column] = await coreOutputProcessing(relatedTable, row[column])
-          link.tableId || getRelatedTableForField(table.schema, column)
+      }
-        const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
+      row[column] = row[column].map((link: Row) => {
         const obj: any = { _id: link._id }
-        obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable)
+        obj.primaryDisplay = getPrimaryDisplayValue(link, relatedTable)
 
         if (viewSchema[column]?.columns) {
-          const enrichedLink = await outputProcessing(linkedTable, link, {
+          const squashFields = Object.entries(viewSchema[column].columns || {})
-            squash: false,
-          })
-          const squashFields = Object.entries(viewSchema[column].columns)
             .filter(([columnName, viewColumnConfig]) => {
-              const tableColumn = linkedTable.schema[columnName]
+              const tableColumn = relatedTable.schema[columnName]
               if (!tableColumn) {
                 return false
               }
@@ -315,13 +310,14 @@ export async function squashLinks<T = Row[] | Row>(
             .map(([columnName]) => columnName)
 
           for (const relField of squashFields) {
-            obj[relField] = enrichedLink[relField]
+            if (link[relField] != null) {
+              obj[relField] = link[relField]
+            }
           }
         }
 
-        newLinks.push(obj)
+        return obj
-      }
+      })
-      row[column] = newLinks
     }
   }
   return (isArray ? enrichedArray : enrichedArray[0]) as T

packages/server/src/utilities/rowProcessor/index.ts

@@ -264,18 +264,46 @@ export async function outputProcessing<T extends Row[] | Row>(
   } else {
     safeRows = rows
   }
+  let enriched: Row[]
+  // SQS returns the rows with full relationship contents
+  if (!(await features.flags.isEnabled("SQS"))) {
     // attach any linked row information
-  let enriched = !opts.preserveLinks
+    enriched = !opts.preserveLinks
       ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
           fromRow: opts?.fromRow,
         })
       : safeRows
+  } else {
+    enriched = cloneDeep(safeRows)
+  }
 
-  // make sure squash is enabled if needed
   if (!opts.squash && utils.hasCircularStructure(rows)) {
     opts.squash = true
   }
 
+  enriched = await coreOutputProcessing(table, enriched, opts)
+
+  if (opts.squash) {
+    enriched = await linkRows.squashLinks(table, enriched, {
+      fromViewId: opts?.fromViewId,
+    })
+  }
+
+  return (wasArray ? enriched : enriched[0]) as T
+}
+
+export async function coreOutputProcessing(
+  table: Table,
+  rows: Row[],
+  opts: {
+    preserveLinks?: boolean
+    skipBBReferences?: boolean
+    fromViewId?: string
+  } = {
+    preserveLinks: false,
+    skipBBReferences: false,
+  }
+): Promise<Row[]> {
   // process complex types: attachments, bb references...
   for (const [property, column] of Object.entries(table.schema)) {
     if (
@@ -283,7 +311,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       column.type === FieldType.ATTACHMENT_SINGLE ||
       column.type === FieldType.SIGNATURE_SINGLE
     ) {
-      for (const row of enriched) {
+      for (const row of rows) {
         if (row[property] == null) {
           continue
         }
@@ -308,7 +336,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE
     ) {
-      for (const row of enriched) {
+      for (const row of rows) {
         row[property] = await processOutputBBReferences(
           row[property],
           column.subtype
@@ -318,14 +346,14 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE_SINGLE
     ) {
-      for (const row of enriched) {
+      for (const row of rows) {
         row[property] = await processOutputBBReference(
           row[property],
           column.subtype
         )
       }
     } else if (column.type === FieldType.DATETIME && column.timeOnly) {
-      for (const row of enriched) {
+      for (const row of rows) {
         if (row[property] instanceof Date) {
           const hours = row[property].getUTCHours().toString().padStart(2, "0")
           const minutes = row[property]
@@ -340,7 +368,7 @@ export async function outputProcessing<T extends Row[] | Row>(
         }
       }
     } else if (column.type === FieldType.LINK) {
-      for (let row of enriched) {
+      for (let row of rows) {
         // if relationship is empty - remove the array, this has been part of the API for some time
         if (Array.isArray(row[property]) && row[property].length === 0) {
           delete row[property]
@@ -350,17 +378,12 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
 
   // process formulas after the complex types had been processed
-  enriched = await processFormulas(table, enriched, { dynamic: true })
+  rows = await processFormulas(table, rows, { dynamic: true })
 
-  if (opts.squash) {
-    enriched = await linkRows.squashLinks(table, enriched, {
-      fromViewId: opts?.fromViewId,
-    })
-  }
   // remove null properties to match internal API
   const isExternal = isExternalTableID(table._id!)
   if (isExternal || (await features.flags.isEnabled("SQS"))) {
-    for (const row of enriched) {
+    for (const row of rows) {
       for (const key of Object.keys(row)) {
         if (row[key] === null) {
           delete row[key]
@@ -388,7 +411,7 @@ export async function outputProcessing<T extends Row[] | Row>(
     const fields = [...tableFields, ...protectedColumns].map(f =>
       f.toLowerCase()
     )
-    for (const row of enriched) {
+    for (const row of rows) {
       for (const key of Object.keys(row)) {
         if (!fields.includes(key.toLowerCase())) {
           delete row[key]
@@ -397,5 +420,5 @@ export async function outputProcessing<T extends Row[] | Row>(
     }
   }
 
-  return (wasArray ? enriched : enriched[0]) as T
+  return rows
 }