Some work towards improving performance of internal DB enrichment, there is a problem with cyclic enrichment due to the outputProcessing, need to decide how to handle formulas on enrichment.

packages/server/src/api/controllers/row/internal.ts

@@ -1,6 +1,7 @@
 import * as linkRows from "../../../db/linkedRows"
 import {
   generateRowID,
+  getMultiIDParams,
   getTableIDFromRowID,
   InternalTables,
 } from "../../../db/utils"
@@ -24,6 +25,8 @@ import {
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
+import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
+import { flatten } from "lodash"
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableId = utils.getTableId(ctx)
@@ -224,57 +227,43 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const db = context.getAppDB()
   const tableId = utils.getTableId(ctx)
   const rowId = ctx.params.rowId
-  // need table to work out where links go in row
-  let [table, row] = await Promise.all([
+  // need table to work out where links go in row, as well as the link docs
+  let response = await Promise.all([
     sdk.tables.getTable(tableId),
     utils.findRow(ctx, tableId, rowId),
+    linkRows.getLinkDocuments({ tableId, rowId }),
   ])
-  // get the link docs
-  const linkVals = (await linkRows.getLinkDocuments({
-    tableId,
-    rowId,
-  })) as LinkDocumentValue[]
+  const table = response[0] as Table
+  const row = response[1] as Row
+  const linkVals = response[2] as LinkDocumentValue[]
   // look up the actual rows based on the ids
-  let response = (
-    await db.allDocs({
-      include_docs: true,
-      keys: linkVals.map(linkVal => linkVal.id),
-    })
-  ).rows.map(row => row.doc)
-  // group responses by table
-  let groups: any = {},
-    tables: Record<string, Table> = {}
-  for (let row of response) {
-    if (!row.tableId) {
-      row.tableId = getTableIDFromRowID(row._id)
-    }
-    const linkedTableId = row.tableId
-    if (groups[linkedTableId] == null) {
-      groups[linkedTableId] = [row]
-      tables[linkedTableId] = await db.get(linkedTableId)
-    } else {
-      groups[linkedTableId].push(row)
-    }
-  }
-  let linkedRows: Row[] = []
-  for (let [tableId, rows] of Object.entries(groups)) {
-    // need to include the IDs in these rows for any links they may have
-    linkedRows = linkedRows.concat(
-      await outputProcessing(tables[tableId], rows as Row[])
+  const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
+  let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc)
+
+  // get the linked tables
+  const linkTableIds = getLinkedTableIDs(table as Table)
+  const linkTables = await sdk.tables.getTables(linkTableIds)
+
+  // perform output processing
+  let final = []
+  for (let linkTable of linkTables) {
+    const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id)
+    final = final.concat(
+      outputProcessing(linkTable, relatedRows, { preserveLinks: true })
     )
   }
+  // finalise the promises
+  linkedRows = flatten(await Promise.all(final))
 
   // insert the link rows in the correct place throughout the main row
   for (let fieldName of Object.keys(table.schema)) {
     let field = table.schema[fieldName]
     if (field.type === FieldTypes.LINK) {
-      // find the links that pertain to this field, get their indexes
-      const linkIndexes = linkVals
-        .filter(link => link.fieldName === fieldName)
-        .map(link => linkVals.indexOf(link))
+      // find the links that pertain to this field
+      const links = linkVals.filter(link => link.fieldName === fieldName)
       // find the rows that the links state are linked to this field
-      row[fieldName] = linkedRows.filter((linkRow, index) =>
-        linkIndexes.includes(index)
+      row[fieldName] = linkedRows.filter(linkRow =>
+        links.find(link => link.id === linkRow._id)
       )
     }
   }

packages/server/src/api/controllers/user.ts

@@ -1,13 +1,11 @@
-import { generateUserFlagID } from "../../db/utils"
-import { InternalTables } from "../../db/utils"
+import { generateUserFlagID, InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
 import { Ctx, UserCtx } from "@budibase/types"
 import sdk from "../../sdk"
 
 export async function fetchMetadata(ctx: Ctx) {
-  const users = await sdk.users.fetchMetadata()
-  ctx.body = users
+  ctx.body = await sdk.users.fetchMetadata()
 }
 
 export async function updateSelfMetadata(ctx: UserCtx) {

packages/server/src/db/linkedRows/index.ts

@@ -9,13 +9,13 @@ import {
   getLinkedTable,
 } from "./linkUtils"
 import flatten from "lodash/flatten"
-import { FieldTypes } from "../../constants"
 import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
-import { Table, Row, LinkDocumentValue } from "@budibase/types"
+import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types"
+import sdk from "../../sdk"
 
 export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
 
@@ -35,7 +35,7 @@ export const EventType = {
 
 function clearRelationshipFields(table: Table, rows: Row[]) {
   for (let [key, field] of Object.entries(table.schema)) {
-    if (field.type === FieldTypes.LINK) {
+    if (field.type === FieldType.LINK) {
       rows = rows.map(row => {
         delete row[key]
         return row
@@ -45,7 +45,7 @@ function clearRelationshipFields(table: Table, rows: Row[]) {
   return rows
 }
 
-async function getLinksForRows(rows: Row[]) {
+async function getLinksForRows(rows: Row[]): Promise<LinkDocumentValue[]> {
   const tableIds = [...new Set(rows.map(el => el.tableId))]
   // start by getting all the link values for performance reasons
   const promises = tableIds.map(tableId =>
@@ -153,15 +153,19 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) {
   if (linkedTableIds.length === 0) {
     return rows
   }
-  // get all the links
-  const links = (await getLinksForRows(rows)).filter(link =>
+  // get tables and links
+  let response = await Promise.all([
+    getLinksForRows(rows),
+    sdk.tables.getTables(linkedTableIds),
+  ])
+  const links = (response[0] as LinkDocumentValue[]).filter(link =>
     rows.some(row => row._id === link.thisId)
   )
+  const linkedTables = response[1] as Table[]
   // clear any existing links that could be dupe'd
   rows = clearRelationshipFields(table, rows)
   // now get the docs and combine into the rows
   let linked = await getFullLinkedDocs(links)
-  const linkedTables: Table[] = []
   for (let row of rows) {
     for (let link of links.filter(link => link.thisId === row._id)) {
       if (row[link.fieldName] == null) {
@@ -171,7 +175,9 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) {
       if (linkedRow) {
         const linkedTableId =
           linkedRow.tableId || getRelatedTableForField(table, link.fieldName)
-        const linkedTable = await getLinkedTable(linkedTableId, linkedTables)
+        const linkedTable = linkedTables.find(
+          table => table._id === linkedTableId
+        )
         if (linkedTable) {
           row[link.fieldName].push(processFormulas(linkedTable, linkedRow))
         }
@@ -199,13 +205,13 @@ export async function squashLinksToPrimaryDisplay(
     // this only fetches the table if its not already in array
     const rowTable = await getLinkedTable(row.tableId!, linkedTables)
     for (let [column, schema] of Object.entries(rowTable?.schema || {})) {
-      if (schema.type !== FieldTypes.LINK || !Array.isArray(row[column])) {
+      if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
         continue
       }
       const newLinks = []
       for (let link of row[column]) {
         const linkTblId = link.tableId || getRelatedTableForField(table, column)
-        const linkedTable = await getLinkedTable(linkTblId, linkedTables)
+        const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
         const obj: any = { _id: link._id }
         if (linkedTable?.primaryDisplay && link[linkedTable.primaryDisplay]) {
           obj.primaryDisplay = link[linkedTable.primaryDisplay]

packages/server/src/db/linkedRows/linkUtils.ts

@@ -91,10 +91,10 @@ export function getUniqueByProp(array: any[], prop: string) {
   })
 }
 
-export function getLinkedTableIDs(table: Table) {
+export function getLinkedTableIDs(table: Table): string[] {
   return Object.values(table.schema)
     .filter((column: FieldSchema) => column.type === FieldTypes.LINK)
-    .map(column => column.tableId)
+    .map(column => column.tableId!)
 }
 
 export async function getLinkedTable(id: string, tables: Table[]) {

packages/server/src/sdk/app/datasources/datasources.ts

@@ -23,10 +23,13 @@ import {
   getTableParams,
 } from "../../../db/utils"
 import sdk from "../../index"
+import datasource from "../../../api/routes/datasource"
 
 const ENV_VAR_PREFIX = "env."
 
-export async function fetch() {
+export async function fetch(opts?: {
+  enriched: boolean
+}): Promise<Datasource[]> {
   // Get internal tables
   const db = context.getAppDB()
   const internalTables = await db.allDocs(
@@ -44,7 +47,7 @@ export async function fetch() {
 
   const bbInternalDb = {
     ...BudibaseInternalDB,
-  }
+  } as Datasource
 
   // Get external datasources
   const datasources = (
@@ -66,7 +69,18 @@ export async function fetch() {
     }
   }
 
-  return [bbInternalDb, ...datasources]
+  if (opts?.enriched) {
+    const envVars = await getEnvironmentVariables()
+    const promises = datasources.map(datasource =>
+      enrichDatasourceWithValues(datasource, envVars)
+    )
+    const enriched = (await Promise.all(promises)).map(
+      result => result.datasource
+    )
+    return [bbInternalDb, ...enriched]
+  } else {
+    return [bbInternalDb, ...datasources]
+  }
 }
 
 export function areRESTVariablesValid(datasource: Datasource) {
@@ -107,9 +121,12 @@ export function checkDatasourceTypes(schema: Integration, config: any) {
   return config
 }
 
-async function enrichDatasourceWithValues(datasource: Datasource) {
+async function enrichDatasourceWithValues(
+  datasource: Datasource,
+  variables?: Record<string, string>
+) {
   const cloned = cloneDeep(datasource)
-  const env = await getEnvironmentVariables()
+  const env = variables ? variables : await getEnvironmentVariables()
   //Do not process entities, as we do not want to process formulas
   const { entities, ...clonedWithoutEntities } = cloned
   const processed = processObjectSync(
@@ -235,9 +252,9 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
     if (value !== PASSWORD_REPLACEMENT) {
       continue
     }
-    if (old.config?.[key]) {
+    if (update.config && old.config && old.config?.[key]) {
       update.config[key] = old.config?.[key]
-    } else {
+    } else if (update.config) {
       delete update.config[key]
     }
   }

packages/server/src/sdk/app/tables/index.ts

@@ -1,11 +1,16 @@
 import { context } from "@budibase/backend-core"
-import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
+import {
+  BudibaseInternalDB,
+  getMultiIDParams,
+  getTableParams,
+} from "../../../db/utils"
 import {
   breakExternalTableId,
   isExternalTable,
   isSQL,
 } from "../../../integrations/utils"
 import {
+  AllDocsResponse,
   Database,
   Table,
   TableResponse,
@@ -15,6 +20,14 @@ import datasources from "../datasources"
 import { populateExternalTableSchemas } from "./validation"
 import sdk from "../../../sdk"
 
+function processInternalTables(docs: AllDocsResponse<Table[]>): Table[] {
+  return docs.rows.map((tableDoc: any) => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
+  }))
+}
+
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
     db = context.getAppDB()
@@ -24,15 +37,49 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
       include_docs: true,
     })
   )
-  return internalTables.rows.map((tableDoc: any) => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
-  }))
+  return processInternalTables(internalTables)
 }
 
-async function getAllExternalTables(
-  datasourceId: any
+async function getAllExternalTables(): Promise<Table[]> {
+  const datasources = await sdk.datasources.fetch({ enriched: true })
+  const allEntities = datasources.map(datasource => datasource.entities)
+  let final = []
+  for (let entities of allEntities) {
+    final = final.concat(Object.values(entities))
+  }
+  return final
+}
+
+async function getAllTables() {
+  const [internal, external] = await Promise.all([
+    getAllInternalTables(),
+    getAllExternalTables(),
+  ])
+  return [...internal, external]
+}
+
+async function getTables(tableIds: string[]): Promise<Table[]> {
+  const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)),
+    internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId))
+  let tables = []
+  if (externalTableIds.length) {
+    const externalTables = await getAllExternalTables()
+    tables = tables.concat(
+      externalTables.filter(
+        table => externalTableIds.indexOf(table._id!) !== -1
+      )
+    )
+  }
+  if (internalTableIds.length) {
+    const db = context.getAppDB()
+    const internalTables = await db.allDocs(getMultiIDParams(internalTableIds))
+    tables = tables.concat(processInternalTables(internalTables))
+  }
+  return tables
+}
+
+async function getExternalTablesInDatasource(
+  datasourceId: string
 ): Promise<Record<string, Table>> {
   const datasource = await datasources.get(datasourceId, { enriched: true })
   if (!datasource || !datasource.entities) {
@@ -42,14 +89,14 @@ async function getAllExternalTables(
 }
 
 async function getExternalTable(
-  datasourceId: any,
-  tableName: any
+  datasourceId: string,
+  tableName: string
 ): Promise<Table> {
-  const entities = await getAllExternalTables(datasourceId)
+  const entities = await getExternalTablesInDatasource(datasourceId)
   return entities[tableName]
 }
 
-async function getTable(tableId: any): Promise<Table> {
+async function getTable(tableId: string): Promise<Table> {
   const db = context.getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
@@ -86,9 +133,11 @@ async function saveTable(table: Table) {
 
 export default {
   getAllInternalTables,
-  getAllExternalTables,
+  getExternalTablesInDatasource,
   getExternalTable,
   getTable,
+  getAllTables,
+  getTables,
   populateExternalTableSchemas,
   enrichViewSchemas,
   saveTable,