Merge pull request #12559 from Budibase/fix/budi-7827

Fix external formulas and issues with external relationship configuration

packages/builder/src/components/backend/DataTable/buttons/ExistingRelationshipButton.svelte

@@ -1,14 +1,19 @@
 <script>
   import { ActionButton, notifications } from "@budibase/bbui"
   import CreateEditRelationshipModal from "../../Datasources/CreateEditRelationshipModal.svelte"
-  import { datasources } from "../../../../stores/backend"
+  import {
+    datasources,
+    tables as tablesStore,
+  } from "../../../../stores/backend"
   import { createEventDispatcher } from "svelte"
 
   export let table
   const dispatch = createEventDispatcher()
 
   $: datasource = findDatasource(table?._id)
-  $: tables = datasource?.plus ? Object.values(datasource?.entities || {}) : []
+  $: tables = datasource?.plus
+    ? $tablesStore.list.filter(tbl => tbl.sourceId === datasource._id)
+    : []
 
   let modal
 
@@ -28,7 +33,12 @@
   }
 
   const onError = err => {
-    notifications.error(`Error saving relationship info: ${err}`)
+    if (err.err) {
+      err = err.err
+    }
+    notifications.error(
+      `Error saving relationship info: ${err?.message || JSON.stringify(err)}`
+    )
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -85,6 +85,7 @@
   let relationshipTableIdSecondary = null
 
   let table = $tables.selected
+
   let confirmDeleteDialog
   let savingColumn
   let deleteColName

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte

@@ -15,7 +15,8 @@
 
   let modal
 
-  $: tables = Object.values(datasource.entities)
+  $: tables =
+    $tablesStore.list.filter(tbl => tbl.sourceId === datasource._id) || []
   $: relationships = getRelationships(tables)
 
   function getRelationships(tables) {

packages/server/src/api/controllers/datasource.ts

@@ -26,6 +26,7 @@ import {
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
+import { isEqual } from "lodash"
 
 async function getConnector(
   datasource: Datasource
@@ -198,19 +199,20 @@ async function invalidateVariables(
 export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
-  let datasource = await sdk.datasources.get(datasourceId)
+  const baseDatasource = await sdk.datasources.get(datasourceId)
-  const auth = datasource.config?.auth
+  const auth = baseDatasource.config?.auth
-  await invalidateVariables(datasource, ctx.request.body)
+  await invalidateVariables(baseDatasource, ctx.request.body)
 
-  const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
+  const isBudibaseSource =
+    baseDatasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
 
   const dataSourceBody = isBudibaseSource
     ? { name: ctx.request.body?.name }
     : ctx.request.body
 
-  datasource = {
+  let datasource: Datasource = {
-    ...datasource,
+    ...baseDatasource,
-    ...sdk.datasources.mergeConfigs(dataSourceBody, datasource),
+    ...sdk.datasources.mergeConfigs(dataSourceBody, baseDatasource),
   }
   if (auth && !ctx.request.body.auth) {
     // don't strip auth config from DB
@@ -245,6 +247,15 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
   }
   builderSocket?.emitDatasourceUpdate(ctx, datasource)
+  // send table updates if they have occurred
+  if (datasource.entities) {
+    for (let table of Object.values(datasource.entities)) {
+      const oldTable = baseDatasource.entities?.[table.name]
+      if (!oldTable || !isEqual(oldTable, table)) {
+        builderSocket?.emitTableUpdate(ctx, table, { includeOriginator: true })
+      }
+    }
+  }
 }
 
 const preSaveAction: Partial<Record<SourceName, any>> = {

packages/server/src/integrations/base/sqlTable.ts

@@ -6,13 +6,19 @@ import {
   QueryJson,
   RenameColumn,
   Table,
+  FieldType,
 } from "@budibase/types"
 import { breakExternalTableId } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
-import { FieldTypes, RelationshipType } from "../../constants"
+import { RelationshipType } from "../../constants"
 import { utils } from "@budibase/shared-core"
 
+function isIgnoredType(type: FieldType) {
+  const ignored = [FieldType.LINK, FieldType.FORMULA]
+  return ignored.indexOf(type) !== -1
+}
+
 function generateSchema(
   schema: CreateTableBuilder,
   table: Table,
@@ -47,13 +53,13 @@ function generateSchema(
       continue
     }
     switch (column.type) {
-      case FieldTypes.STRING:
+      case FieldType.STRING:
-      case FieldTypes.OPTIONS:
+      case FieldType.OPTIONS:
-      case FieldTypes.LONGFORM:
+      case FieldType.LONGFORM:
-      case FieldTypes.BARCODEQR:
+      case FieldType.BARCODEQR:
         schema.text(key)
         break
-      case FieldTypes.BB_REFERENCE:
+      case FieldType.BB_REFERENCE:
         const subtype = column.subtype as FieldSubtype
         switch (subtype) {
           case FieldSubtype.USER:
@@ -66,7 +72,7 @@ function generateSchema(
             throw utils.unreachable(subtype)
         }
         break
-      case FieldTypes.NUMBER:
+      case FieldType.NUMBER:
         // if meta is specified then this is a junction table entry
         if (column.meta && column.meta.toKey && column.meta.toTable) {
           const { toKey, toTable } = column.meta
@@ -76,21 +82,21 @@ function generateSchema(
           schema.float(key)
         }
         break
-      case FieldTypes.BIGINT:
+      case FieldType.BIGINT:
         schema.bigint(key)
         break
-      case FieldTypes.BOOLEAN:
+      case FieldType.BOOLEAN:
         schema.boolean(key)
         break
-      case FieldTypes.DATETIME:
+      case FieldType.DATETIME:
         schema.datetime(key, {
           useTz: !column.ignoreTimezones,
         })
         break
-      case FieldTypes.ARRAY:
+      case FieldType.ARRAY:
         schema.json(key)
         break
-      case FieldTypes.LINK:
+      case FieldType.LINK:
         // this side of the relationship doesn't need any SQL work
         if (
           column.relationshipType !== RelationshipType.MANY_TO_ONE &&
@@ -121,22 +127,18 @@ function generateSchema(
     }
   }
 
-  if (renamed) {
+  const oldType = renamed ? oldTable?.schema[renamed.old].type : undefined
+  if (renamed && oldType && !isIgnoredType(oldType)) {
     schema.renameColumn(renamed.old, renamed.updated)
   }
 
   // need to check if any columns have been deleted
   if (oldTable) {
-    const deletedColumns = Object.entries(oldTable.schema)
+    const deletedColumns = Object.entries(oldTable.schema).filter(
-      .filter(
+      ([key, column]) => isIgnoredType(column.type) && table.schema[key] == null
-        ([key, schema]) =>
+    )
-          schema.type !== FieldTypes.LINK &&
+    deletedColumns.forEach(([key, column]) => {
-          schema.type !== FieldTypes.FORMULA &&
+      if (renamed?.old === key || isIgnoredType(column.type)) {
-          table.schema[key] == null
-      )
-      .map(([key]) => key)
-    deletedColumns.forEach(key => {
-      if (renamed?.old === key) {
         return
       }
       if (oldTable.constrained && oldTable.constrained.indexOf(key) !== -1) {