From fe6535a65f301d1c7fa4f8b6c001e42c4877184f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 5 Oct 2023 12:06:31 +0200
Subject: [PATCH 01/35] Type schema fields

---
 packages/server/src/constants/index.ts        | 20 ++----
 .../src/documents/app/table/constants.ts      | 13 ++++
 .../types/src/documents/app/table/schema.ts   | 72 ++++++++++++-------
 3 files changed, 66 insertions(+), 39 deletions(-)

diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts
index c8c17e1d32..326389996d 100644
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@@ -1,6 +1,11 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
 import { FieldType as FieldTypes } from "@budibase/types"
-export { FieldType as FieldTypes, RelationshipType } from "@budibase/types"
+export {
+  FieldType as FieldTypes,
+  RelationshipType,
+  AutoFieldSubTypes,
+  FormulaTypes,
+} from "@budibase/types"
 
 export enum FilterTypes {
   STRING = "string",
@@ -39,11 +44,6 @@ export const SwitchableTypes = CanSwitchTypes.reduce((prev, current) =>
   prev ? prev.concat(current) : current
 )
 
-export enum FormulaTypes {
-  STATIC = "static",
-  DYNAMIC = "dynamic",
-}
-
 export enum AuthTypes {
   APP = "app",
   BUILDER = "builder",
@@ -132,14 +132,6 @@ export const USERS_TABLE_SCHEMA = {
   primaryDisplay: "email",
 }
 
-export enum AutoFieldSubTypes {
-  CREATED_BY = "createdBy",
-  CREATED_AT = "createdAt",
-  UPDATED_BY = "updatedBy",
-  UPDATED_AT = "updatedAt",
-  AUTO_ID = "autoID",
-}
-
 export enum AutoFieldDefaultNames {
   CREATED_BY = "Created By",
   CREATED_AT = "Created At",
diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts
index 9a0ea4d135..783eae0671 100644
--- a/packages/types/src/documents/app/table/constants.ts
+++ b/packages/types/src/documents/app/table/constants.ts
@@ -7,3 +7,16 @@ export enum RelationshipType {
 export enum AutoReason {
   FOREIGN_KEY = "foreign_key",
 }
+
+export enum AutoFieldSubTypes {
+  CREATED_BY = "createdBy",
+  CREATED_AT = "createdAt",
+  UPDATED_BY = "updatedBy",
+  UPDATED_AT = "updatedAt",
+  AUTO_ID = "autoID",
+}
+
+export enum FormulaTypes {
+  STATIC = "static",
+  DYNAMIC = "dynamic",
+}
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 42a0838231..5e7fba25b9 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -1,7 +1,12 @@
 // all added by grid/table when defining the
 // column size, position and whether it can be viewed
 import { FieldType } from "../row"
-import { AutoReason, RelationshipType } from "./constants"
+import {
+  AutoFieldSubTypes,
+  AutoReason,
+  FormulaTypes,
+  RelationshipType,
+} from "./constants"
 
 export interface UIFieldMetadata {
   order?: number
@@ -10,20 +15,25 @@ export interface UIFieldMetadata {
   icon?: string
 }
 
-export interface RelationshipFieldMetadata {
+interface ManyToManyRelationshipFieldMetadata {
+  relationshipType: RelationshipType.MANY_TO_MANY
+  through: string
+  throughFrom: string
+  throughTo: string
+}
+interface OneSidedRelationshipFieldMetadata {
+  relationshipType: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE
+  foreignKey: string
+}
+export type RelationshipFieldMetadata = BaseFieldSchema & {
+  type: FieldType.LINK
   main?: boolean
   fieldName?: string
-  tableId?: string
-  // below is used for SQL relationships, needed to define the foreign keys
-  // or the tables used for many-to-many relationships (through)
-  relationshipType?: RelationshipType
-  through?: string
-  foreignKey?: string
-  throughFrom?: string
-  throughTo?: string
-}
+  tableId: string
+} & (ManyToManyRelationshipFieldMetadata | OneSidedRelationshipFieldMetadata)
 
-export interface AutoColumnFieldMetadata {
+export interface AutoColumnFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AUTO
   autocolumn?: boolean
   subtype?: string
   lastID?: number
@@ -31,7 +41,10 @@ export interface AutoColumnFieldMetadata {
   autoReason?: AutoReason
 }
 
-export interface NumberFieldMetadata {
+interface NumberForeignKeyMetadata {
+  subtype: AutoFieldSubTypes.AUTO_ID
+  autoReason: AutoReason.FOREIGN_KEY
+  autocolumn: true
   // used specifically when Budibase generates external tables, this denotes if a number field
   // is a foreign key used for a many-to-many relationship
   meta?: {
@@ -40,18 +53,26 @@ export interface NumberFieldMetadata {
   }
 }
 
-export interface DateFieldMetadata {
+export type NumberFieldMetadata = BaseFieldSchema & {
+  type: FieldType.NUMBER
+  autocolumn: boolean
+} & (NumberForeignKeyMetadata | {})
+
+export interface DateFieldMetadata extends BaseFieldSchema {
+  type: FieldType.DATETIME
   ignoreTimezones?: boolean
   timeOnly?: boolean
 }
 
-export interface StringFieldMetadata {
+export interface StringFieldMetadata extends BaseFieldSchema {
+  type: FieldType.STRING
   useRichText?: boolean | null
 }
 
-export interface FormulaFieldMetadata {
+export interface FormulaFieldMetadata extends BaseFieldSchema {
+  type: FieldType.FORMULA
   formula?: string
-  formulaType?: string
+  formulaType?: FormulaTypes
 }
 
 export interface FieldConstraints {
@@ -77,14 +98,7 @@ export interface FieldConstraints {
   }
 }
 
-export interface FieldSchema
-  extends UIFieldMetadata,
-    DateFieldMetadata,
-    RelationshipFieldMetadata,
-    AutoColumnFieldMetadata,
-    StringFieldMetadata,
-    FormulaFieldMetadata,
-    NumberFieldMetadata {
+interface BaseFieldSchema extends UIFieldMetadata {
   type: FieldType
   name: string
   sortable?: boolean
@@ -93,6 +107,14 @@ export interface FieldSchema
   constraints?: FieldConstraints
 }
 
+export type FieldSchema =
+  | DateFieldMetadata
+  | RelationshipFieldMetadata
+  | AutoColumnFieldMetadata
+  | StringFieldMetadata
+  | FormulaFieldMetadata
+  | NumberFieldMetadata
+
 export interface TableSchema {
   [key: string]: FieldSchema
 }

From dd373cd5e9d12b44e93f885e8d0065a3c6174d8a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 5 Oct 2023 12:06:42 +0200
Subject: [PATCH 02/35] Fix types

---
 packages/server/src/sdk/app/tables/validation.ts | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/sdk/app/tables/validation.ts b/packages/server/src/sdk/app/tables/validation.ts
index 56f3e84c7a..f97cce39e3 100644
--- a/packages/server/src/sdk/app/tables/validation.ts
+++ b/packages/server/src/sdk/app/tables/validation.ts
@@ -15,10 +15,11 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   // make sure all foreign key columns are marked as auto columns
   const foreignKeys: { tableId: string; key: string }[] = []
   for (let table of tables) {
-    const relationships = Object.values(table.schema).filter(
-      column => column.type === FieldType.LINK
-    )
-    relationships.forEach(relationship => {
+    Object.values(table.schema).forEach(column => {
+      if (column.type !== FieldType.LINK) {
+        return
+      }
+      const relationship = column
       if (relationship.relationshipType === RelationshipType.MANY_TO_MANY) {
         const tableId = relationship.through!
         foreignKeys.push({ key: relationship.throughTo!, tableId })
@@ -36,8 +37,9 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   }
 
   // now make sure schemas are all accurate
-  for (let table of tables) {
-    for (let column of Object.values(table.schema)) {
+  for (const table of tables) {
+    for (let column of Object.values(table.schema) as any[]) {
+      // TODO: any[]
       const shouldBeForeign = foreignKeys.find(
         options => options.tableId === table._id && options.key === column.name
       )

From c8ffa9884413924d674067bcd97a332d0b9fbbfa Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 5 Oct 2023 12:29:33 +0200
Subject: [PATCH 03/35] Fixes

---
 .../src/api/controllers/table/internal.ts     |  9 ++++++---
 .../src/utilities/rowProcessor/utils.ts       | 12 ++++++-----
 .../types/src/documents/app/table/schema.ts   | 20 +++++++++++++++++--
 3 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts
index e468848c57..81e203d405 100644
--- a/packages/server/src/api/controllers/table/internal.ts
+++ b/packages/server/src/api/controllers/table/internal.ts
@@ -10,6 +10,7 @@ import {
 } from "../../../utilities/rowProcessor"
 import { runStaticFormulaChecks } from "./bulkFormula"
 import {
+  AutoColumnFieldMetadata,
   RenameColumn,
   SaveTableRequest,
   SaveTableResponse,
@@ -35,7 +36,9 @@ function checkAutoColumns(table: Table, oldTable?: Table) {
     if (oldSchema && oldSchema.subtype) {
       table.schema[key].subtype = oldSchema.subtype
     } else {
-      table.schema[key] = fixAutoColumnSubType(schema)
+      table.schema[key] = fixAutoColumnSubType(
+        schema as AutoColumnFieldMetadata
+      )
     }
   }
   return table
@@ -78,10 +81,10 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   // make sure that types don't change of a column, have to remove
   // the column if you want to change the type
   if (oldTable && oldTable.schema) {
-    for (let propKey of Object.keys(tableToSave.schema)) {
+    for (const propKey of Object.keys(tableToSave.schema)) {
       let oldColumn = oldTable.schema[propKey]
       if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) {
-        oldColumn.type = FieldTypes.AUTO
+        oldColumn.type = FieldTypes.AUTO as any // TODO
       }
     }
   }
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 0d7eace369..435174b4e7 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -5,13 +5,13 @@ import {
   FormulaTypes,
 } from "../../constants"
 import { processStringSync } from "@budibase/string-templates"
-import { FieldSchema, Row, Table } from "@budibase/types"
+import { AutoColumnFieldMetadata, Row, Table } from "@budibase/types"
 
 /**
  * If the subtype has been lost for any reason this works out what
  * subtype the auto column should be.
  */
-export function fixAutoColumnSubType(column: FieldSchema) {
+export function fixAutoColumnSubType(column: AutoColumnFieldMetadata) {
   if (!column.autocolumn || !column.name || column.subtype) {
     return column
   }
@@ -47,12 +47,14 @@ export function processFormulas(
     rowArray = rows
   }
   for (let [column, schema] of Object.entries(table.schema)) {
-    const isStatic = schema.formulaType === FormulaTypes.STATIC
+    const isStaticFormula =
+      schema.type === FieldTypes.FORMULA &&
+      schema.formulaType === FormulaTypes.STATIC
     if (
       schema.type !== FieldTypes.FORMULA ||
       schema.formula == null ||
-      (dynamic && isStatic) ||
-      (!dynamic && !isStatic)
+      (dynamic && isStaticFormula) ||
+      (!dynamic && !isStaticFormula)
     ) {
       continue
     }
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 5e7fba25b9..1cdbb75303 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -34,8 +34,8 @@ export type RelationshipFieldMetadata = BaseFieldSchema & {
 
 export interface AutoColumnFieldMetadata extends BaseFieldSchema {
   type: FieldType.AUTO
-  autocolumn?: boolean
-  subtype?: string
+  autocolumn: true
+  subtype: AutoFieldSubTypes
   lastID?: number
   // if the column was turned to an auto-column for SQL, explains why (primary, foreign etc)
   autoReason?: AutoReason
@@ -105,9 +105,25 @@ interface BaseFieldSchema extends UIFieldMetadata {
   // only used by external databases, to denote the real type
   externalType?: string
   constraints?: FieldConstraints
+  autocolumn?: boolean
+  subtype?: string
+}
+
+interface OtherFieldMetadata extends BaseFieldSchema {
+  type: Exclude<
+    FieldType,
+    | FieldType.DATETIME
+    | FieldType.DATETIME
+    | FieldType.LINK
+    | FieldType.AUTO
+    | FieldType.STRING
+    | FieldType.FORMULA
+    | FieldType.NUMBER
+  >
 }
 
 export type FieldSchema =
+  | OtherFieldMetadata
   | DateFieldMetadata
   | RelationshipFieldMetadata
   | AutoColumnFieldMetadata

From 21e2d7ddbeea047402f2643f78c7ba2f616280ee Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 5 Oct 2023 14:25:25 +0200
Subject: [PATCH 04/35] Fix relationship types

---
 .../api/controllers/row/ExternalRequest.ts    | 51 +++++++++++++------
 packages/server/src/integrations/oracle.ts    |  3 +-
 .../src/utilities/rowProcessor/index.ts       | 18 +++++--
 .../types/src/documents/app/table/schema.ts   | 31 +++++++----
 4 files changed, 73 insertions(+), 30 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 2ad1afe202..0fea102f7b 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -5,8 +5,11 @@ import {
   FieldType,
   FilterType,
   IncludeRelationship,
+  ManyToManyRelationshipFieldMetadata,
+  OneToManyRelationshipFieldMetadata,
   Operation,
   PaginationJson,
+  RelationshipFieldMetadata,
   RelationshipsJson,
   RelationshipType,
   Row,
@@ -254,12 +257,20 @@ function fixArrayTypes(row: Row, table: Table) {
   return row
 }
 
-function isOneSide(field: FieldSchema) {
+function isOneSide(
+  field: RelationshipFieldMetadata
+): field is OneToManyRelationshipFieldMetadata {
   return (
     field.relationshipType && field.relationshipType.split("-")[0] === "one"
   )
 }
 
+function isManyToMany(
+  field: RelationshipFieldMetadata
+): field is ManyToManyRelationshipFieldMetadata {
+  return !!(field as ManyToManyRelationshipFieldMetadata).through
+}
+
 function isEditableColumn(column: FieldSchema) {
   const isExternalAutoColumn =
     column.autocolumn &&
@@ -352,11 +363,11 @@ export class ExternalRequest<T extends Operation> {
         }
       }
       // many to many
-      else if (field.through) {
+      else if (isManyToMany(field)) {
         // we're not inserting a doc, will be a bunch of update calls
         const otherKey: string = field.throughFrom || linkTablePrimary
         const thisKey: string = field.throughTo || tablePrimary
-        row[key].forEach((relationship: any) => {
+        for (const relationship of row[key]) {
           manyRelationships.push({
             tableId: field.through || field.tableId,
             isUpdate: false,
@@ -365,14 +376,14 @@ export class ExternalRequest<T extends Operation> {
             // leave the ID for enrichment later
             [thisKey]: `{{ literal ${tablePrimary} }}`,
           })
-        })
+        }
       }
       // many to one
       else {
         const thisKey: string = "id"
         // @ts-ignore
         const otherKey: string = field.fieldName
-        row[key].forEach((relationship: any) => {
+        for (const relationship of row[key]) {
           manyRelationships.push({
             tableId: field.tableId,
             isUpdate: true,
@@ -381,7 +392,7 @@ export class ExternalRequest<T extends Operation> {
             // leave the ID for enrichment later
             [otherKey]: `{{ literal ${tablePrimary} }}`,
           })
-        })
+        }
       }
     }
     // we return the relationships that may need to be created in the through table
@@ -551,20 +562,24 @@ export class ExternalRequest<T extends Operation> {
       }
       const definition: any = {
         // if no foreign key specified then use the name of the field in other table
-        from: field.foreignKey || table.primary[0],
+        from: (field as any).foreignKey || table.primary[0],
         to: field.fieldName,
         tableName: linkTableName,
         // need to specify where to put this back into
         column: fieldName,
       }
-      if (field.through) {
+      if ((field as ManyToManyRelationshipFieldMetadata).through) {
         const { tableName: throughTableName } = breakExternalTableId(
-          field.through
+          (field as ManyToManyRelationshipFieldMetadata).through
         )
         definition.through = throughTableName
         // don't support composite keys for relationships
-        definition.from = field.throughTo || table.primary[0]
-        definition.to = field.throughFrom || linkTable.primary[0]
+        definition.from =
+          (field as ManyToManyRelationshipFieldMetadata).throughTo ||
+          table.primary[0]
+        definition.to =
+          (field as ManyToManyRelationshipFieldMetadata).throughFrom ||
+          linkTable.primary[0]
         definition.fromPrimary = table.primary[0]
         definition.toPrimary = linkTable.primary[0]
       }
@@ -597,12 +612,15 @@ export class ExternalRequest<T extends Operation> {
         continue
       }
       const isMany = field.relationshipType === RelationshipType.MANY_TO_MANY
-      const tableId = isMany ? field.through : field.tableId
+      const tableId = isMany
+        ? (field as ManyToManyRelationshipFieldMetadata).through
+        : field.tableId
       const { tableName: relatedTableName } = breakExternalTableId(tableId)
       // @ts-ignore
       const linkPrimaryKey = this.tables[relatedTableName].primary[0]
-      const manyKey = field.throughTo || primaryKey
-      const lookupField = isMany ? primaryKey : field.foreignKey
+      const manyKey =
+        (field as ManyToManyRelationshipFieldMetadata).throughTo || primaryKey
+      const lookupField = isMany ? primaryKey : (field as any).foreignKey
       const fieldName = isMany ? manyKey : field.fieldName
       if (!lookupField || !row[lookupField]) {
         continue
@@ -617,7 +635,10 @@ export class ExternalRequest<T extends Operation> {
       })
       // this is the response from knex if no rows found
       const rows = !response[0].read ? response : []
-      const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName
+      const storeTo = isMany
+        ? (field as ManyToManyRelationshipFieldMetadata).throughFrom ||
+          linkPrimaryKey
+        : fieldName
       related[storeTo] = { rows, isMany, tableId }
     }
     return related
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index 3e8ec15423..0ad1fdb698 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -306,7 +306,8 @@ class OracleIntegration extends Sql implements DatasourcePlus {
                 presence: false,
               },
               ...this.internalConvertType(oracleColumn),
-            }
+            } as any // TODO
+
             table.schema[columnName] = fieldSchema
           }
 
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 0bdaaa393e..0a7e774fc9 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -5,7 +5,13 @@ import { ObjectStoreBuckets } from "../../constants"
 import { context, db as dbCore, objectStore } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
+import {
+  AutoColumnFieldMetadata,
+  FieldSubtype,
+  Row,
+  RowAttachment,
+  Table,
+} from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import {
   processInputBBReferences,
@@ -71,7 +77,7 @@ export function processAutoColumn(
       continue
     }
     if (!schema.subtype) {
-      schema = fixAutoColumnSubType(schema)
+      schema = fixAutoColumnSubType(schema as AutoColumnFieldMetadata)
     }
     switch (schema.subtype) {
       case AutoFieldSubTypes.CREATED_BY:
@@ -94,8 +100,12 @@ export function processAutoColumn(
         break
       case AutoFieldSubTypes.AUTO_ID:
         if (creating) {
-          schema.lastID = !schema.lastID ? BASE_AUTO_ID : schema.lastID + 1
-          row[key] = schema.lastID
+          const safeSchema = schema as AutoColumnFieldMetadata
+
+          safeSchema.lastID = !safeSchema.lastID
+            ? BASE_AUTO_ID
+            : safeSchema.lastID + 1
+          row[key] = safeSchema.lastID
         }
         break
     }
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 1cdbb75303..53ad39ac96 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -15,22 +15,33 @@ export interface UIFieldMetadata {
   icon?: string
 }
 
-interface ManyToManyRelationshipFieldMetadata {
+interface BaseRelationshipFieldMetadata extends BaseFieldSchema {
+  type: FieldType.LINK
+  main?: boolean
+  fieldName?: string
+  tableId: string
+}
+export interface ManyToManyRelationshipFieldMetadata
+  extends BaseRelationshipFieldMetadata {
   relationshipType: RelationshipType.MANY_TO_MANY
   through: string
   throughFrom: string
   throughTo: string
 }
-interface OneSidedRelationshipFieldMetadata {
-  relationshipType: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE
+export interface OneToManyRelationshipFieldMetadata
+  extends BaseRelationshipFieldMetadata {
+  relationshipType: RelationshipType.ONE_TO_MANY
   foreignKey: string
 }
-export type RelationshipFieldMetadata = BaseFieldSchema & {
-  type: FieldType.LINK
-  main?: boolean
-  fieldName?: string
-  tableId: string
-} & (ManyToManyRelationshipFieldMetadata | OneSidedRelationshipFieldMetadata)
+export interface ManyToOneRelationshipFieldMetadata
+  extends BaseRelationshipFieldMetadata {
+  relationshipType: RelationshipType.MANY_TO_ONE
+  foreignKey: string
+}
+export type RelationshipFieldMetadata =
+  | ManyToManyRelationshipFieldMetadata
+  | OneToManyRelationshipFieldMetadata
+  | ManyToOneRelationshipFieldMetadata
 
 export interface AutoColumnFieldMetadata extends BaseFieldSchema {
   type: FieldType.AUTO
@@ -106,6 +117,7 @@ interface BaseFieldSchema extends UIFieldMetadata {
   externalType?: string
   constraints?: FieldConstraints
   autocolumn?: boolean
+  autoReason?: AutoReason
   subtype?: string
 }
 
@@ -113,7 +125,6 @@ interface OtherFieldMetadata extends BaseFieldSchema {
   type: Exclude<
     FieldType,
     | FieldType.DATETIME
-    | FieldType.DATETIME
     | FieldType.LINK
     | FieldType.AUTO
     | FieldType.STRING

From 40226c5c3aae19be67b3752f0d3121d45bfab4f3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 5 Oct 2023 16:14:30 +0200
Subject: [PATCH 05/35] Fix more types

---
 packages/server/src/api/controllers/row/utils.ts      | 11 ++++++++++-
 .../server/src/tests/utilities/TestConfiguration.ts   |  2 +-
 packages/types/src/documents/app/table/schema.ts      |  2 +-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts
index 5f10fd9ad4..1243d18847 100644
--- a/packages/server/src/api/controllers/row/utils.ts
+++ b/packages/server/src/api/controllers/row/utils.ts
@@ -4,6 +4,8 @@ import { context } from "@budibase/backend-core"
 import {
   Ctx,
   FieldType,
+  ManyToOneRelationshipFieldMetadata,
+  OneToManyRelationshipFieldMetadata,
   Row,
   SearchFilters,
   Table,
@@ -19,7 +21,14 @@ function isForeignKey(key: string, table: Table) {
   const relationships = Object.values(table.schema).filter(
     column => column.type === FieldType.LINK
   )
-  return relationships.some(relationship => relationship.foreignKey === key)
+  return relationships.some(
+    relationship =>
+      (
+        relationship as
+          | OneToManyRelationshipFieldMetadata
+          | ManyToOneRelationshipFieldMetadata
+      ).foreignKey === key
+  )
 }
 
 validateJs.extend(validateJs.validators.datetime, {
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 21b6463ce7..50aec9afc7 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -584,7 +584,7 @@ class TestConfiguration {
       tableConfig.schema[link] = {
         type: FieldType.LINK,
         fieldName: link,
-        tableId: this.table._id,
+        tableId: this.table._id!,
         name: link,
         relationshipType,
       }
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 53ad39ac96..343aa16699 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -66,7 +66,7 @@ interface NumberForeignKeyMetadata {
 
 export type NumberFieldMetadata = BaseFieldSchema & {
   type: FieldType.NUMBER
-  autocolumn: boolean
+  autocolumn?: boolean
 } & (NumberForeignKeyMetadata | {})
 
 export interface DateFieldMetadata extends BaseFieldSchema {

From 7ee63365f78e6f9260d0d44c63dc42a7ee3d0444 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 5 Oct 2023 16:22:01 +0200
Subject: [PATCH 06/35] Fix

---
 .../src/api/controllers/table/bulkFormula.ts  | 27 +++++++++++++------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/controllers/table/bulkFormula.ts b/packages/server/src/api/controllers/table/bulkFormula.ts
index e44cb95391..4cc3849c5b 100644
--- a/packages/server/src/api/controllers/table/bulkFormula.ts
+++ b/packages/server/src/api/controllers/table/bulkFormula.ts
@@ -6,10 +6,17 @@ import isEqual from "lodash/isEqual"
 import uniq from "lodash/uniq"
 import { updateAllFormulasInTable } from "../row/staticFormula"
 import { context } from "@budibase/backend-core"
-import { FieldSchema, Table } from "@budibase/types"
+import {
+  FieldSchema,
+  FormulaFieldMetadata,
+  RelationshipFieldMetadata,
+  Table,
+} from "@budibase/types"
 import sdk from "../../../sdk"
 
-function isStaticFormula(column: FieldSchema) {
+function isStaticFormula(
+  column: FieldSchema
+): column is FormulaFieldMetadata & { formulaType: FormulaTypes.STATIC } {
   return (
     column.type === FieldTypes.FORMULA &&
     column.formulaType === FormulaTypes.STATIC
@@ -57,7 +64,8 @@ async function checkIfFormulaNeedsCleared(
     let tableToUse: Table | undefined = table
     // if relationship, get the related table
     if (removed.type === FieldTypes.LINK) {
-      tableToUse = tables.find(table => table._id === removed.tableId)
+      const removedTableId = removed.tableId
+      tableToUse = tables.find(table => table._id === removedTableId)
     }
     if (!tableToUse) {
       continue
@@ -73,7 +81,7 @@ async function checkIfFormulaNeedsCleared(
     }
     for (let relatedTableId of table.relatedFormula) {
       const relatedColumns = Object.values(table.schema).filter(
-        column => column.tableId === relatedTableId
+        column => (column as any).tableId === relatedTableId
       )
       const relatedTable = tables.find(table => table._id === relatedTableId)
       // look to see if the column was used in a relationship formula,
@@ -83,7 +91,7 @@ async function checkIfFormulaNeedsCleared(
         for (let column of relatedColumns) {
           relatedFormulaToRemove = relatedFormulaToRemove.concat(
             getFormulaThatUseColumn(relatedTable, [
-              column.fieldName!,
+              (column as any).fieldName!,
               removed.name,
             ])
           )
@@ -96,6 +104,10 @@ async function checkIfFormulaNeedsCleared(
   }
 }
 
+function isLink(column: FieldSchema): column is RelationshipFieldMetadata {
+  return column.type === FieldTypes.LINK
+}
+
 /**
  * This function adds a note to related tables that they are
  * used in a static formula - so that the link controller
@@ -115,9 +127,7 @@ async function updateRelatedFormulaLinksOnTables(
   // clone the tables, so we can compare at end
   const initialTables = cloneDeep(tables)
   // first find the related column names
-  const relatedColumns = Object.values(table.schema).filter(
-    col => col.type === FieldTypes.LINK
-  )
+  const relatedColumns = Object.values(table.schema).filter(isLink)
   // we start by removing the formula field from all tables
   for (let otherTable of tables) {
     if (!otherTable.relatedFormula) {
@@ -135,6 +145,7 @@ async function updateRelatedFormulaLinksOnTables(
       if (!columns || columns.length === 0) {
         continue
       }
+
       const relatedTable = tables.find(
         related => related._id === relatedCol.tableId
       )

From ea86ded912266be3e920ef1ac00785e9f0c81234 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 5 Oct 2023 16:29:27 +0200
Subject: [PATCH 07/35] Fixes

---
 .../src/api/controllers/table/bulkFormula.ts     | 16 +++++-----------
 packages/server/src/db/linkedRows/linkUtils.ts   | 14 +++++---------
 packages/server/src/db/utils.ts                  | 14 +++++++++++++-
 packages/server/src/sdk/app/tables/validation.ts |  2 --
 4 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/packages/server/src/api/controllers/table/bulkFormula.ts b/packages/server/src/api/controllers/table/bulkFormula.ts
index 4cc3849c5b..8f03036c0d 100644
--- a/packages/server/src/api/controllers/table/bulkFormula.ts
+++ b/packages/server/src/api/controllers/table/bulkFormula.ts
@@ -6,13 +6,9 @@ import isEqual from "lodash/isEqual"
 import uniq from "lodash/uniq"
 import { updateAllFormulasInTable } from "../row/staticFormula"
 import { context } from "@budibase/backend-core"
-import {
-  FieldSchema,
-  FormulaFieldMetadata,
-  RelationshipFieldMetadata,
-  Table,
-} from "@budibase/types"
+import { FieldSchema, FormulaFieldMetadata, Table } from "@budibase/types"
 import sdk from "../../../sdk"
+import { isRelationshipColumn } from "../../../db/utils"
 
 function isStaticFormula(
   column: FieldSchema
@@ -104,10 +100,6 @@ async function checkIfFormulaNeedsCleared(
   }
 }
 
-function isLink(column: FieldSchema): column is RelationshipFieldMetadata {
-  return column.type === FieldTypes.LINK
-}
-
 /**
  * This function adds a note to related tables that they are
  * used in a static formula - so that the link controller
@@ -127,7 +119,9 @@ async function updateRelatedFormulaLinksOnTables(
   // clone the tables, so we can compare at end
   const initialTables = cloneDeep(tables)
   // first find the related column names
-  const relatedColumns = Object.values(table.schema).filter(isLink)
+  const relatedColumns = Object.values(table.schema).filter(
+    isRelationshipColumn
+  )
   // we start by removing the formula field from all tables
   for (let otherTable of tables) {
     if (!otherTable.relatedFormula) {
diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts
index c7db7d522a..ba0d2b3b0e 100644
--- a/packages/server/src/db/linkedRows/linkUtils.ts
+++ b/packages/server/src/db/linkedRows/linkUtils.ts
@@ -1,13 +1,9 @@
-import { ViewName, getQueryIndex } from "../utils"
+import { ViewName, getQueryIndex, isRelationshipColumn } from "../utils"
 import { FieldTypes } from "../../constants"
 import { createLinkView } from "../views/staticViews"
 import { context, logging } from "@budibase/backend-core"
-import {
-  FieldSchema,
-  LinkDocument,
-  LinkDocumentValue,
-  Table,
-} from "@budibase/types"
+import { LinkDocument, LinkDocumentValue, Table } from "@budibase/types"
+
 export { createLinkView } from "../views/staticViews"
 
 /**
@@ -93,7 +89,7 @@ export function getUniqueByProp(array: any[], prop: string) {
 
 export function getLinkedTableIDs(table: Table) {
   return Object.values(table.schema)
-    .filter((column: FieldSchema) => column.type === FieldTypes.LINK)
+    .filter(isRelationshipColumn)
     .map(column => column.tableId)
 }
 
@@ -114,7 +110,7 @@ export function getRelatedTableForField(table: Table, fieldName: string) {
   // look to see if its on the table, straight in the schema
   const field = table.schema[fieldName]
   if (field != null) {
-    return field.tableId
+    return (field as any).tableId
   }
   for (let column of Object.values(table.schema)) {
     if (column.type === FieldTypes.LINK && column.fieldName === fieldName) {
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index abea725707..0ffc2ac43c 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -1,6 +1,12 @@
 import newid from "./newid"
 import { db as dbCore } from "@budibase/backend-core"
-import { DocumentType, VirtualDocumentType } from "@budibase/types"
+import {
+  DocumentType,
+  FieldSchema,
+  RelationshipFieldMetadata,
+  VirtualDocumentType,
+} from "@budibase/types"
+import { FieldTypes } from "src/constants"
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
 
 type Optional = string | null
@@ -307,3 +313,9 @@ export function extractViewInfoFromID(viewId: string) {
     tableId: res!.groups!["tableId"],
   }
 }
+
+export function isRelationshipColumn(
+  column: FieldSchema
+): column is RelationshipFieldMetadata {
+  return column.type === FieldTypes.LINK
+}
diff --git a/packages/server/src/sdk/app/tables/validation.ts b/packages/server/src/sdk/app/tables/validation.ts
index f97cce39e3..816ddd14ff 100644
--- a/packages/server/src/sdk/app/tables/validation.ts
+++ b/packages/server/src/sdk/app/tables/validation.ts
@@ -1,11 +1,9 @@
 import {
   AutoReason,
   Datasource,
-  FieldSchema,
   FieldType,
   RelationshipType,
 } from "@budibase/types"
-import { FieldTypes } from "../../../constants"
 
 function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   if (!datasource.entities) {

From 6c328109b69f548d46bc65e016c67b828a12f2c5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 5 Oct 2023 19:47:00 +0200
Subject: [PATCH 08/35] Fixes

---
 .../src/api/controllers/table/external.ts     | 37 +++++++++++--------
 .../server/src/api/routes/tests/table.spec.ts |  2 +-
 .../db/defaultData/datasource_bb_default.ts   | 10 ++---
 .../src/db/linkedRows/LinkController.ts       | 16 +++++---
 .../server/src/integrations/base/sqlTable.ts  | 14 +++++--
 packages/server/src/sdk/app/rows/utils.ts     |  9 +++--
 .../rowProcessor/tests/utils.spec.ts          |  2 +-
 .../types/src/documents/app/table/schema.ts   | 24 ++++++------
 8 files changed, 69 insertions(+), 45 deletions(-)

diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts
index 327904666d..dac39a4a99 100644
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@@ -15,11 +15,14 @@ import { handleRequest } from "../row/external"
 import { context, events } from "@budibase/backend-core"
 import { isRows, isSchema, parse } from "../../../utilities/schema"
 import {
-  AutoReason,
   Datasource,
   FieldSchema,
+  ManyToManyRelationshipFieldMetadata,
+  ManyToOneRelationshipFieldMetadata,
+  OneToManyRelationshipFieldMetadata,
   Operation,
   QueryJson,
+  RelationshipFieldMetadata,
   RelationshipType,
   RenameColumn,
   SaveTableRequest,
@@ -74,10 +77,11 @@ function cleanupRelationships(
       schema.type === FieldTypes.LINK &&
       (!oldTable || table.schema[key] == null)
     ) {
+      const schemaTableId = schema.tableId
       const relatedTable = Object.values(tables).find(
-        table => table._id === schema.tableId
+        table => table._id === schemaTableId
       )
-      const foreignKey = schema.foreignKey
+      const foreignKey = (schema as any).foreignKey
       if (!relatedTable || !foreignKey) {
         continue
       }
@@ -116,7 +120,7 @@ function otherRelationshipType(type?: string) {
 
 function generateManyLinkSchema(
   datasource: Datasource,
-  column: FieldSchema,
+  column: ManyToManyRelationshipFieldMetadata,
   table: Table,
   relatedTable: Table
 ): Table {
@@ -151,10 +155,12 @@ function generateManyLinkSchema(
 }
 
 function generateLinkSchema(
-  column: FieldSchema,
+  column:
+    | OneToManyRelationshipFieldMetadata
+    | ManyToOneRelationshipFieldMetadata,
   table: Table,
   relatedTable: Table,
-  type: RelationshipType
+  type: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE
 ) {
   if (!table.primary || !relatedTable.primary) {
     throw new Error("Unable to generate link schema, no primary keys")
@@ -170,7 +176,7 @@ function generateLinkSchema(
 }
 
 function generateRelatedSchema(
-  linkColumn: FieldSchema,
+  linkColumn: RelationshipFieldMetadata,
   table: Table,
   relatedTable: Table,
   columnName: string
@@ -178,16 +184,16 @@ function generateRelatedSchema(
   // generate column for other table
   const relatedSchema = cloneDeep(linkColumn)
   // swap them from the main link
-  if (linkColumn.foreignKey) {
-    relatedSchema.fieldName = linkColumn.foreignKey
+  if ((linkColumn as any).foreignKey) {
+    relatedSchema.fieldName = (linkColumn as any).foreignKey
     relatedSchema.foreignKey = linkColumn.fieldName
   }
   // is many to many
   else {
     // don't need to copy through, already got it
-    relatedSchema.fieldName = linkColumn.throughTo
-    relatedSchema.throughTo = linkColumn.throughFrom
-    relatedSchema.throughFrom = linkColumn.throughTo
+    relatedSchema.fieldName = (linkColumn as any).throughTo
+    relatedSchema.throughTo = (linkColumn as any).throughFrom
+    relatedSchema.throughFrom = (linkColumn as any).throughTo
   }
   relatedSchema.relationshipType = otherRelationshipType(
     linkColumn.relationshipType
@@ -198,7 +204,7 @@ function generateRelatedSchema(
 }
 
 function isRelationshipSetup(column: FieldSchema) {
-  return column.foreignKey || column.through
+  return (column as any).foreignKey || (column as any).through
 }
 
 export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
@@ -257,14 +263,15 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
     if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) {
       continue
     }
+    const schemaTableId = schema.tableId
     const relatedTable = Object.values(tables).find(
-      table => table._id === schema.tableId
+      table => table._id === schemaTableId
     )
     if (!relatedTable) {
       continue
     }
     const relatedColumnName = schema.fieldName!
-    const relationType = schema.relationshipType!
+    const relationType = schema.relationshipType
     if (relationType === RelationshipType.MANY_TO_MANY) {
       const junctionTable = generateManyLinkSchema(
         datasource,
diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index f56c6e4e44..fc49335fc7 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -354,7 +354,7 @@ describe("/tables", () => {
             type: FieldType.LINK,
             name: "TestTable",
             fieldName: "TestTable",
-            tableId: testTable._id,
+            tableId: testTable._id!,
             constraints: {
               type: "array",
             },
diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts
index a4821667ff..d99c8132bc 100644
--- a/packages/server/src/db/defaultData/datasource_bb_default.ts
+++ b/packages/server/src/db/defaultData/datasource_bb_default.ts
@@ -29,7 +29,7 @@ function syncLastIds(table: Table, rowCount: number) {
   Object.keys(table.schema).forEach(key => {
     const entry = table.schema[key]
     if (entry.autocolumn && entry.subtype == "autoID") {
-      entry.lastID = rowCount
+      ;(entry as any).lastID = rowCount
     }
   })
 }
@@ -184,7 +184,7 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = {
     },
     ...AUTO_COLUMNS,
   },
-}
+} as any
 
 export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
   _id: DEFAULT_EMPLOYEE_TABLE_ID,
@@ -332,7 +332,7 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
     },
     ...AUTO_COLUMNS,
   },
-}
+} as any
 
 export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
   _id: DEFAULT_JOBS_TABLE_ID,
@@ -489,7 +489,7 @@ export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
     },
     ...AUTO_COLUMNS,
   },
-}
+} as any
 
 export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
   _id: DEFAULT_EXPENSES_TABLE_ID,
@@ -599,7 +599,7 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
     },
     ...AUTO_COLUMNS,
   },
-}
+} as any
 
 export async function buildDefaultDocs() {
   const inventoryData = await tableImport(
diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts
index 457819251a..34dd0c6f80 100644
--- a/packages/server/src/db/linkedRows/LinkController.ts
+++ b/packages/server/src/db/linkedRows/LinkController.ts
@@ -8,6 +8,7 @@ import {
   Database,
   FieldSchema,
   LinkDocumentValue,
+  RelationshipFieldMetadata,
   RelationshipType,
   Row,
   Table,
@@ -133,7 +134,10 @@ class LinkController {
    * Given the link field of this table, and the link field of the linked table, this makes sure
    * the state of relationship type is accurate on both.
    */
-  handleRelationshipType(linkerField: FieldSchema, linkedField: FieldSchema) {
+  handleRelationshipType(
+    linkerField: RelationshipFieldMetadata,
+    linkedField: RelationshipFieldMetadata
+  ) {
     if (
       !linkerField.relationshipType ||
       linkerField.relationshipType === RelationshipType.MANY_TO_MANY
@@ -183,7 +187,9 @@ class LinkController {
 
         // if 1:N, ensure that this ID is not already attached to another record
         const linkedTable = await this._db.get<Table>(field.tableId)
-        const linkedSchema = linkedTable.schema[field.fieldName!]
+        const linkedSchema = linkedTable.schema[
+          field.fieldName!
+        ] as RelationshipFieldMetadata
 
         // We need to map the global users to metadata in each app for relationships
         if (field.tableId === InternalTables.USER_METADATA) {
@@ -291,7 +297,7 @@ class LinkController {
    */
   async removeFieldFromTable(fieldName: string) {
     let oldTable = this._oldTable
-    let field = oldTable?.schema[fieldName] as FieldSchema
+    let field = oldTable?.schema[fieldName] as RelationshipFieldMetadata
     const linkDocs = await this.getTableLinkDocs()
     let toDelete = linkDocs.filter(linkDoc => {
       let correctFieldName =
@@ -351,9 +357,9 @@ class LinkController {
           name: field.fieldName,
           type: FieldTypes.LINK,
           // these are the props of the table that initiated the link
-          tableId: table._id,
+          tableId: table._id!,
           fieldName: fieldName,
-        })
+        } as any)
 
         // update table schema after checking relationship types
         schema[fieldName] = fields.linkerField
diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts
index 4383167f4a..5915a6e868 100644
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@@ -1,5 +1,11 @@
 import { Knex, knex } from "knex"
-import { Operation, QueryJson, RenameColumn, Table } from "@budibase/types"
+import {
+  NumberFieldMetadata,
+  Operation,
+  QueryJson,
+  RenameColumn,
+  Table,
+} from "@budibase/types"
 import { breakExternalTableId } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
@@ -15,7 +21,7 @@ function generateSchema(
   let primaryKey = table && table.primary ? table.primary[0] : null
   const columns = Object.values(table.schema)
   // all columns in a junction table will be meta
-  let metaCols = columns.filter(col => col.meta)
+  let metaCols = columns.filter(col => (col as NumberFieldMetadata).meta)
   let isJunction = metaCols.length === columns.length
   // can't change primary once its set for now
   if (primaryKey && !oldTable && !isJunction) {
@@ -25,7 +31,9 @@ function generateSchema(
   }
 
   // check if any columns need added
-  const foreignKeys = Object.values(table.schema).map(col => col.foreignKey)
+  const foreignKeys = Object.values(table.schema).map(
+    col => (col as any).foreignKey
+  )
   for (let [key, column] of Object.entries(table.schema)) {
     // skip things that are already correct
     const oldColumn = oldTable ? oldTable.schema[key] : null
diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index 51e418c324..4049d898fb 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -1,10 +1,11 @@
 import cloneDeep from "lodash/cloneDeep"
 import validateJs from "validate.js"
-import { FieldType, Row, Table, TableSchema } from "@budibase/types"
+import { Row, Table, TableSchema } from "@budibase/types"
 import { FieldTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
+import { isRelationshipColumn } from "../../../db/utils"
 
 export async function getDatasourceAndQuery(json: any) {
   const datasourceId = json.endpoint.datasourceId
@@ -50,10 +51,10 @@ export function cleanExportRows(
 }
 
 function isForeignKey(key: string, table: Table) {
-  const relationships = Object.values(table.schema).filter(
-    column => column.type === FieldType.LINK
+  const relationships = Object.values(table.schema).filter(isRelationshipColumn)
+  return relationships.some(
+    relationship => (relationship as any).foreignKey === key
   )
-  return relationships.some(relationship => relationship.foreignKey === key)
 }
 
 export async function validate({
diff --git a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
index 68b62a3291..4a943ddd67 100644
--- a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
@@ -4,7 +4,7 @@ import { FieldSchema, FieldType, RelationshipType } from "@budibase/types"
 
 describe("rowProcessor utility", () => {
   describe("fixAutoColumnSubType", () => {
-    let schema: FieldSchema = {
+    const schema: FieldSchema = {
       name: "",
       type: FieldType.LINK,
       subtype: "", // missing subtype
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 343aa16699..3cfddea770 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -36,7 +36,7 @@ export interface OneToManyRelationshipFieldMetadata
 export interface ManyToOneRelationshipFieldMetadata
   extends BaseRelationshipFieldMetadata {
   relationshipType: RelationshipType.MANY_TO_ONE
-  foreignKey: string
+  foreignKey?: string
 }
 export type RelationshipFieldMetadata =
   | ManyToManyRelationshipFieldMetadata
@@ -52,10 +52,11 @@ export interface AutoColumnFieldMetadata extends BaseFieldSchema {
   autoReason?: AutoReason
 }
 
-interface NumberForeignKeyMetadata {
-  subtype: AutoFieldSubTypes.AUTO_ID
-  autoReason: AutoReason.FOREIGN_KEY
-  autocolumn: true
+export interface NumberFieldMetadata extends BaseFieldSchema {
+  type: FieldType.NUMBER
+  autocolumn?: boolean
+  subtype?: AutoFieldSubTypes.AUTO_ID
+  autoReason?: AutoReason.FOREIGN_KEY
   // used specifically when Budibase generates external tables, this denotes if a number field
   // is a foreign key used for a many-to-many relationship
   meta?: {
@@ -64,11 +65,6 @@ interface NumberForeignKeyMetadata {
   }
 }
 
-export type NumberFieldMetadata = BaseFieldSchema & {
-  type: FieldType.NUMBER
-  autocolumn?: boolean
-} & (NumberForeignKeyMetadata | {})
-
 export interface DateFieldMetadata extends BaseFieldSchema {
   type: FieldType.DATETIME
   ignoreTimezones?: boolean
@@ -77,6 +73,10 @@ export interface DateFieldMetadata extends BaseFieldSchema {
 
 export interface StringFieldMetadata extends BaseFieldSchema {
   type: FieldType.STRING
+}
+
+export interface LongFormFieldMetadata extends BaseFieldSchema {
+  type: FieldType.LONGFORM
   useRichText?: boolean | null
 }
 
@@ -117,7 +117,7 @@ interface BaseFieldSchema extends UIFieldMetadata {
   externalType?: string
   constraints?: FieldConstraints
   autocolumn?: boolean
-  autoReason?: AutoReason
+  autoReason?: AutoReason.FOREIGN_KEY
   subtype?: string
 }
 
@@ -130,6 +130,7 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.STRING
     | FieldType.FORMULA
     | FieldType.NUMBER
+    | FieldType.LONGFORM
   >
 }
 
@@ -141,6 +142,7 @@ export type FieldSchema =
   | StringFieldMetadata
   | FormulaFieldMetadata
   | NumberFieldMetadata
+  | LongFormFieldMetadata
 
 export interface TableSchema {
   [key: string]: FieldSchema

From 1f1ebc82e58ffd9591e9131648c1d8cb70ad4e9d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Oct 2023 09:12:45 +0200
Subject: [PATCH 09/35] Fix more types

---
 .../server/src/api/routes/tests/row.spec.ts   | 27 ++++++++++---------
 .../server/src/api/routes/tests/table.spec.ts |  8 +++++-
 .../src/integration-test/postgres.spec.ts     |  7 ++---
 .../sdk/app/tables/tests/validation.spec.ts   |  4 +--
 .../types/src/documents/app/table/schema.ts   | 12 ++++-----
 5 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index fa329dbb4b..f1f56ef1fd 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -6,6 +6,8 @@ import * as setup from "./utilities"
 import { context, InternalTable, roles, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
+  AutoFieldSubTypes,
+  FieldSchema,
   FieldType,
   FieldTypeSubtypes,
   MonthlyQuotaName,
@@ -171,7 +173,7 @@ describe.each([
             "Row ID": {
               name: "Row ID",
               type: FieldType.NUMBER,
-              subtype: "autoID",
+              subtype: AutoFieldSubTypes.AUTO_ID,
               icon: "ri-magic-line",
               autocolumn: true,
               constraints: {
@@ -272,27 +274,27 @@ describe.each([
 
     isInternal &&
       it("row values are coerced", async () => {
-        const str = {
+        const str: FieldSchema = {
           type: FieldType.STRING,
           name: "str",
           constraints: { type: "string", presence: false },
         }
-        const attachment = {
+        const attachment: FieldSchema = {
           type: FieldType.ATTACHMENT,
           name: "attachment",
           constraints: { type: "array", presence: false },
         }
-        const bool = {
+        const bool: FieldSchema = {
           type: FieldType.BOOLEAN,
           name: "boolean",
           constraints: { type: "boolean", presence: false },
         }
-        const number = {
+        const number: FieldSchema = {
           type: FieldType.NUMBER,
           name: "str",
           constraints: { type: "number", presence: false },
         }
-        const datetime = {
+        const datetime: FieldSchema = {
           type: FieldType.DATETIME,
           name: "datetime",
           constraints: {
@@ -301,7 +303,7 @@ describe.each([
             datetime: { earliest: "", latest: "" },
           },
         }
-        const arrayField = {
+        const arrayField: FieldSchema = {
           type: FieldType.ARRAY,
           constraints: {
             type: "array",
@@ -311,8 +313,7 @@ describe.each([
           name: "Sample Tags",
           sortable: false,
         }
-        const optsField = {
-          fieldName: "Sample Opts",
+        const optsField: FieldSchema = {
           name: "Sample Opts",
           type: FieldType.OPTIONS,
           constraints: {
@@ -1534,7 +1535,7 @@ describe.each([
   describe.each([
     [
       "relationship fields",
-      () => ({
+      (): Record<string, FieldSchema> => ({
         user: {
           name: "user",
           relationshipType: RelationshipType.ONE_TO_MANY,
@@ -1563,19 +1564,19 @@ describe.each([
     ],
     [
       "bb reference fields",
-      () => ({
+      (): Record<string, FieldSchema> => ({
         user: {
           name: "user",
           relationshipType: RelationshipType.ONE_TO_MANY,
           type: FieldType.BB_REFERENCE,
           subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
-        },
+        } as any,
         users: {
           name: "users",
           type: FieldType.BB_REFERENCE,
           subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
           relationshipType: RelationshipType.MANY_TO_MANY,
-        },
+        } as any,
       }),
       () => config.createUser(),
       (row: Row) => ({
diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index fc49335fc7..fe2f4eaac3 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -1,6 +1,11 @@
 import { generator } from "@budibase/backend-core/tests"
 import { events, context } from "@budibase/backend-core"
-import { FieldType, Table, ViewCalculation } from "@budibase/types"
+import {
+  FieldType,
+  RelationshipType,
+  Table,
+  ViewCalculation,
+} from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 const { basicTable } = setup.structures
@@ -352,6 +357,7 @@ describe("/tables", () => {
           },
           TestTable: {
             type: FieldType.LINK,
+            relationshipType: RelationshipType.ONE_TO_MANY,
             name: "TestTable",
             fieldName: "TestTable",
             tableId: testTable._id!,
diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 86a43c8c42..bcd1c14389 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -111,7 +111,7 @@ describe("postgres integrations", () => {
           fieldName: oneToManyRelationshipInfo.fieldName,
           name: "oneToManyRelation",
           relationshipType: RelationshipType.ONE_TO_MANY,
-          tableId: oneToManyRelationshipInfo.table._id,
+          tableId: oneToManyRelationshipInfo.table._id!,
           main: true,
         },
         manyToOneRelation: {
@@ -122,7 +122,7 @@ describe("postgres integrations", () => {
           fieldName: manyToOneRelationshipInfo.fieldName,
           name: "manyToOneRelation",
           relationshipType: RelationshipType.MANY_TO_ONE,
-          tableId: manyToOneRelationshipInfo.table._id,
+          tableId: manyToOneRelationshipInfo.table._id!,
           main: true,
         },
         manyToManyRelation: {
@@ -133,7 +133,7 @@ describe("postgres integrations", () => {
           fieldName: manyToManyRelationshipInfo.fieldName,
           name: "manyToManyRelation",
           relationshipType: RelationshipType.MANY_TO_MANY,
-          tableId: manyToManyRelationshipInfo.table._id,
+          tableId: manyToManyRelationshipInfo.table._id!,
           main: true,
         },
       },
@@ -250,6 +250,7 @@ describe("postgres integrations", () => {
         id: {
           name: "id",
           type: FieldType.AUTO,
+          autocolumn: true,
         },
       },
       sourceId: postgresDatasource._id,
diff --git a/packages/server/src/sdk/app/tables/tests/validation.spec.ts b/packages/server/src/sdk/app/tables/tests/validation.spec.ts
index ffc34d0afd..5347eede90 100644
--- a/packages/server/src/sdk/app/tables/tests/validation.spec.ts
+++ b/packages/server/src/sdk/app/tables/tests/validation.spec.ts
@@ -1,6 +1,6 @@
 import { populateExternalTableSchemas } from "../validation"
 import { cloneDeep } from "lodash/fp"
-import { Datasource, Table } from "@budibase/types"
+import { AutoReason, Datasource, Table } from "@budibase/types"
 import { isEqual } from "lodash"
 
 const SCHEMA = {
@@ -109,7 +109,7 @@ describe("validation and update of external table schemas", () => {
     const response = populateExternalTableSchemas(cloneDeep(SCHEMA) as any)
     const foreignKey = getForeignKeyColumn(response)
     expect(foreignKey.autocolumn).toBe(true)
-    expect(foreignKey.autoReason).toBe("foreign_key")
+    expect(foreignKey.autoReason).toBe(AutoReason.FOREIGN_KEY)
     noOtherTableChanges(response)
   })
 
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 3cfddea770..39ec477a24 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -24,14 +24,14 @@ interface BaseRelationshipFieldMetadata extends BaseFieldSchema {
 export interface ManyToManyRelationshipFieldMetadata
   extends BaseRelationshipFieldMetadata {
   relationshipType: RelationshipType.MANY_TO_MANY
-  through: string
-  throughFrom: string
-  throughTo: string
+  through?: string
+  throughFrom?: string
+  throughTo?: string
 }
 export interface OneToManyRelationshipFieldMetadata
   extends BaseRelationshipFieldMetadata {
   relationshipType: RelationshipType.ONE_TO_MANY
-  foreignKey: string
+  foreignKey?: string
 }
 export interface ManyToOneRelationshipFieldMetadata
   extends BaseRelationshipFieldMetadata {
@@ -46,7 +46,7 @@ export type RelationshipFieldMetadata =
 export interface AutoColumnFieldMetadata extends BaseFieldSchema {
   type: FieldType.AUTO
   autocolumn: true
-  subtype: AutoFieldSubTypes
+  subtype?: AutoFieldSubTypes
   lastID?: number
   // if the column was turned to an auto-column for SQL, explains why (primary, foreign etc)
   autoReason?: AutoReason
@@ -121,7 +121,7 @@ interface BaseFieldSchema extends UIFieldMetadata {
   subtype?: string
 }
 
-interface OtherFieldMetadata extends BaseFieldSchema {
+export interface OtherFieldMetadata extends BaseFieldSchema {
   type: Exclude<
     FieldType,
     | FieldType.DATETIME

From a8c814e714a63ab9b3eeb4dce90874845979ac03 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Oct 2023 09:32:28 +0200
Subject: [PATCH 10/35] Fixes fixes

---
 .../server/src/sdk/app/views/tests/views.spec.ts   | 14 ++++++++++----
 .../server/src/utilities/rowProcessor/utils.ts     |  4 ++--
 packages/types/src/documents/app/table/schema.ts   |  2 +-
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/sdk/app/views/tests/views.spec.ts b/packages/server/src/sdk/app/views/tests/views.spec.ts
index 2314f362e9..8fcc6405ef 100644
--- a/packages/server/src/sdk/app/views/tests/views.spec.ts
+++ b/packages/server/src/sdk/app/views/tests/views.spec.ts
@@ -1,5 +1,11 @@
 import _ from "lodash"
-import { FieldType, Table, TableSchema, ViewV2 } from "@budibase/types"
+import {
+  FieldSchema,
+  FieldType,
+  Table,
+  TableSchema,
+  ViewV2,
+} from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
 import { enrichSchema, syncSchema } from ".."
 
@@ -316,7 +322,7 @@ describe("table sdk", () => {
           ...basicView,
         }
 
-        const newTableSchema = {
+        const newTableSchema: TableSchema = {
           ...basicTable.schema,
           newField1: {
             type: FieldType.STRING,
@@ -403,7 +409,7 @@ describe("table sdk", () => {
           },
         }
 
-        const newTableSchema = {
+        const newTableSchema: TableSchema = {
           ...basicTable.schema,
           newField1: {
             type: FieldType.STRING,
@@ -531,7 +537,7 @@ describe("table sdk", () => {
             id: {
               ...basicTable.schema.id,
               type: FieldType.NUMBER,
-            },
+            } as FieldSchema,
           },
           undefined
         )
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 435174b4e7..0a051a02e2 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -5,13 +5,13 @@ import {
   FormulaTypes,
 } from "../../constants"
 import { processStringSync } from "@budibase/string-templates"
-import { AutoColumnFieldMetadata, Row, Table } from "@budibase/types"
+import { FieldSchema, Row, Table } from "@budibase/types"
 
 /**
  * If the subtype has been lost for any reason this works out what
  * subtype the auto column should be.
  */
-export function fixAutoColumnSubType(column: AutoColumnFieldMetadata) {
+export function fixAutoColumnSubType(column: FieldSchema) {
   if (!column.autocolumn || !column.name || column.subtype) {
     return column
   }
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 39ec477a24..9cfd67bf80 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -121,7 +121,7 @@ interface BaseFieldSchema extends UIFieldMetadata {
   subtype?: string
 }
 
-export interface OtherFieldMetadata extends BaseFieldSchema {
+interface OtherFieldMetadata extends BaseFieldSchema {
   type: Exclude<
     FieldType,
     | FieldType.DATETIME

From 1b7be0d637998e831bd35acf362742aeb3aecb8e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Oct 2023 10:46:58 +0200
Subject: [PATCH 11/35] Fix import

---
 packages/server/src/db/utils.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index 0ffc2ac43c..428c955eb2 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -6,7 +6,7 @@ import {
   RelationshipFieldMetadata,
   VirtualDocumentType,
 } from "@budibase/types"
-import { FieldTypes } from "src/constants"
+import { FieldTypes } from "../constants"
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
 
 type Optional = string | null

From 74cba9de27846608ce6a0ea30cd0a988d4003490 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Oct 2023 10:57:43 +0200
Subject: [PATCH 12/35] Lint

---
 .../src/db/defaultData/datasource_bb_default.ts       | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts
index d99c8132bc..9b0db067a7 100644
--- a/packages/server/src/db/defaultData/datasource_bb_default.ts
+++ b/packages/server/src/db/defaultData/datasource_bb_default.ts
@@ -7,7 +7,12 @@ import { employeeImport } from "./employeeImport"
 import { jobsImport } from "./jobsImport"
 import { expensesImport } from "./expensesImport"
 import { db as dbCore } from "@budibase/backend-core"
-import { Table, Row, RelationshipType } from "@budibase/types"
+import {
+  Table,
+  Row,
+  RelationshipType,
+  AutoColumnFieldMetadata,
+} from "@budibase/types"
 
 export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
 export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
@@ -27,9 +32,9 @@ export const DEFAULT_BB_DATASOURCE = defaultDatasource
 
 function syncLastIds(table: Table, rowCount: number) {
   Object.keys(table.schema).forEach(key => {
-    const entry = table.schema[key]
+    const entry = table.schema[key] as AutoColumnFieldMetadata
     if (entry.autocolumn && entry.subtype == "autoID") {
-      ;(entry as any).lastID = rowCount
+      entry.lastID = rowCount
     }
   })
 }

From f57104730ea7eafdfb84931feaf7926627e4dbca Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 12:23:54 +0200
Subject: [PATCH 13/35] Remove subtype from the base field schema

---
 .../rowProcessor/tests/utils.spec.ts          | 12 +++++------
 .../src/utilities/rowProcessor/utils.ts       | 11 ++++++++--
 .../types/src/documents/app/table/schema.ts   | 20 ++++++++++++++-----
 3 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
index 4a943ddd67..2437fa8864 100644
--- a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
@@ -7,7 +7,7 @@ describe("rowProcessor utility", () => {
     const schema: FieldSchema = {
       name: "",
       type: FieldType.LINK,
-      subtype: "", // missing subtype
+      subtype: undefined, // missing subtype
       icon: "ri-magic-line",
       autocolumn: true,
       constraints: { type: "array", presence: false },
@@ -22,31 +22,31 @@ describe("rowProcessor utility", () => {
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.CREATED_BY
       )
-      schema.subtype = ""
+      schema.subtype = undefined
 
       schema.name = AutoFieldDefaultNames.UPDATED_BY
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.UPDATED_BY
       )
-      schema.subtype = ""
+      schema.subtype = undefined
 
       schema.name = AutoFieldDefaultNames.CREATED_AT
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.CREATED_AT
       )
-      schema.subtype = ""
+      schema.subtype = undefined
 
       schema.name = AutoFieldDefaultNames.UPDATED_AT
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.UPDATED_AT
       )
-      schema.subtype = ""
+      schema.subtype = undefined
 
       schema.name = AutoFieldDefaultNames.AUTO_ID
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.AUTO_ID
       )
-      schema.subtype = ""
+      schema.subtype = undefined
     })
 
     it("returns the column if subtype exists", async () => {
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 0a051a02e2..f93fdc4c46 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -5,13 +5,20 @@ import {
   FormulaTypes,
 } from "../../constants"
 import { processStringSync } from "@budibase/string-templates"
-import { FieldSchema, Row, Table } from "@budibase/types"
+import {
+  AutoColumnFieldMetadata,
+  FieldSchema,
+  Row,
+  Table,
+} from "@budibase/types"
 
 /**
  * If the subtype has been lost for any reason this works out what
  * subtype the auto column should be.
  */
-export function fixAutoColumnSubType(column: FieldSchema) {
+export function fixAutoColumnSubType(
+  column: FieldSchema
+): AutoColumnFieldMetadata | FieldSchema {
   if (!column.autocolumn || !column.name || column.subtype) {
     return column
   }
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 9cfd67bf80..6ecc32dca4 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -1,6 +1,6 @@
 // all added by grid/table when defining the
 // column size, position and whether it can be viewed
-import { FieldType } from "../row"
+import { FieldSubtype, FieldType } from "../row"
 import {
   AutoFieldSubTypes,
   AutoReason,
@@ -15,11 +15,13 @@ export interface UIFieldMetadata {
   icon?: string
 }
 
-interface BaseRelationshipFieldMetadata extends BaseFieldSchema {
+interface BaseRelationshipFieldMetadata
+  extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.LINK
   main?: boolean
   fieldName?: string
   tableId: string
+  subtype?: Omit<AutoFieldSubTypes, AutoFieldSubTypes.AUTO_ID>
 }
 export interface ManyToManyRelationshipFieldMetadata
   extends BaseRelationshipFieldMetadata {
@@ -43,7 +45,8 @@ export type RelationshipFieldMetadata =
   | OneToManyRelationshipFieldMetadata
   | ManyToOneRelationshipFieldMetadata
 
-export interface AutoColumnFieldMetadata extends BaseFieldSchema {
+export interface AutoColumnFieldMetadata
+  extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.AUTO
   autocolumn: true
   subtype?: AutoFieldSubTypes
@@ -52,7 +55,7 @@ export interface AutoColumnFieldMetadata extends BaseFieldSchema {
   autoReason?: AutoReason
 }
 
-export interface NumberFieldMetadata extends BaseFieldSchema {
+export interface NumberFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.NUMBER
   autocolumn?: boolean
   subtype?: AutoFieldSubTypes.AUTO_ID
@@ -86,6 +89,12 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
   formulaType?: FormulaTypes
 }
 
+export interface BBReferenceFieldMetadata
+  extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.BB_REFERENCE
+  subtype: FieldSubtype.USER
+}
+
 export interface FieldConstraints {
   type?: string
   email?: boolean
@@ -118,7 +127,7 @@ interface BaseFieldSchema extends UIFieldMetadata {
   constraints?: FieldConstraints
   autocolumn?: boolean
   autoReason?: AutoReason.FOREIGN_KEY
-  subtype?: string
+  subtype?: never
 }
 
 interface OtherFieldMetadata extends BaseFieldSchema {
@@ -143,6 +152,7 @@ export type FieldSchema =
   | FormulaFieldMetadata
   | NumberFieldMetadata
   | LongFormFieldMetadata
+  | BBReferenceFieldMetadata
 
 export interface TableSchema {
   [key: string]: FieldSchema

From 0113b5d8e8f41b2b5bc7e7de7741bd2d796b3bce Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 12:33:01 +0200
Subject: [PATCH 14/35] Remove unnused StringFieldMetadata

---
 packages/types/src/documents/app/table/schema.ts | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 6ecc32dca4..5cab8e5da0 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -57,7 +57,6 @@ export interface AutoColumnFieldMetadata
 
 export interface NumberFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.NUMBER
-  autocolumn?: boolean
   subtype?: AutoFieldSubTypes.AUTO_ID
   autoReason?: AutoReason.FOREIGN_KEY
   // used specifically when Budibase generates external tables, this denotes if a number field
@@ -74,10 +73,6 @@ export interface DateFieldMetadata extends BaseFieldSchema {
   timeOnly?: boolean
 }
 
-export interface StringFieldMetadata extends BaseFieldSchema {
-  type: FieldType.STRING
-}
-
 export interface LongFormFieldMetadata extends BaseFieldSchema {
   type: FieldType.LONGFORM
   useRichText?: boolean | null
@@ -136,7 +131,6 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.DATETIME
     | FieldType.LINK
     | FieldType.AUTO
-    | FieldType.STRING
     | FieldType.FORMULA
     | FieldType.NUMBER
     | FieldType.LONGFORM
@@ -148,7 +142,6 @@ export type FieldSchema =
   | DateFieldMetadata
   | RelationshipFieldMetadata
   | AutoColumnFieldMetadata
-  | StringFieldMetadata
   | FormulaFieldMetadata
   | NumberFieldMetadata
   | LongFormFieldMetadata

From 5edc3d3de6e23efcaeb5911b5c681a0fc8aa2bef Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 12:34:58 +0200
Subject: [PATCH 15/35] Change formula to be mandatory on FormulaFieldMetadata

---
 packages/types/src/documents/app/table/schema.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 5cab8e5da0..0e4837bc50 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -80,7 +80,7 @@ export interface LongFormFieldMetadata extends BaseFieldSchema {
 
 export interface FormulaFieldMetadata extends BaseFieldSchema {
   type: FieldType.FORMULA
-  formula?: string
+  formula: string
   formulaType?: FormulaTypes
 }
 

From 248c44ac2aab5218e74dc24b49ddee02898a01db Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 12:55:23 +0200
Subject: [PATCH 16/35] Fix relationship field metadata

---
 .../src/tests/utilities/TestConfiguration.ts  |  3 +-
 .../types/src/documents/app/table/schema.ts   | 28 +++++++++++++------
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 50aec9afc7..cec8c8aa12 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -54,6 +54,7 @@ import {
   FieldType,
   RelationshipType,
   CreateViewRequest,
+  RelationshipFieldMetadata,
 } from "@budibase/types"
 
 import API from "./api"
@@ -587,7 +588,7 @@ class TestConfiguration {
         tableId: this.table._id!,
         name: link,
         relationshipType,
-      }
+      } as RelationshipFieldMetadata
     }
 
     if (this.datasource && !tableConfig.sourceId) {
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 0e4837bc50..5af5b9c6d3 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -19,17 +19,29 @@ interface BaseRelationshipFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.LINK
   main?: boolean
-  fieldName?: string
+  fieldName: string
   tableId: string
   subtype?: Omit<AutoFieldSubTypes, AutoFieldSubTypes.AUTO_ID>
 }
-export interface ManyToManyRelationshipFieldMetadata
-  extends BaseRelationshipFieldMetadata {
-  relationshipType: RelationshipType.MANY_TO_MANY
-  through?: string
-  throughFrom?: string
-  throughTo?: string
-}
+
+// External tables use junction tables, internal tables don't require them
+type ManyToManyJunctionTableMetadata =
+  | {
+      through: string
+      throughFrom: string
+      throughTo: string
+    }
+  | {
+      through?: never
+      throughFrom?: never
+      throughTo?: never
+    }
+
+export type ManyToManyRelationshipFieldMetadata =
+  BaseRelationshipFieldMetadata & {
+    relationshipType: RelationshipType.MANY_TO_MANY
+  } & ManyToManyJunctionTableMetadata
+
 export interface OneToManyRelationshipFieldMetadata
   extends BaseRelationshipFieldMetadata {
   relationshipType: RelationshipType.ONE_TO_MANY

From 3f6d48da1f8e644d6c4747e4b2e52bcb78943535 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:03:05 +0200
Subject: [PATCH 17/35] Fix autoid numbers

---
 .../server/src/db/defaultData/datasource_bb_default.ts   | 9 +++++++--
 packages/types/src/documents/app/table/schema.ts         | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts
index 9b0db067a7..7eb3bfd26c 100644
--- a/packages/server/src/db/defaultData/datasource_bb_default.ts
+++ b/packages/server/src/db/defaultData/datasource_bb_default.ts
@@ -12,6 +12,7 @@ import {
   Row,
   RelationshipType,
   AutoColumnFieldMetadata,
+  FieldType,
 } from "@budibase/types"
 
 export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
@@ -32,8 +33,12 @@ export const DEFAULT_BB_DATASOURCE = defaultDatasource
 
 function syncLastIds(table: Table, rowCount: number) {
   Object.keys(table.schema).forEach(key => {
-    const entry = table.schema[key] as AutoColumnFieldMetadata
-    if (entry.autocolumn && entry.subtype == "autoID") {
+    const entry = table.schema[key]
+    if (
+      entry.autocolumn &&
+      entry.type === FieldType.NUMBER &&
+      entry.subtype == AutoFieldSubTypes.AUTO_ID
+    ) {
       entry.lastID = rowCount
     }
   })
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 5af5b9c6d3..ca8fbf7cba 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -70,6 +70,7 @@ export interface AutoColumnFieldMetadata
 export interface NumberFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.NUMBER
   subtype?: AutoFieldSubTypes.AUTO_ID
+  lastID?: number
   autoReason?: AutoReason.FOREIGN_KEY
   // used specifically when Budibase generates external tables, this denotes if a number field
   // is a foreign key used for a many-to-many relationship

From 79ef97387bccde2ce5bba15ccfa521650444496a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:06:18 +0200
Subject: [PATCH 18/35] Subtype on datetime

---
 .../src/db/defaultData/datasource_bb_default.ts      | 12 ++++++------
 packages/types/src/documents/app/table/schema.ts     |  5 +++--
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts
index 7eb3bfd26c..48d4876de1 100644
--- a/packages/server/src/db/defaultData/datasource_bb_default.ts
+++ b/packages/server/src/db/defaultData/datasource_bb_default.ts
@@ -11,8 +11,8 @@ import {
   Table,
   Row,
   RelationshipType,
-  AutoColumnFieldMetadata,
   FieldType,
+  TableSchema,
 } from "@budibase/types"
 
 export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
@@ -52,7 +52,7 @@ async function tableImport(table: Table, data: Row[]) {
 }
 
 // AUTO COLUMNS
-const AUTO_COLUMNS = {
+const AUTO_COLUMNS: TableSchema = {
   "Created At": {
     name: "Created At",
     type: FieldTypes.DATETIME,
@@ -194,7 +194,7 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = {
     },
     ...AUTO_COLUMNS,
   },
-} as any
+}
 
 export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
   _id: DEFAULT_EMPLOYEE_TABLE_ID,
@@ -342,7 +342,7 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
     },
     ...AUTO_COLUMNS,
   },
-} as any
+}
 
 export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
   _id: DEFAULT_JOBS_TABLE_ID,
@@ -499,7 +499,7 @@ export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
     },
     ...AUTO_COLUMNS,
   },
-} as any
+}
 
 export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
   _id: DEFAULT_EXPENSES_TABLE_ID,
@@ -609,7 +609,7 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
     },
     ...AUTO_COLUMNS,
   },
-} as any
+}
 
 export async function buildDefaultDocs() {
   const inventoryData = await tableImport(
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index ca8fbf7cba..4106254fc4 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -21,7 +21,7 @@ interface BaseRelationshipFieldMetadata
   main?: boolean
   fieldName: string
   tableId: string
-  subtype?: Omit<AutoFieldSubTypes, AutoFieldSubTypes.AUTO_ID>
+  subtype?: AutoFieldSubTypes.CREATED_BY | AutoFieldSubTypes.UPDATED_BY
 }
 
 // External tables use junction tables, internal tables don't require them
@@ -80,10 +80,11 @@ export interface NumberFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   }
 }
 
-export interface DateFieldMetadata extends BaseFieldSchema {
+export interface DateFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.DATETIME
   ignoreTimezones?: boolean
   timeOnly?: boolean
+  subtype?: AutoFieldSubTypes.CREATED_AT | AutoFieldSubTypes.UPDATED_AT
 }
 
 export interface LongFormFieldMetadata extends BaseFieldSchema {

From 164e1f4c180a6e993c42e7a234e087bff59734ba Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:15:59 +0200
Subject: [PATCH 19/35] Clean

---
 packages/server/src/integrations/oracle.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index 0ad1fdb698..38f0a9d5ac 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -249,7 +249,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     )
   }
 
-  private internalConvertType(column: OracleColumn): { type: FieldTypes } {
+  private internalConvertType(column: OracleColumn) {
     if (this.isBooleanType(column)) {
       return { type: FieldTypes.BOOLEAN }
     }
@@ -306,7 +306,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
                 presence: false,
               },
               ...this.internalConvertType(oracleColumn),
-            } as any // TODO
+            }
 
             table.schema[columnName] = fieldSchema
           }

From ea1c273de0086934bb149de8da9c86ea932ab55c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:18:04 +0200
Subject: [PATCH 20/35] Clean

---
 packages/server/src/sdk/app/tables/validation.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/server/src/sdk/app/tables/validation.ts b/packages/server/src/sdk/app/tables/validation.ts
index 816ddd14ff..1609bdfcda 100644
--- a/packages/server/src/sdk/app/tables/validation.ts
+++ b/packages/server/src/sdk/app/tables/validation.ts
@@ -36,8 +36,7 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
 
   // now make sure schemas are all accurate
   for (const table of tables) {
-    for (let column of Object.values(table.schema) as any[]) {
-      // TODO: any[]
+    for (let column of Object.values(table.schema)) {
       const shouldBeForeign = foreignKeys.find(
         options => options.tableId === table._id && options.key === column.name
       )

From 6fa5727166b83b37383c33bad52f9ccc5766a005 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:19:05 +0200
Subject: [PATCH 21/35] Clean

---
 packages/server/src/utilities/rowProcessor/index.ts | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 0a7e774fc9..766e1e6ba5 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -77,7 +77,7 @@ export function processAutoColumn(
       continue
     }
     if (!schema.subtype) {
-      schema = fixAutoColumnSubType(schema as AutoColumnFieldMetadata)
+      schema = fixAutoColumnSubType(schema)
     }
     switch (schema.subtype) {
       case AutoFieldSubTypes.CREATED_BY:
@@ -100,12 +100,8 @@ export function processAutoColumn(
         break
       case AutoFieldSubTypes.AUTO_ID:
         if (creating) {
-          const safeSchema = schema as AutoColumnFieldMetadata
-
-          safeSchema.lastID = !safeSchema.lastID
-            ? BASE_AUTO_ID
-            : safeSchema.lastID + 1
-          row[key] = safeSchema.lastID
+          schema.lastID = !schema.lastID ? BASE_AUTO_ID : schema.lastID + 1
+          row[key] = schema.lastID
         }
         break
     }

From 9391854f46552056f91c266bc12ef195556df148 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:21:19 +0200
Subject: [PATCH 22/35] Types

---
 packages/server/src/utilities/rowProcessor/utils.ts | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index f93fdc4c46..1232e674ac 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -54,14 +54,15 @@ export function processFormulas(
     rowArray = rows
   }
   for (let [column, schema] of Object.entries(table.schema)) {
-    const isStaticFormula =
+    const isStatic =
       schema.type === FieldTypes.FORMULA &&
       schema.formulaType === FormulaTypes.STATIC
+
     if (
       schema.type !== FieldTypes.FORMULA ||
       schema.formula == null ||
-      (dynamic && isStaticFormula) ||
-      (!dynamic && !isStaticFormula)
+      (dynamic && isStatic) ||
+      (!dynamic && !isStatic)
     ) {
       continue
     }

From 715db97436e992e4aeb4163ba42267991c321833 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:21:51 +0200
Subject: [PATCH 23/35] Clean code

---
 packages/server/src/utilities/rowProcessor/utils.ts | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 1232e674ac..48697af6a9 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -54,12 +54,13 @@ export function processFormulas(
     rowArray = rows
   }
   for (let [column, schema] of Object.entries(table.schema)) {
-    const isStatic =
-      schema.type === FieldTypes.FORMULA &&
-      schema.formulaType === FormulaTypes.STATIC
+    if (schema.type !== FieldTypes.FORMULA) {
+      continue
+    }
+
+    const isStatic = schema.formulaType === FormulaTypes.STATIC
 
     if (
-      schema.type !== FieldTypes.FORMULA ||
       schema.formula == null ||
       (dynamic && isStatic) ||
       (!dynamic && !isStatic)

From d03e0cb1f67360f2e7ec702dcd701b80597f3767 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:27:03 +0200
Subject: [PATCH 24/35] Clean code

---
 .../src/api/controllers/table/bulkFormula.ts   | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/controllers/table/bulkFormula.ts b/packages/server/src/api/controllers/table/bulkFormula.ts
index 8f03036c0d..638a87ebea 100644
--- a/packages/server/src/api/controllers/table/bulkFormula.ts
+++ b/packages/server/src/api/controllers/table/bulkFormula.ts
@@ -1,4 +1,4 @@
-import { FieldTypes, FormulaTypes } from "../../../constants"
+import { FormulaTypes } from "../../../constants"
 import { clearColumns } from "./utils"
 import { doesContainStrings } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
@@ -6,7 +6,12 @@ import isEqual from "lodash/isEqual"
 import uniq from "lodash/uniq"
 import { updateAllFormulasInTable } from "../row/staticFormula"
 import { context } from "@budibase/backend-core"
-import { FieldSchema, FormulaFieldMetadata, Table } from "@budibase/types"
+import {
+  FieldSchema,
+  FieldType,
+  FormulaFieldMetadata,
+  Table,
+} from "@budibase/types"
 import sdk from "../../../sdk"
 import { isRelationshipColumn } from "../../../db/utils"
 
@@ -14,7 +19,7 @@ function isStaticFormula(
   column: FieldSchema
 ): column is FormulaFieldMetadata & { formulaType: FormulaTypes.STATIC } {
   return (
-    column.type === FieldTypes.FORMULA &&
+    column.type === FieldType.FORMULA &&
     column.formulaType === FormulaTypes.STATIC
   )
 }
@@ -59,7 +64,7 @@ async function checkIfFormulaNeedsCleared(
   for (let removed of removedColumns) {
     let tableToUse: Table | undefined = table
     // if relationship, get the related table
-    if (removed.type === FieldTypes.LINK) {
+    if (removed.type === FieldType.LINK) {
       const removedTableId = removed.tableId
       tableToUse = tables.find(table => table._id === removedTableId)
     }
@@ -77,12 +82,13 @@ async function checkIfFormulaNeedsCleared(
     }
     for (let relatedTableId of table.relatedFormula) {
       const relatedColumns = Object.values(table.schema).filter(
-        column => (column as any).tableId === relatedTableId
+        column =>
+          column.type === FieldType.LINK && column.tableId === relatedTableId
       )
       const relatedTable = tables.find(table => table._id === relatedTableId)
       // look to see if the column was used in a relationship formula,
       // relationships won't be used for this
-      if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
+      if (relatedTable && relatedColumns && removed.type !== FieldType.LINK) {
         let relatedFormulaToRemove: string[] = []
         for (let column of relatedColumns) {
           relatedFormulaToRemove = relatedFormulaToRemove.concat(

From f3762885c68a02cd5fabe4e93adadaf6395b57bd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:29:35 +0200
Subject: [PATCH 25/35] Clean types

---
 packages/server/src/api/controllers/table/internal.ts | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts
index 81e203d405..702dd1bb9d 100644
--- a/packages/server/src/api/controllers/table/internal.ts
+++ b/packages/server/src/api/controllers/table/internal.ts
@@ -36,9 +36,7 @@ function checkAutoColumns(table: Table, oldTable?: Table) {
     if (oldSchema && oldSchema.subtype) {
       table.schema[key].subtype = oldSchema.subtype
     } else {
-      table.schema[key] = fixAutoColumnSubType(
-        schema as AutoColumnFieldMetadata
-      )
+      table.schema[key] = fixAutoColumnSubType(schema)
     }
   }
   return table
@@ -84,7 +82,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
     for (const propKey of Object.keys(tableToSave.schema)) {
       let oldColumn = oldTable.schema[propKey]
       if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) {
-        oldColumn.type = FieldTypes.AUTO as any // TODO
+        oldTable.schema[propKey].type = FieldTypes.AUTO
       }
     }
   }

From 1e8aa9eb6030a37106b47d5e2f2833a3ae1cbf97 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:36:56 +0200
Subject: [PATCH 26/35] More types

---
 packages/server/src/db/linkedRows/LinkController.ts | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts
index 34dd0c6f80..c9ad7bc71f 100644
--- a/packages/server/src/db/linkedRows/LinkController.ts
+++ b/packages/server/src/db/linkedRows/LinkController.ts
@@ -7,6 +7,7 @@ import LinkDocument from "./LinkDocument"
 import {
   Database,
   FieldSchema,
+  FieldType,
   LinkDocumentValue,
   RelationshipFieldMetadata,
   RelationshipType,
@@ -187,9 +188,7 @@ class LinkController {
 
         // if 1:N, ensure that this ID is not already attached to another record
         const linkedTable = await this._db.get<Table>(field.tableId)
-        const linkedSchema = linkedTable.schema[
-          field.fieldName!
-        ] as RelationshipFieldMetadata
+        const linkedSchema = linkedTable.schema[field.fieldName]
 
         // We need to map the global users to metadata in each app for relationships
         if (field.tableId === InternalTables.USER_METADATA) {
@@ -206,7 +205,10 @@ class LinkController {
 
         // iterate through the link IDs in the row field, see if any don't exist already
         for (let linkId of rowField) {
-          if (linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY) {
+          if (
+            linkedSchema?.type === FieldType.LINK &&
+            linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY
+          ) {
             let links = (
               (await getLinkDocuments({
                 tableId: field.tableId,
@@ -359,7 +361,7 @@ class LinkController {
           // these are the props of the table that initiated the link
           tableId: table._id!,
           fieldName: fieldName,
-        } as any)
+        } as RelationshipFieldMetadata)
 
         // update table schema after checking relationship types
         schema[fieldName] = fields.linkerField

From a3b3c176d3722c631172fc57015a3edd32446966 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:37:50 +0200
Subject: [PATCH 27/35] More types

---
 packages/server/src/db/linkedRows/linkUtils.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts
index ba0d2b3b0e..5129299520 100644
--- a/packages/server/src/db/linkedRows/linkUtils.ts
+++ b/packages/server/src/db/linkedRows/linkUtils.ts
@@ -109,8 +109,8 @@ export async function getLinkedTable(id: string, tables: Table[]) {
 export function getRelatedTableForField(table: Table, fieldName: string) {
   // look to see if its on the table, straight in the schema
   const field = table.schema[fieldName]
-  if (field != null) {
-    return (field as any).tableId
+  if (field?.type === FieldTypes.LINK) {
+    return field.tableId
   }
   for (let column of Object.values(table.schema)) {
     if (column.type === FieldTypes.LINK && column.fieldName === fieldName) {

From 514f2b0cef2ffb80c33722e53820a5c27e8dc5ee Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:48:28 +0200
Subject: [PATCH 28/35] Types

---
 .../api/controllers/row/ExternalRequest.ts    | 23 +++++++------------
 .../src/api/controllers/table/external.ts     | 20 +++++++++-------
 2 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 0fea102f7b..cf8f7b5be4 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -568,18 +568,14 @@ export class ExternalRequest<T extends Operation> {
         // need to specify where to put this back into
         column: fieldName,
       }
-      if ((field as ManyToManyRelationshipFieldMetadata).through) {
+      if (isManyToMany(field)) {
         const { tableName: throughTableName } = breakExternalTableId(
-          (field as ManyToManyRelationshipFieldMetadata).through
+          field.through
         )
         definition.through = throughTableName
         // don't support composite keys for relationships
-        definition.from =
-          (field as ManyToManyRelationshipFieldMetadata).throughTo ||
-          table.primary[0]
-        definition.to =
-          (field as ManyToManyRelationshipFieldMetadata).throughFrom ||
-          linkTable.primary[0]
+        definition.from = field.throughTo || table.primary[0]
+        definition.to = field.throughFrom || linkTable.primary[0]
         definition.fromPrimary = table.primary[0]
         definition.toPrimary = linkTable.primary[0]
       }
@@ -603,7 +599,7 @@ export class ExternalRequest<T extends Operation> {
     const primaryKey = table.primary[0]
     // make a new request to get the row with all its relationships
     // we need this to work out if any relationships need removed
-    for (let field of Object.values(table.schema)) {
+    for (const field of Object.values(table.schema)) {
       if (
         field.type !== FieldTypes.LINK ||
         !field.fieldName ||
@@ -612,16 +608,13 @@ export class ExternalRequest<T extends Operation> {
         continue
       }
       const isMany = field.relationshipType === RelationshipType.MANY_TO_MANY
-      const tableId = isMany
-        ? (field as ManyToManyRelationshipFieldMetadata).through
-        : field.tableId
+      const tableId = isMany ? field.through : field.tableId
       const { tableName: relatedTableName } = breakExternalTableId(tableId)
       // @ts-ignore
       const linkPrimaryKey = this.tables[relatedTableName].primary[0]
-      const manyKey =
-        (field as ManyToManyRelationshipFieldMetadata).throughTo || primaryKey
+
       const lookupField = isMany ? primaryKey : (field as any).foreignKey
-      const fieldName = isMany ? manyKey : field.fieldName
+      const fieldName = isMany ? field.throughTo || primaryKey : field.fieldName
       if (!lookupField || !row[lookupField]) {
         continue
       }
diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts
index dac39a4a99..e84c23b5ca 100644
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@@ -81,7 +81,9 @@ function cleanupRelationships(
       const relatedTable = Object.values(tables).find(
         table => table._id === schemaTableId
       )
-      const foreignKey = (schema as any).foreignKey
+      const foreignKey =
+        schema.relationshipType !== RelationshipType.MANY_TO_MANY &&
+        schema.foreignKey
       if (!relatedTable || !foreignKey) {
         continue
       }
@@ -183,17 +185,19 @@ function generateRelatedSchema(
 ) {
   // generate column for other table
   const relatedSchema = cloneDeep(linkColumn)
+  const isMany2Many =
+    linkColumn.relationshipType === RelationshipType.MANY_TO_MANY
   // swap them from the main link
-  if ((linkColumn as any).foreignKey) {
-    relatedSchema.fieldName = (linkColumn as any).foreignKey
+  if (!isMany2Many && linkColumn.foreignKey) {
+    relatedSchema.fieldName = linkColumn.foreignKey
     relatedSchema.foreignKey = linkColumn.fieldName
   }
   // is many to many
-  else {
+  else if (isMany2Many) {
     // don't need to copy through, already got it
-    relatedSchema.fieldName = (linkColumn as any).throughTo
-    relatedSchema.throughTo = (linkColumn as any).throughFrom
-    relatedSchema.throughFrom = (linkColumn as any).throughTo
+    relatedSchema.fieldName = linkColumn.throughTo
+    relatedSchema.throughTo = linkColumn.throughFrom
+    relatedSchema.throughFrom = linkColumn.throughTo
   }
   relatedSchema.relationshipType = otherRelationshipType(
     linkColumn.relationshipType
@@ -203,7 +207,7 @@ function generateRelatedSchema(
   table.schema[columnName] = relatedSchema
 }
 
-function isRelationshipSetup(column: FieldSchema) {
+function isRelationshipSetup(column: RelationshipFieldMetadata) {
   return (column as any).foreignKey || (column as any).through
 }
 

From fcaf657e539761557f3d80df5fa0888dbf4afcbc Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:53:14 +0200
Subject: [PATCH 29/35] More types

---
 .../server/src/api/controllers/row/ExternalRequest.ts    | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index cf8f7b5be4..220e0cf2b9 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -560,10 +560,7 @@ export class ExternalRequest<T extends Operation> {
       if (!table.primary || !linkTable.primary) {
         continue
       }
-      const definition: any = {
-        // if no foreign key specified then use the name of the field in other table
-        from: (field as any).foreignKey || table.primary[0],
-        to: field.fieldName,
+      const definition: RelationshipsJson = {
         tableName: linkTableName,
         // need to specify where to put this back into
         column: fieldName,
@@ -578,6 +575,10 @@ export class ExternalRequest<T extends Operation> {
         definition.to = field.throughFrom || linkTable.primary[0]
         definition.fromPrimary = table.primary[0]
         definition.toPrimary = linkTable.primary[0]
+      } else {
+        // if no foreign key specified then use the name of the field in other table
+        definition.from = (field as any).foreignKey || table.primary[0]
+        definition.to = field.fieldName
       }
       relationships.push(definition)
     }

From 8fa71e2e1b041c8b81885f0f8757feac3edc7f5e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:54:51 +0200
Subject: [PATCH 30/35] Clean

---
 packages/server/src/api/controllers/row/ExternalRequest.ts | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 220e0cf2b9..fd9c5bbb52 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -614,7 +614,7 @@ export class ExternalRequest<T extends Operation> {
       // @ts-ignore
       const linkPrimaryKey = this.tables[relatedTableName].primary[0]
 
-      const lookupField = isMany ? primaryKey : (field as any).foreignKey
+      const lookupField = isMany ? primaryKey : field.foreignKey
       const fieldName = isMany ? field.throughTo || primaryKey : field.fieldName
       if (!lookupField || !row[lookupField]) {
         continue
@@ -629,10 +629,7 @@ export class ExternalRequest<T extends Operation> {
       })
       // this is the response from knex if no rows found
       const rows = !response[0].read ? response : []
-      const storeTo = isMany
-        ? (field as ManyToManyRelationshipFieldMetadata).throughFrom ||
-          linkPrimaryKey
-        : fieldName
+      const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName
       related[storeTo] = { rows, isMany, tableId }
     }
     return related

From 0720aa8b98abd9d5c26f58ac8ade57af875fa428 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 13:55:35 +0200
Subject: [PATCH 31/35] Remove any

---
 packages/server/src/api/controllers/row/ExternalRequest.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index fd9c5bbb52..71532c37d5 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -577,7 +577,7 @@ export class ExternalRequest<T extends Operation> {
         definition.toPrimary = linkTable.primary[0]
       } else {
         // if no foreign key specified then use the name of the field in other table
-        definition.from = (field as any).foreignKey || table.primary[0]
+        definition.from = field.foreignKey || table.primary[0]
         definition.to = field.fieldName
       }
       relationships.push(definition)

From 1bfde57391a15ea7a21698af465321fcc581767b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Oct 2023 14:08:03 +0200
Subject: [PATCH 32/35] Remove anys on tests

---
 packages/server/src/api/routes/tests/row.spec.ts | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index f1f56ef1fd..a2bda2bc83 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1567,16 +1567,15 @@ describe.each([
       (): Record<string, FieldSchema> => ({
         user: {
           name: "user",
-          relationshipType: RelationshipType.ONE_TO_MANY,
           type: FieldType.BB_REFERENCE,
           subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
-        } as any,
+        },
         users: {
           name: "users",
           type: FieldType.BB_REFERENCE,
           subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
-          relationshipType: RelationshipType.MANY_TO_MANY,
-        } as any,
+          // TODO: users when all merged
+        },
       }),
       () => config.createUser(),
       (row: Row) => ({

From b2a19eeb825a58355641271b00017d5e834c0c7a Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 11 Oct 2023 15:37:45 +0000
Subject: [PATCH 33/35] Bump version to 2.11.20

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index f56bd4e540..e65f5bb265 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.11.19",
+  "version": "2.11.20",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From c01fafef3a5226f041727cc932cf2b9266bc7ec2 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martin@budibase.com>
Date: Wed, 11 Oct 2023 16:41:45 +0100
Subject: [PATCH 34/35] Delete .github/workflows/check_unreleased_changes.yml

---
 .../workflows/check_unreleased_changes.yml    | 29 -------------------
 1 file changed, 29 deletions(-)
 delete mode 100644 .github/workflows/check_unreleased_changes.yml

diff --git a/.github/workflows/check_unreleased_changes.yml b/.github/workflows/check_unreleased_changes.yml
deleted file mode 100644
index d558330545..0000000000
--- a/.github/workflows/check_unreleased_changes.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-name: check_unreleased_changes
-
-on:
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  check_unreleased:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check for unreleased changes
-        env:
-          REPO: "Budibase/budibase"
-          TOKEN: ${{ secrets.GITHUB_TOKEN }} 
-        run: |
-          RELEASE_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
-              "https://api.github.com/repos/$REPO/releases/latest" | \
-              jq -r .published_at)
-          COMMIT_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
-              "https://api.github.com/repos/$REPO/commits/master" | \
-              jq -r .commit.committer.date)
-          RELEASE_SECONDS=$(date --date="$RELEASE_TIMESTAMP" "+%s")
-          COMMIT_SECONDS=$(date --date="$COMMIT_TIMESTAMP" "+%s")
-          if (( COMMIT_SECONDS > RELEASE_SECONDS )); then
-              echo "There are unreleased changes. Please release these changes before merging."
-              exit 1
-          fi
-          echo "No unreleased changes detected."

From c143b3eca822ddba027aa23f0182d393c003c030 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 11 Oct 2023 15:42:03 +0000
Subject: [PATCH 35/35] Bump version to 2.11.21

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index e65f5bb265..6e3bd1399c 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.11.20",
+  "version": "2.11.21",
   "npmClient": "yarn",
   "packages": [
     "packages/*"