diff --git a/.github/workflows/check_unreleased_changes.yml b/.github/workflows/check_unreleased_changes.yml deleted file mode 100644 index d558330545..0000000000 --- a/.github/workflows/check_unreleased_changes.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: check_unreleased_changes - -on: - pull_request: - branches: - - master - -jobs: - check_unreleased: - runs-on: ubuntu-latest - steps: - - name: Check for unreleased changes - env: - REPO: "Budibase/budibase" - TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - RELEASE_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \ - "https://api.github.com/repos/$REPO/releases/latest" | \ - jq -r .published_at) - COMMIT_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \ - "https://api.github.com/repos/$REPO/commits/master" | \ - jq -r .commit.committer.date) - RELEASE_SECONDS=$(date --date="$RELEASE_TIMESTAMP" "+%s") - COMMIT_SECONDS=$(date --date="$COMMIT_TIMESTAMP" "+%s") - if (( COMMIT_SECONDS > RELEASE_SECONDS )); then - echo "There are unreleased changes. Please release these changes before merging." - exit 1 - fi - echo "No unreleased changes detected." diff --git a/lerna.json b/lerna.json index f56bd4e540..6e3bd1399c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.19", + "version": "2.11.21", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 2ad1afe202..71532c37d5 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -5,8 +5,11 @@ import { FieldType, FilterType, IncludeRelationship, + ManyToManyRelationshipFieldMetadata, + OneToManyRelationshipFieldMetadata, Operation, PaginationJson, + RelationshipFieldMetadata, RelationshipsJson, RelationshipType, Row, @@ -254,12 +257,20 @@ function fixArrayTypes(row: Row, table: Table) { return row } -function isOneSide(field: FieldSchema) { +function isOneSide( + field: RelationshipFieldMetadata +): field is OneToManyRelationshipFieldMetadata { return ( field.relationshipType && field.relationshipType.split("-")[0] === "one" ) } +function isManyToMany( + field: RelationshipFieldMetadata +): field is ManyToManyRelationshipFieldMetadata { + return !!(field as ManyToManyRelationshipFieldMetadata).through +} + function isEditableColumn(column: FieldSchema) { const isExternalAutoColumn = column.autocolumn && @@ -352,11 +363,11 @@ export class ExternalRequest { } } // many to many - else if (field.through) { + else if (isManyToMany(field)) { // we're not inserting a doc, will be a bunch of update calls const otherKey: string = field.throughFrom || linkTablePrimary const thisKey: string = field.throughTo || tablePrimary - row[key].forEach((relationship: any) => { + for (const relationship of row[key]) { manyRelationships.push({ tableId: field.through || field.tableId, isUpdate: false, @@ -365,14 +376,14 @@ export class ExternalRequest { // leave the ID for enrichment later [thisKey]: `{{ literal ${tablePrimary} }}`, }) - }) + } } // many to one else { const thisKey: string = "id" // @ts-ignore const otherKey: string = field.fieldName - row[key].forEach((relationship: any) => { + for (const relationship of row[key]) { manyRelationships.push({ tableId: field.tableId, isUpdate: true, @@ -381,7 +392,7 @@ export class ExternalRequest { // leave the ID for enrichment later [otherKey]: `{{ literal ${tablePrimary} }}`, }) - }) + } } } // we return the relationships that may need to be created in the through table @@ -549,15 +560,12 @@ export class ExternalRequest { if (!table.primary || !linkTable.primary) { continue } - const definition: any = { - // if no foreign key specified then use the name of the field in other table - from: field.foreignKey || table.primary[0], - to: field.fieldName, + const definition: RelationshipsJson = { tableName: linkTableName, // need to specify where to put this back into column: fieldName, } - if (field.through) { + if (isManyToMany(field)) { const { tableName: throughTableName } = breakExternalTableId( field.through ) @@ -567,6 +575,10 @@ export class ExternalRequest { definition.to = field.throughFrom || linkTable.primary[0] definition.fromPrimary = table.primary[0] definition.toPrimary = linkTable.primary[0] + } else { + // if no foreign key specified then use the name of the field in other table + definition.from = field.foreignKey || table.primary[0] + definition.to = field.fieldName } relationships.push(definition) } @@ -588,7 +600,7 @@ export class ExternalRequest { const primaryKey = table.primary[0] // make a new request to get the row with all its relationships // we need this to work out if any relationships need removed - for (let field of Object.values(table.schema)) { + for (const field of Object.values(table.schema)) { if ( field.type !== FieldTypes.LINK || !field.fieldName || @@ -601,9 +613,9 @@ export class ExternalRequest { const { tableName: relatedTableName } = breakExternalTableId(tableId) // @ts-ignore const linkPrimaryKey = this.tables[relatedTableName].primary[0] - const manyKey = field.throughTo || primaryKey + const lookupField = isMany ? primaryKey : field.foreignKey - const fieldName = isMany ? manyKey : field.fieldName + const fieldName = isMany ? field.throughTo || primaryKey : field.fieldName if (!lookupField || !row[lookupField]) { continue } diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 5f10fd9ad4..1243d18847 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -4,6 +4,8 @@ import { context } from "@budibase/backend-core" import { Ctx, FieldType, + ManyToOneRelationshipFieldMetadata, + OneToManyRelationshipFieldMetadata, Row, SearchFilters, Table, @@ -19,7 +21,14 @@ function isForeignKey(key: string, table: Table) { const relationships = Object.values(table.schema).filter( column => column.type === FieldType.LINK ) - return relationships.some(relationship => relationship.foreignKey === key) + return relationships.some( + relationship => + ( + relationship as + | OneToManyRelationshipFieldMetadata + | ManyToOneRelationshipFieldMetadata + ).foreignKey === key + ) } validateJs.extend(validateJs.validators.datetime, { diff --git a/packages/server/src/api/controllers/table/bulkFormula.ts b/packages/server/src/api/controllers/table/bulkFormula.ts index e44cb95391..638a87ebea 100644 --- a/packages/server/src/api/controllers/table/bulkFormula.ts +++ b/packages/server/src/api/controllers/table/bulkFormula.ts @@ -1,4 +1,4 @@ -import { FieldTypes, FormulaTypes } from "../../../constants" +import { FormulaTypes } from "../../../constants" import { clearColumns } from "./utils" import { doesContainStrings } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" @@ -6,12 +6,20 @@ import isEqual from "lodash/isEqual" import uniq from "lodash/uniq" import { updateAllFormulasInTable } from "../row/staticFormula" import { context } from "@budibase/backend-core" -import { FieldSchema, Table } from "@budibase/types" +import { + FieldSchema, + FieldType, + FormulaFieldMetadata, + Table, +} from "@budibase/types" import sdk from "../../../sdk" +import { isRelationshipColumn } from "../../../db/utils" -function isStaticFormula(column: FieldSchema) { +function isStaticFormula( + column: FieldSchema +): column is FormulaFieldMetadata & { formulaType: FormulaTypes.STATIC } { return ( - column.type === FieldTypes.FORMULA && + column.type === FieldType.FORMULA && column.formulaType === FormulaTypes.STATIC ) } @@ -56,8 +64,9 @@ async function checkIfFormulaNeedsCleared( for (let removed of removedColumns) { let tableToUse: Table | undefined = table // if relationship, get the related table - if (removed.type === FieldTypes.LINK) { - tableToUse = tables.find(table => table._id === removed.tableId) + if (removed.type === FieldType.LINK) { + const removedTableId = removed.tableId + tableToUse = tables.find(table => table._id === removedTableId) } if (!tableToUse) { continue @@ -73,17 +82,18 @@ async function checkIfFormulaNeedsCleared( } for (let relatedTableId of table.relatedFormula) { const relatedColumns = Object.values(table.schema).filter( - column => column.tableId === relatedTableId + column => + column.type === FieldType.LINK && column.tableId === relatedTableId ) const relatedTable = tables.find(table => table._id === relatedTableId) // look to see if the column was used in a relationship formula, // relationships won't be used for this - if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) { + if (relatedTable && relatedColumns && removed.type !== FieldType.LINK) { let relatedFormulaToRemove: string[] = [] for (let column of relatedColumns) { relatedFormulaToRemove = relatedFormulaToRemove.concat( getFormulaThatUseColumn(relatedTable, [ - column.fieldName!, + (column as any).fieldName!, removed.name, ]) ) @@ -116,7 +126,7 @@ async function updateRelatedFormulaLinksOnTables( const initialTables = cloneDeep(tables) // first find the related column names const relatedColumns = Object.values(table.schema).filter( - col => col.type === FieldTypes.LINK + isRelationshipColumn ) // we start by removing the formula field from all tables for (let otherTable of tables) { @@ -135,6 +145,7 @@ async function updateRelatedFormulaLinksOnTables( if (!columns || columns.length === 0) { continue } + const relatedTable = tables.find( related => related._id === relatedCol.tableId ) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index a332247a26..d877ffd707 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -15,13 +15,16 @@ import { handleRequest } from "../row/external" import { context, events } from "@budibase/backend-core" import { isRows, isSchema, parse } from "../../../utilities/schema" import { - AutoReason, Datasource, FieldSchema, ImportRowsRequest, ImportRowsResponse, + ManyToManyRelationshipFieldMetadata, + ManyToOneRelationshipFieldMetadata, + OneToManyRelationshipFieldMetadata, Operation, QueryJson, + RelationshipFieldMetadata, RelationshipType, RenameColumn, SaveTableRequest, @@ -76,10 +79,13 @@ function cleanupRelationships( schema.type === FieldTypes.LINK && (!oldTable || table.schema[key] == null) ) { + const schemaTableId = schema.tableId const relatedTable = Object.values(tables).find( - table => table._id === schema.tableId + table => table._id === schemaTableId ) - const foreignKey = schema.foreignKey + const foreignKey = + schema.relationshipType !== RelationshipType.MANY_TO_MANY && + schema.foreignKey if (!relatedTable || !foreignKey) { continue } @@ -118,7 +124,7 @@ function otherRelationshipType(type?: string) { function generateManyLinkSchema( datasource: Datasource, - column: FieldSchema, + column: ManyToManyRelationshipFieldMetadata, table: Table, relatedTable: Table ): Table { @@ -153,10 +159,12 @@ function generateManyLinkSchema( } function generateLinkSchema( - column: FieldSchema, + column: + | OneToManyRelationshipFieldMetadata + | ManyToOneRelationshipFieldMetadata, table: Table, relatedTable: Table, - type: RelationshipType + type: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE ) { if (!table.primary || !relatedTable.primary) { throw new Error("Unable to generate link schema, no primary keys") @@ -172,20 +180,22 @@ function generateLinkSchema( } function generateRelatedSchema( - linkColumn: FieldSchema, + linkColumn: RelationshipFieldMetadata, table: Table, relatedTable: Table, columnName: string ) { // generate column for other table const relatedSchema = cloneDeep(linkColumn) + const isMany2Many = + linkColumn.relationshipType === RelationshipType.MANY_TO_MANY // swap them from the main link - if (linkColumn.foreignKey) { + if (!isMany2Many && linkColumn.foreignKey) { relatedSchema.fieldName = linkColumn.foreignKey relatedSchema.foreignKey = linkColumn.fieldName } // is many to many - else { + else if (isMany2Many) { // don't need to copy through, already got it relatedSchema.fieldName = linkColumn.throughTo relatedSchema.throughTo = linkColumn.throughFrom @@ -199,8 +209,8 @@ function generateRelatedSchema( table.schema[columnName] = relatedSchema } -function isRelationshipSetup(column: FieldSchema) { - return column.foreignKey || column.through +function isRelationshipSetup(column: RelationshipFieldMetadata) { + return (column as any).foreignKey || (column as any).through } export async function save(ctx: UserCtx) { @@ -259,14 +269,15 @@ export async function save(ctx: UserCtx) { if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) { continue } + const schemaTableId = schema.tableId const relatedTable = Object.values(tables).find( - table => table._id === schema.tableId + table => table._id === schemaTableId ) if (!relatedTable) { continue } const relatedColumnName = schema.fieldName! - const relationType = schema.relationshipType! + const relationType = schema.relationshipType if (relationType === RelationshipType.MANY_TO_MANY) { const junctionTable = generateManyLinkSchema( datasource, diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 7765e630db..d8f1212825 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -80,10 +80,10 @@ export async function save(ctx: UserCtx) { // make sure that types don't change of a column, have to remove // the column if you want to change the type if (oldTable && oldTable.schema) { - for (let propKey of Object.keys(tableToSave.schema)) { + for (const propKey of Object.keys(tableToSave.schema)) { let oldColumn = oldTable.schema[propKey] if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) { - oldColumn.type = FieldTypes.AUTO + oldTable.schema[propKey].type = FieldTypes.AUTO } } } diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index fa329dbb4b..a2bda2bc83 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -6,6 +6,8 @@ import * as setup from "./utilities" import { context, InternalTable, roles, tenancy } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { + AutoFieldSubTypes, + FieldSchema, FieldType, FieldTypeSubtypes, MonthlyQuotaName, @@ -171,7 +173,7 @@ describe.each([ "Row ID": { name: "Row ID", type: FieldType.NUMBER, - subtype: "autoID", + subtype: AutoFieldSubTypes.AUTO_ID, icon: "ri-magic-line", autocolumn: true, constraints: { @@ -272,27 +274,27 @@ describe.each([ isInternal && it("row values are coerced", async () => { - const str = { + const str: FieldSchema = { type: FieldType.STRING, name: "str", constraints: { type: "string", presence: false }, } - const attachment = { + const attachment: FieldSchema = { type: FieldType.ATTACHMENT, name: "attachment", constraints: { type: "array", presence: false }, } - const bool = { + const bool: FieldSchema = { type: FieldType.BOOLEAN, name: "boolean", constraints: { type: "boolean", presence: false }, } - const number = { + const number: FieldSchema = { type: FieldType.NUMBER, name: "str", constraints: { type: "number", presence: false }, } - const datetime = { + const datetime: FieldSchema = { type: FieldType.DATETIME, name: "datetime", constraints: { @@ -301,7 +303,7 @@ describe.each([ datetime: { earliest: "", latest: "" }, }, } - const arrayField = { + const arrayField: FieldSchema = { type: FieldType.ARRAY, constraints: { type: "array", @@ -311,8 +313,7 @@ describe.each([ name: "Sample Tags", sortable: false, } - const optsField = { - fieldName: "Sample Opts", + const optsField: FieldSchema = { name: "Sample Opts", type: FieldType.OPTIONS, constraints: { @@ -1534,7 +1535,7 @@ describe.each([ describe.each([ [ "relationship fields", - () => ({ + (): Record => ({ user: { name: "user", relationshipType: RelationshipType.ONE_TO_MANY, @@ -1563,10 +1564,9 @@ describe.each([ ], [ "bb reference fields", - () => ({ + (): Record => ({ user: { name: "user", - relationshipType: RelationshipType.ONE_TO_MANY, type: FieldType.BB_REFERENCE, subtype: FieldTypeSubtypes.BB_REFERENCE.USER, }, @@ -1574,7 +1574,7 @@ describe.each([ name: "users", type: FieldType.BB_REFERENCE, subtype: FieldTypeSubtypes.BB_REFERENCE.USER, - relationshipType: RelationshipType.MANY_TO_MANY, + // TODO: users when all merged }, }), () => config.createUser(), diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 12472eaa21..8546154d54 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -1,5 +1,10 @@ import { events, context } from "@budibase/backend-core" -import { FieldType, Table, ViewCalculation } from "@budibase/types" +import { + FieldType, + RelationshipType, + Table, + ViewCalculation, +} from "@budibase/types" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" const { basicTable } = setup.structures @@ -381,9 +386,10 @@ describe("/tables", () => { }, TestTable: { type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, name: "TestTable", fieldName: "TestTable", - tableId: testTable._id, + tableId: testTable._id!, constraints: { type: "array", }, diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index c8c17e1d32..326389996d 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -1,6 +1,11 @@ import { objectStore, roles, constants } from "@budibase/backend-core" import { FieldType as FieldTypes } from "@budibase/types" -export { FieldType as FieldTypes, RelationshipType } from "@budibase/types" +export { + FieldType as FieldTypes, + RelationshipType, + AutoFieldSubTypes, + FormulaTypes, +} from "@budibase/types" export enum FilterTypes { STRING = "string", @@ -39,11 +44,6 @@ export const SwitchableTypes = CanSwitchTypes.reduce((prev, current) => prev ? prev.concat(current) : current ) -export enum FormulaTypes { - STATIC = "static", - DYNAMIC = "dynamic", -} - export enum AuthTypes { APP = "app", BUILDER = "builder", @@ -132,14 +132,6 @@ export const USERS_TABLE_SCHEMA = { primaryDisplay: "email", } -export enum AutoFieldSubTypes { - CREATED_BY = "createdBy", - CREATED_AT = "createdAt", - UPDATED_BY = "updatedBy", - UPDATED_AT = "updatedAt", - AUTO_ID = "autoID", -} - export enum AutoFieldDefaultNames { CREATED_BY = "Created By", CREATED_AT = "Created At", diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index a4821667ff..48d4876de1 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -7,7 +7,13 @@ import { employeeImport } from "./employeeImport" import { jobsImport } from "./jobsImport" import { expensesImport } from "./expensesImport" import { db as dbCore } from "@budibase/backend-core" -import { Table, Row, RelationshipType } from "@budibase/types" +import { + Table, + Row, + RelationshipType, + FieldType, + TableSchema, +} from "@budibase/types" export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" @@ -28,7 +34,11 @@ export const DEFAULT_BB_DATASOURCE = defaultDatasource function syncLastIds(table: Table, rowCount: number) { Object.keys(table.schema).forEach(key => { const entry = table.schema[key] - if (entry.autocolumn && entry.subtype == "autoID") { + if ( + entry.autocolumn && + entry.type === FieldType.NUMBER && + entry.subtype == AutoFieldSubTypes.AUTO_ID + ) { entry.lastID = rowCount } }) @@ -42,7 +52,7 @@ async function tableImport(table: Table, data: Row[]) { } // AUTO COLUMNS -const AUTO_COLUMNS = { +const AUTO_COLUMNS: TableSchema = { "Created At": { name: "Created At", type: FieldTypes.DATETIME, diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts index 457819251a..c9ad7bc71f 100644 --- a/packages/server/src/db/linkedRows/LinkController.ts +++ b/packages/server/src/db/linkedRows/LinkController.ts @@ -7,7 +7,9 @@ import LinkDocument from "./LinkDocument" import { Database, FieldSchema, + FieldType, LinkDocumentValue, + RelationshipFieldMetadata, RelationshipType, Row, Table, @@ -133,7 +135,10 @@ class LinkController { * Given the link field of this table, and the link field of the linked table, this makes sure * the state of relationship type is accurate on both. */ - handleRelationshipType(linkerField: FieldSchema, linkedField: FieldSchema) { + handleRelationshipType( + linkerField: RelationshipFieldMetadata, + linkedField: RelationshipFieldMetadata + ) { if ( !linkerField.relationshipType || linkerField.relationshipType === RelationshipType.MANY_TO_MANY @@ -183,7 +188,7 @@ class LinkController { // if 1:N, ensure that this ID is not already attached to another record const linkedTable = await this._db.get(field.tableId) - const linkedSchema = linkedTable.schema[field.fieldName!] + const linkedSchema = linkedTable.schema[field.fieldName] // We need to map the global users to metadata in each app for relationships if (field.tableId === InternalTables.USER_METADATA) { @@ -200,7 +205,10 @@ class LinkController { // iterate through the link IDs in the row field, see if any don't exist already for (let linkId of rowField) { - if (linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY) { + if ( + linkedSchema?.type === FieldType.LINK && + linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY + ) { let links = ( (await getLinkDocuments({ tableId: field.tableId, @@ -291,7 +299,7 @@ class LinkController { */ async removeFieldFromTable(fieldName: string) { let oldTable = this._oldTable - let field = oldTable?.schema[fieldName] as FieldSchema + let field = oldTable?.schema[fieldName] as RelationshipFieldMetadata const linkDocs = await this.getTableLinkDocs() let toDelete = linkDocs.filter(linkDoc => { let correctFieldName = @@ -351,9 +359,9 @@ class LinkController { name: field.fieldName, type: FieldTypes.LINK, // these are the props of the table that initiated the link - tableId: table._id, + tableId: table._id!, fieldName: fieldName, - }) + } as RelationshipFieldMetadata) // update table schema after checking relationship types schema[fieldName] = fields.linkerField diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index c7db7d522a..5129299520 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -1,13 +1,9 @@ -import { ViewName, getQueryIndex } from "../utils" +import { ViewName, getQueryIndex, isRelationshipColumn } from "../utils" import { FieldTypes } from "../../constants" import { createLinkView } from "../views/staticViews" import { context, logging } from "@budibase/backend-core" -import { - FieldSchema, - LinkDocument, - LinkDocumentValue, - Table, -} from "@budibase/types" +import { LinkDocument, LinkDocumentValue, Table } from "@budibase/types" + export { createLinkView } from "../views/staticViews" /** @@ -93,7 +89,7 @@ export function getUniqueByProp(array: any[], prop: string) { export function getLinkedTableIDs(table: Table) { return Object.values(table.schema) - .filter((column: FieldSchema) => column.type === FieldTypes.LINK) + .filter(isRelationshipColumn) .map(column => column.tableId) } @@ -113,7 +109,7 @@ export async function getLinkedTable(id: string, tables: Table[]) { export function getRelatedTableForField(table: Table, fieldName: string) { // look to see if its on the table, straight in the schema const field = table.schema[fieldName] - if (field != null) { + if (field?.type === FieldTypes.LINK) { return field.tableId } for (let column of Object.values(table.schema)) { diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index abea725707..428c955eb2 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -1,6 +1,12 @@ import newid from "./newid" import { db as dbCore } from "@budibase/backend-core" -import { DocumentType, VirtualDocumentType } from "@budibase/types" +import { + DocumentType, + FieldSchema, + RelationshipFieldMetadata, + VirtualDocumentType, +} from "@budibase/types" +import { FieldTypes } from "../constants" export { DocumentType, VirtualDocumentType } from "@budibase/types" type Optional = string | null @@ -307,3 +313,9 @@ export function extractViewInfoFromID(viewId: string) { tableId: res!.groups!["tableId"], } } + +export function isRelationshipColumn( + column: FieldSchema +): column is RelationshipFieldMetadata { + return column.type === FieldTypes.LINK +} diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 86a43c8c42..bcd1c14389 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -111,7 +111,7 @@ describe("postgres integrations", () => { fieldName: oneToManyRelationshipInfo.fieldName, name: "oneToManyRelation", relationshipType: RelationshipType.ONE_TO_MANY, - tableId: oneToManyRelationshipInfo.table._id, + tableId: oneToManyRelationshipInfo.table._id!, main: true, }, manyToOneRelation: { @@ -122,7 +122,7 @@ describe("postgres integrations", () => { fieldName: manyToOneRelationshipInfo.fieldName, name: "manyToOneRelation", relationshipType: RelationshipType.MANY_TO_ONE, - tableId: manyToOneRelationshipInfo.table._id, + tableId: manyToOneRelationshipInfo.table._id!, main: true, }, manyToManyRelation: { @@ -133,7 +133,7 @@ describe("postgres integrations", () => { fieldName: manyToManyRelationshipInfo.fieldName, name: "manyToManyRelation", relationshipType: RelationshipType.MANY_TO_MANY, - tableId: manyToManyRelationshipInfo.table._id, + tableId: manyToManyRelationshipInfo.table._id!, main: true, }, }, @@ -250,6 +250,7 @@ describe("postgres integrations", () => { id: { name: "id", type: FieldType.AUTO, + autocolumn: true, }, }, sourceId: postgresDatasource._id, diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index 4383167f4a..5915a6e868 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -1,5 +1,11 @@ import { Knex, knex } from "knex" -import { Operation, QueryJson, RenameColumn, Table } from "@budibase/types" +import { + NumberFieldMetadata, + Operation, + QueryJson, + RenameColumn, + Table, +} from "@budibase/types" import { breakExternalTableId } from "../utils" import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder @@ -15,7 +21,7 @@ function generateSchema( let primaryKey = table && table.primary ? table.primary[0] : null const columns = Object.values(table.schema) // all columns in a junction table will be meta - let metaCols = columns.filter(col => col.meta) + let metaCols = columns.filter(col => (col as NumberFieldMetadata).meta) let isJunction = metaCols.length === columns.length // can't change primary once its set for now if (primaryKey && !oldTable && !isJunction) { @@ -25,7 +31,9 @@ function generateSchema( } // check if any columns need added - const foreignKeys = Object.values(table.schema).map(col => col.foreignKey) + const foreignKeys = Object.values(table.schema).map( + col => (col as any).foreignKey + ) for (let [key, column] of Object.entries(table.schema)) { // skip things that are already correct const oldColumn = oldTable ? oldTable.schema[key] : null diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 3e8ec15423..38f0a9d5ac 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -249,7 +249,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { ) } - private internalConvertType(column: OracleColumn): { type: FieldTypes } { + private internalConvertType(column: OracleColumn) { if (this.isBooleanType(column)) { return { type: FieldTypes.BOOLEAN } } @@ -307,6 +307,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { }, ...this.internalConvertType(oracleColumn), } + table.schema[columnName] = fieldSchema } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 51e418c324..4049d898fb 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -1,10 +1,11 @@ import cloneDeep from "lodash/cloneDeep" import validateJs from "validate.js" -import { FieldType, Row, Table, TableSchema } from "@budibase/types" +import { Row, Table, TableSchema } from "@budibase/types" import { FieldTypes } from "../../../constants" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." +import { isRelationshipColumn } from "../../../db/utils" export async function getDatasourceAndQuery(json: any) { const datasourceId = json.endpoint.datasourceId @@ -50,10 +51,10 @@ export function cleanExportRows( } function isForeignKey(key: string, table: Table) { - const relationships = Object.values(table.schema).filter( - column => column.type === FieldType.LINK + const relationships = Object.values(table.schema).filter(isRelationshipColumn) + return relationships.some( + relationship => (relationship as any).foreignKey === key ) - return relationships.some(relationship => relationship.foreignKey === key) } export async function validate({ diff --git a/packages/server/src/sdk/app/tables/tests/validation.spec.ts b/packages/server/src/sdk/app/tables/tests/validation.spec.ts index ffc34d0afd..5347eede90 100644 --- a/packages/server/src/sdk/app/tables/tests/validation.spec.ts +++ b/packages/server/src/sdk/app/tables/tests/validation.spec.ts @@ -1,6 +1,6 @@ import { populateExternalTableSchemas } from "../validation" import { cloneDeep } from "lodash/fp" -import { Datasource, Table } from "@budibase/types" +import { AutoReason, Datasource, Table } from "@budibase/types" import { isEqual } from "lodash" const SCHEMA = { @@ -109,7 +109,7 @@ describe("validation and update of external table schemas", () => { const response = populateExternalTableSchemas(cloneDeep(SCHEMA) as any) const foreignKey = getForeignKeyColumn(response) expect(foreignKey.autocolumn).toBe(true) - expect(foreignKey.autoReason).toBe("foreign_key") + expect(foreignKey.autoReason).toBe(AutoReason.FOREIGN_KEY) noOtherTableChanges(response) }) diff --git a/packages/server/src/sdk/app/tables/validation.ts b/packages/server/src/sdk/app/tables/validation.ts index 56f3e84c7a..1609bdfcda 100644 --- a/packages/server/src/sdk/app/tables/validation.ts +++ b/packages/server/src/sdk/app/tables/validation.ts @@ -1,11 +1,9 @@ import { AutoReason, Datasource, - FieldSchema, FieldType, RelationshipType, } from "@budibase/types" -import { FieldTypes } from "../../../constants" function checkForeignKeysAreAutoColumns(datasource: Datasource) { if (!datasource.entities) { @@ -15,10 +13,11 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) { // make sure all foreign key columns are marked as auto columns const foreignKeys: { tableId: string; key: string }[] = [] for (let table of tables) { - const relationships = Object.values(table.schema).filter( - column => column.type === FieldType.LINK - ) - relationships.forEach(relationship => { + Object.values(table.schema).forEach(column => { + if (column.type !== FieldType.LINK) { + return + } + const relationship = column if (relationship.relationshipType === RelationshipType.MANY_TO_MANY) { const tableId = relationship.through! foreignKeys.push({ key: relationship.throughTo!, tableId }) @@ -36,7 +35,7 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) { } // now make sure schemas are all accurate - for (let table of tables) { + for (const table of tables) { for (let column of Object.values(table.schema)) { const shouldBeForeign = foreignKeys.find( options => options.tableId === table._id && options.key === column.name diff --git a/packages/server/src/sdk/app/views/tests/views.spec.ts b/packages/server/src/sdk/app/views/tests/views.spec.ts index 2314f362e9..8fcc6405ef 100644 --- a/packages/server/src/sdk/app/views/tests/views.spec.ts +++ b/packages/server/src/sdk/app/views/tests/views.spec.ts @@ -1,5 +1,11 @@ import _ from "lodash" -import { FieldType, Table, TableSchema, ViewV2 } from "@budibase/types" +import { + FieldSchema, + FieldType, + Table, + TableSchema, + ViewV2, +} from "@budibase/types" import { generator } from "@budibase/backend-core/tests" import { enrichSchema, syncSchema } from ".." @@ -316,7 +322,7 @@ describe("table sdk", () => { ...basicView, } - const newTableSchema = { + const newTableSchema: TableSchema = { ...basicTable.schema, newField1: { type: FieldType.STRING, @@ -403,7 +409,7 @@ describe("table sdk", () => { }, } - const newTableSchema = { + const newTableSchema: TableSchema = { ...basicTable.schema, newField1: { type: FieldType.STRING, @@ -531,7 +537,7 @@ describe("table sdk", () => { id: { ...basicTable.schema.id, type: FieldType.NUMBER, - }, + } as FieldSchema, }, undefined ) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 21b6463ce7..cec8c8aa12 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -54,6 +54,7 @@ import { FieldType, RelationshipType, CreateViewRequest, + RelationshipFieldMetadata, } from "@budibase/types" import API from "./api" @@ -584,10 +585,10 @@ class TestConfiguration { tableConfig.schema[link] = { type: FieldType.LINK, fieldName: link, - tableId: this.table._id, + tableId: this.table._id!, name: link, relationshipType, - } + } as RelationshipFieldMetadata } if (this.datasource && !tableConfig.sourceId) { diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 0bdaaa393e..766e1e6ba5 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -5,7 +5,13 @@ import { ObjectStoreBuckets } from "../../constants" import { context, db as dbCore, objectStore } from "@budibase/backend-core" import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" -import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types" +import { + AutoColumnFieldMetadata, + FieldSubtype, + Row, + RowAttachment, + Table, +} from "@budibase/types" import { cloneDeep } from "lodash/fp" import { processInputBBReferences, diff --git a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts index 68b62a3291..2437fa8864 100644 --- a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts @@ -4,10 +4,10 @@ import { FieldSchema, FieldType, RelationshipType } from "@budibase/types" describe("rowProcessor utility", () => { describe("fixAutoColumnSubType", () => { - let schema: FieldSchema = { + const schema: FieldSchema = { name: "", type: FieldType.LINK, - subtype: "", // missing subtype + subtype: undefined, // missing subtype icon: "ri-magic-line", autocolumn: true, constraints: { type: "array", presence: false }, @@ -22,31 +22,31 @@ describe("rowProcessor utility", () => { expect(fixAutoColumnSubType(schema).subtype).toEqual( AutoFieldSubTypes.CREATED_BY ) - schema.subtype = "" + schema.subtype = undefined schema.name = AutoFieldDefaultNames.UPDATED_BY expect(fixAutoColumnSubType(schema).subtype).toEqual( AutoFieldSubTypes.UPDATED_BY ) - schema.subtype = "" + schema.subtype = undefined schema.name = AutoFieldDefaultNames.CREATED_AT expect(fixAutoColumnSubType(schema).subtype).toEqual( AutoFieldSubTypes.CREATED_AT ) - schema.subtype = "" + schema.subtype = undefined schema.name = AutoFieldDefaultNames.UPDATED_AT expect(fixAutoColumnSubType(schema).subtype).toEqual( AutoFieldSubTypes.UPDATED_AT ) - schema.subtype = "" + schema.subtype = undefined schema.name = AutoFieldDefaultNames.AUTO_ID expect(fixAutoColumnSubType(schema).subtype).toEqual( AutoFieldSubTypes.AUTO_ID ) - schema.subtype = "" + schema.subtype = undefined }) it("returns the column if subtype exists", async () => { diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 0d7eace369..48697af6a9 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -5,13 +5,20 @@ import { FormulaTypes, } from "../../constants" import { processStringSync } from "@budibase/string-templates" -import { FieldSchema, Row, Table } from "@budibase/types" +import { + AutoColumnFieldMetadata, + FieldSchema, + Row, + Table, +} from "@budibase/types" /** * If the subtype has been lost for any reason this works out what * subtype the auto column should be. */ -export function fixAutoColumnSubType(column: FieldSchema) { +export function fixAutoColumnSubType( + column: FieldSchema +): AutoColumnFieldMetadata | FieldSchema { if (!column.autocolumn || !column.name || column.subtype) { return column } @@ -47,9 +54,13 @@ export function processFormulas( rowArray = rows } for (let [column, schema] of Object.entries(table.schema)) { + if (schema.type !== FieldTypes.FORMULA) { + continue + } + const isStatic = schema.formulaType === FormulaTypes.STATIC + if ( - schema.type !== FieldTypes.FORMULA || schema.formula == null || (dynamic && isStatic) || (!dynamic && !isStatic) diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts index 9a0ea4d135..783eae0671 100644 --- a/packages/types/src/documents/app/table/constants.ts +++ b/packages/types/src/documents/app/table/constants.ts @@ -7,3 +7,16 @@ export enum RelationshipType { export enum AutoReason { FOREIGN_KEY = "foreign_key", } + +export enum AutoFieldSubTypes { + CREATED_BY = "createdBy", + CREATED_AT = "createdAt", + UPDATED_BY = "updatedBy", + UPDATED_AT = "updatedAt", + AUTO_ID = "autoID", +} + +export enum FormulaTypes { + STATIC = "static", + DYNAMIC = "dynamic", +} diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 42a0838231..4106254fc4 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -1,7 +1,12 @@ // all added by grid/table when defining the // column size, position and whether it can be viewed -import { FieldType } from "../row" -import { AutoReason, RelationshipType } from "./constants" +import { FieldSubtype, FieldType } from "../row" +import { + AutoFieldSubTypes, + AutoReason, + FormulaTypes, + RelationshipType, +} from "./constants" export interface UIFieldMetadata { order?: number @@ -10,28 +15,63 @@ export interface UIFieldMetadata { icon?: string } -export interface RelationshipFieldMetadata { +interface BaseRelationshipFieldMetadata + extends Omit { + type: FieldType.LINK main?: boolean - fieldName?: string - tableId?: string - // below is used for SQL relationships, needed to define the foreign keys - // or the tables used for many-to-many relationships (through) - relationshipType?: RelationshipType - through?: string - foreignKey?: string - throughFrom?: string - throughTo?: string + fieldName: string + tableId: string + subtype?: AutoFieldSubTypes.CREATED_BY | AutoFieldSubTypes.UPDATED_BY } -export interface AutoColumnFieldMetadata { - autocolumn?: boolean - subtype?: string +// External tables use junction tables, internal tables don't require them +type ManyToManyJunctionTableMetadata = + | { + through: string + throughFrom: string + throughTo: string + } + | { + through?: never + throughFrom?: never + throughTo?: never + } + +export type ManyToManyRelationshipFieldMetadata = + BaseRelationshipFieldMetadata & { + relationshipType: RelationshipType.MANY_TO_MANY + } & ManyToManyJunctionTableMetadata + +export interface OneToManyRelationshipFieldMetadata + extends BaseRelationshipFieldMetadata { + relationshipType: RelationshipType.ONE_TO_MANY + foreignKey?: string +} +export interface ManyToOneRelationshipFieldMetadata + extends BaseRelationshipFieldMetadata { + relationshipType: RelationshipType.MANY_TO_ONE + foreignKey?: string +} +export type RelationshipFieldMetadata = + | ManyToManyRelationshipFieldMetadata + | OneToManyRelationshipFieldMetadata + | ManyToOneRelationshipFieldMetadata + +export interface AutoColumnFieldMetadata + extends Omit { + type: FieldType.AUTO + autocolumn: true + subtype?: AutoFieldSubTypes lastID?: number // if the column was turned to an auto-column for SQL, explains why (primary, foreign etc) autoReason?: AutoReason } -export interface NumberFieldMetadata { +export interface NumberFieldMetadata extends Omit { + type: FieldType.NUMBER + subtype?: AutoFieldSubTypes.AUTO_ID + lastID?: number + autoReason?: AutoReason.FOREIGN_KEY // used specifically when Budibase generates external tables, this denotes if a number field // is a foreign key used for a many-to-many relationship meta?: { @@ -40,18 +80,28 @@ export interface NumberFieldMetadata { } } -export interface DateFieldMetadata { +export interface DateFieldMetadata extends Omit { + type: FieldType.DATETIME ignoreTimezones?: boolean timeOnly?: boolean + subtype?: AutoFieldSubTypes.CREATED_AT | AutoFieldSubTypes.UPDATED_AT } -export interface StringFieldMetadata { +export interface LongFormFieldMetadata extends BaseFieldSchema { + type: FieldType.LONGFORM useRichText?: boolean | null } -export interface FormulaFieldMetadata { - formula?: string - formulaType?: string +export interface FormulaFieldMetadata extends BaseFieldSchema { + type: FieldType.FORMULA + formula: string + formulaType?: FormulaTypes +} + +export interface BBReferenceFieldMetadata + extends Omit { + type: FieldType.BB_REFERENCE + subtype: FieldSubtype.USER } export interface FieldConstraints { @@ -77,22 +127,40 @@ export interface FieldConstraints { } } -export interface FieldSchema - extends UIFieldMetadata, - DateFieldMetadata, - RelationshipFieldMetadata, - AutoColumnFieldMetadata, - StringFieldMetadata, - FormulaFieldMetadata, - NumberFieldMetadata { +interface BaseFieldSchema extends UIFieldMetadata { type: FieldType name: string sortable?: boolean // only used by external databases, to denote the real type externalType?: string constraints?: FieldConstraints + autocolumn?: boolean + autoReason?: AutoReason.FOREIGN_KEY + subtype?: never } +interface OtherFieldMetadata extends BaseFieldSchema { + type: Exclude< + FieldType, + | FieldType.DATETIME + | FieldType.LINK + | FieldType.AUTO + | FieldType.FORMULA + | FieldType.NUMBER + | FieldType.LONGFORM + > +} + +export type FieldSchema = + | OtherFieldMetadata + | DateFieldMetadata + | RelationshipFieldMetadata + | AutoColumnFieldMetadata + | FormulaFieldMetadata + | NumberFieldMetadata + | LongFormFieldMetadata + | BBReferenceFieldMetadata + export interface TableSchema { [key: string]: FieldSchema }