diff --git a/lerna.json b/lerna.json index a2be7be7b4..19f6486f22 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.6", + "version": "2.23.7", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/nx.json b/nx.json index 618395ec90..8ba8798946 100644 --- a/nx.json +++ b/nx.json @@ -9,10 +9,7 @@ }, "targetDefaults": { "build": { - "inputs": [ - "{workspaceRoot}/scripts/build.js", - "{workspaceRoot}/lerna.json" - ] + "inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"] } } } diff --git a/package.json b/package.json index e520b7c2cf..e60a086e17 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "dev:camunda": "./scripts/deploy-camunda.sh", "dev:all": "yarn run kill-all && lerna run --stream dev", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "./scripts/devDocker.sh", "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream", "lint:eslint": "eslint packages --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 92501bec3b..d271462f3e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -13,6 +13,7 @@ Layout, AbsTooltip, } from "@budibase/bbui" + import { SWITCHABLE_TYPES, ValidColumnNameRegex } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -20,11 +21,6 @@ import { FIELDS, RelationshipType, - ALLOWABLE_STRING_OPTIONS, - ALLOWABLE_NUMBER_OPTIONS, - ALLOWABLE_STRING_TYPES, - ALLOWABLE_NUMBER_TYPES, - SWITCHABLE_TYPES, PrettyRelationshipDefinitions, DB_TYPE_EXTERNAL, } from "constants/backend" @@ -33,21 +29,20 @@ import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import { getBindings } from "components/backend/DataTable/formula" import JSONSchemaModal from "./JSONSchemaModal.svelte" - import { ValidColumnNameRegex } from "@budibase/shared-core" import { FieldType, FieldSubtype, SourceName } from "@budibase/types" import RelationshipSelector from "components/common/RelationshipSelector.svelte" import { RowUtils } from "@budibase/frontend-core" import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" - const AUTO_TYPE = FIELDS.AUTO.type - const FORMULA_TYPE = FIELDS.FORMULA.type - const LINK_TYPE = FIELDS.LINK.type - const STRING_TYPE = FIELDS.STRING.type - const NUMBER_TYPE = FIELDS.NUMBER.type - const JSON_TYPE = FIELDS.JSON.type - const DATE_TYPE = FIELDS.DATETIME.type - const USER_TYPE = FIELDS.USER.subtype - const USERS_TYPE = FIELDS.USERS.subtype + const AUTO_TYPE = FieldType.AUTO + const FORMULA_TYPE = FieldType.FORMULA + const LINK_TYPE = FieldType.LINK + const STRING_TYPE = FieldType.STRING + const NUMBER_TYPE = FieldType.NUMBER + const JSON_TYPE = FieldType.JSON + const DATE_TYPE = FieldType.DATETIME + const USER_TYPE = FieldSubtype.USER + const USERS_TYPE = FieldSubtype.USERS const dispatch = createEventDispatcher() const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] @@ -61,8 +56,8 @@ let primaryDisplay let indexes = [...($tables.selected.indexes || [])] let isCreating = undefined - let relationshipPart1 = PrettyRelationshipDefinitions.Many - let relationshipPart2 = PrettyRelationshipDefinitions.One + let relationshipPart1 = PrettyRelationshipDefinitions.MANY + let relationshipPart2 = PrettyRelationshipDefinitions.ONE let relationshipTableIdPrimary = null let relationshipTableIdSecondary = null let table = $tables.selected @@ -175,7 +170,7 @@ $: typeEnabled = !originalName || (originalName && - SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 && + SWITCHABLE_TYPES[field.type] && !editableColumn?.autocolumn) const fieldDefinitions = Object.values(FIELDS).reduce( @@ -367,16 +362,15 @@ } function getAllowedTypes() { - if ( - originalName && - ALLOWABLE_STRING_TYPES.indexOf(editableColumn.type) !== -1 - ) { - return ALLOWABLE_STRING_OPTIONS - } else if ( - originalName && - ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1 - ) { - return ALLOWABLE_NUMBER_OPTIONS + if (originalName) { + const possibleTypes = ( + SWITCHABLE_TYPES[field.type] || [editableColumn.type] + ).map(t => t.toLowerCase()) + return Object.entries(FIELDS) + .filter(([fieldType]) => + possibleTypes.includes(fieldType.toLowerCase()) + ) + .map(([_, fieldDefinition]) => fieldDefinition) } const isUsers = @@ -632,7 +626,7 @@ /> - {:else if editableColumn.type === FieldType.LINK} + {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn} opt.type -) - -export const ALLOWABLE_NUMBER_OPTIONS = [FIELDS.NUMBER, FIELDS.BOOLEAN] -export const ALLOWABLE_NUMBER_TYPES = ALLOWABLE_NUMBER_OPTIONS.map( - opt => opt.type -) - -export const SWITCHABLE_TYPES = [ - ...ALLOWABLE_STRING_TYPES, - ...ALLOWABLE_NUMBER_TYPES, -] - export const BUDIBASE_INTERNAL_DB_ID = INTERNAL_TABLE_SOURCE_ID export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" export const BUDIBASE_DATASOURCE_TYPE = "budibase" diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index 0163281480..88b26929ad 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -1,8 +1,8 @@ import { FieldType } from "@budibase/types" +import { SWITCHABLE_TYPES } from "@budibase/shared-core" import { get, writable, derived } from "svelte/store" import { cloneDeep } from "lodash/fp" import { API } from "api" -import { SWITCHABLE_TYPES } from "constants/backend" export function createTablesStore() { const store = writable({ @@ -64,7 +64,7 @@ export function createTablesStore() { if ( oldField != null && oldField?.type !== field.type && - SWITCHABLE_TYPES.indexOf(oldField?.type) === -1 + !SWITCHABLE_TYPES[oldField?.type]?.includes(field.type) ) { updatedTable.schema[key] = oldField } @@ -148,12 +148,6 @@ export function createTablesStore() { if (indexes) { draft.indexes = indexes } - // Add object to indicate if column is being added - if (draft.schema[field.name] === undefined) { - draft._add = { - name: field.name, - } - } draft.schema = { ...draft.schema, [field.name]: cloneDeep(field), diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 7c036bec9d..e526af4ecb 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -31,7 +31,6 @@ export async function save( renaming?: RenameColumn ) { const inputs = ctx.request.body - const adding = inputs?._add // can't do this right now delete inputs.rows const tableId = ctx.request.body._id @@ -44,7 +43,7 @@ export async function save( const { datasource, table } = await sdk.tables.external.save( datasourceId!, inputs, - { tableId, renaming, adding } + { tableId, renaming } ) builderSocket?.emitDatasourceUpdate(ctx, datasource) return table diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index f799113333..63ce00c5ef 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -77,11 +77,6 @@ export async function save(ctx: UserCtx) { const renaming = ctx.request.body._rename const api = pickApi({ table }) - // do not pass _rename or _add if saving to CouchDB - if (api === internal) { - delete ctx.request.body._add - delete ctx.request.body._rename - } let savedTable = await api.save(ctx, renaming) if (!table._id) { savedTable = sdk.tables.enrichViewSchemas(savedTable) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index eb5e4b6c41..a06cc4dee3 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -16,7 +16,7 @@ export async function save( ctx: UserCtx, renaming?: RenameColumn ) { - const { rows, ...rest } = ctx.request.body + const { _rename, rows, ...rest } = ctx.request.body let tableToSave: Table = { _id: generateTableID(), ...rest, diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 7639b840dc..77e05b8e07 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -219,9 +219,6 @@ describe.each([ it("should add a new column for an internal DB table", async () => { const saveTableRequest: SaveTableRequest = { - _add: { - name: "NEW_COLUMN", - }, ...basicTable(), } @@ -235,7 +232,6 @@ describe.each([ updatedAt: expect.stringMatching(ISO_REGEX_PATTERN), views: {}, } - delete expectedResponse._add expect(response).toEqual(expectedResponse) }) }) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 7e54b53b15..b4eb1035d6 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -16,7 +16,6 @@ import { getDatasource, rawQuery, } from "../integrations/tests/utils" -import { builderSocket } from "../websockets" import { generator } from "@budibase/backend-core/tests" // @ts-ignore fetch.mockSearch() @@ -233,72 +232,6 @@ describe("mysql integrations", () => { }) describe("POST /api/tables/", () => { - const emitDatasourceUpdateMock = jest.fn() - - it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { - const addColumnToTable: TableRequest = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - name: uniqueTableName(), - sourceId: datasource._id!, - primary: ["id"], - schema: { - id: { - type: FieldType.AUTO, - name: "id", - autocolumn: true, - }, - new_column: { - type: FieldType.NUMBER, - name: "new_column", - }, - }, - _add: { - name: "new_column", - }, - } - - jest - .spyOn(builderSocket!, "emitDatasourceUpdate") - .mockImplementation(emitDatasourceUpdateMock) - - await makeRequest("post", "/api/tables/", addColumnToTable) - - const expectedTable: TableRequest = { - ...addColumnToTable, - schema: { - id: { - type: FieldType.NUMBER, - name: "id", - autocolumn: true, - constraints: { - presence: false, - }, - externalType: "int unsigned", - }, - new_column: { - type: FieldType.NUMBER, - name: "new_column", - autocolumn: false, - constraints: { - presence: false, - }, - externalType: "float(8,2)", - }, - }, - created: true, - _id: `${datasource._id}__${addColumnToTable.name}`, - } - delete expectedTable._add - - expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1) - const emittedDatasource: Datasource = - emitDatasourceUpdateMock.mock.calls[0][1] - expect(emittedDatasource.entities![expectedTable.name]).toEqual( - expectedTable - ) - }) - it("will rename a column", async () => { await makeRequest("post", "/api/tables/", primaryMySqlTable) diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 8f2bed04a2..cc75f0444d 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -7,7 +7,7 @@ import { } from "@budibase/types" import { DocumentType, SEPARATOR } from "../../db/utils" import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants" -import { helpers } from "@budibase/shared-core" +import { SWITCHABLE_TYPES, helpers } from "@budibase/shared-core" import env from "../../environment" import { Knex } from "knex" @@ -284,8 +284,8 @@ export function isIsoDateString(str: string) { * @param column The column to check, to see if it is a valid relationship. * @param tableIds The IDs of the tables which currently exist. */ -export function shouldCopyRelationship( - column: { type: string; tableId?: string }, +function shouldCopyRelationship( + column: { type: FieldType.LINK; tableId?: string }, tableIds: string[] ) { return ( @@ -303,28 +303,18 @@ export function shouldCopyRelationship( * @param column The column to check for options or boolean type. * @param fetchedColumn The fetched column to check for the type in the external database. */ -export function shouldCopySpecialColumn( - column: { type: string }, - fetchedColumn: { type: string } | undefined +function shouldCopySpecialColumn( + column: { type: FieldType }, + fetchedColumn: { type: FieldType } | undefined ) { const isFormula = column.type === FieldType.FORMULA - const specialTypes = [ - FieldType.OPTIONS, - FieldType.LONGFORM, - FieldType.ARRAY, - FieldType.FORMULA, - FieldType.BB_REFERENCE, - ] // column has been deleted, remove - formulas will never exist, always copy if (!isFormula && column && !fetchedColumn) { return false } const fetchedIsNumber = !fetchedColumn || fetchedColumn.type === FieldType.NUMBER - return ( - specialTypes.indexOf(column.type as FieldType) !== -1 || - (fetchedIsNumber && column.type === FieldType.BOOLEAN) - ) + return fetchedIsNumber && column.type === FieldType.BOOLEAN } /** @@ -357,11 +347,44 @@ function copyExistingPropsOver( continue } const column = existingTableSchema[key] + + const existingColumnType = column?.type + const updatedColumnType = table.schema[key]?.type + + // If the db column type changed to a non-compatible one, we want to re-fetch it if ( - shouldCopyRelationship(column, tableIds) || - shouldCopySpecialColumn(column, table.schema[key]) + updatedColumnType !== existingColumnType && + !SWITCHABLE_TYPES[updatedColumnType]?.includes(existingColumnType) ) { - table.schema[key] = existingTableSchema[key] + continue + } + + if ( + column.type === FieldType.LINK && + !shouldCopyRelationship(column, tableIds) + ) { + continue + } + + const specialTypes = [ + FieldType.OPTIONS, + FieldType.LONGFORM, + FieldType.ARRAY, + FieldType.FORMULA, + FieldType.BB_REFERENCE, + ] + if ( + specialTypes.includes(column.type) && + !shouldCopySpecialColumn(column, table.schema[key]) + ) { + continue + } + + table.schema[key] = { + ...existingTableSchema[key], + externalType: + existingTableSchema[key].externalType || + table.schema[key].externalType, } } } diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 336a94636b..84e1601152 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -348,8 +348,7 @@ const preSaveAction: Partial> = { * Make sure all datasource entities have a display name selected */ export function setDefaultDisplayColumns(datasource: Datasource) { - // - for (let entity of Object.values(datasource.entities || {})) { + for (const entity of Object.values(datasource.entities || {})) { if (entity.primaryDisplay) { continue } diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index bc8430c72c..2a78600cfc 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -3,7 +3,6 @@ import { Operation, RelationshipType, RenameColumn, - AddColumn, Table, TableRequest, ViewV2, @@ -33,7 +32,7 @@ import * as viewSdk from "../../views" export async function save( datasourceId: string, update: Table, - opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn } + opts?: { tableId?: string; renaming?: RenameColumn } ) { let tableToSave: TableRequest = { ...update, @@ -185,14 +184,7 @@ export async function save( // remove the rename prop delete tableToSave._rename - // if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column - if (opts?.adding) { - datasource.entities[tableToSave.name] = ( - await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name]) - ).tables[tableToSave.name] - } else { - datasource.entities[tableToSave.name] = tableToSave - } + datasource.entities[tableToSave.name] = tableToSave // store it into couch now for budibase reference await db.put(populateExternalTableSchemas(datasource)) diff --git a/packages/shared-core/src/constants/fields.ts b/packages/shared-core/src/constants/fields.ts new file mode 100644 index 0000000000..5acf07d863 --- /dev/null +++ b/packages/shared-core/src/constants/fields.ts @@ -0,0 +1,33 @@ +import { FieldType } from "@budibase/types" + +type SwitchableTypes = Partial<{ + [K in FieldType]: [K, ...FieldType[]] +}> + +export const SWITCHABLE_TYPES: SwitchableTypes = { + [FieldType.STRING]: [ + FieldType.STRING, + FieldType.OPTIONS, + FieldType.LONGFORM, + FieldType.BARCODEQR, + ], + [FieldType.OPTIONS]: [ + FieldType.OPTIONS, + FieldType.STRING, + FieldType.LONGFORM, + FieldType.BARCODEQR, + ], + [FieldType.LONGFORM]: [ + FieldType.LONGFORM, + FieldType.STRING, + FieldType.OPTIONS, + FieldType.BARCODEQR, + ], + [FieldType.BARCODEQR]: [ + FieldType.BARCODEQR, + FieldType.STRING, + FieldType.OPTIONS, + FieldType.LONGFORM, + ], + [FieldType.NUMBER]: [FieldType.NUMBER, FieldType.BOOLEAN], +} diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index 922f0d4387..afb7e659e1 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,4 +1,5 @@ export * from "./api" +export * from "./fields" export const OperatorOptions = { Equals: { diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index b284e9a840..f0e6079aef 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -1,6 +1,6 @@ import { Document } from "../../document" import { View, ViewV2 } from "../view" -import { AddColumn, RenameColumn } from "../../../sdk" +import { RenameColumn } from "../../../sdk" import { TableSchema } from "./schema" export const INTERNAL_TABLE_SOURCE_ID = "bb_internal" @@ -30,6 +30,5 @@ export interface Table extends Document { export interface TableRequest extends Table { _rename?: RenameColumn - _add?: AddColumn created?: boolean } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index b19d40534e..40f411f02a 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -80,10 +80,6 @@ export interface RenameColumn { updated: string } -export interface AddColumn { - name: string -} - export interface RelationshipsJson { through?: string from?: string diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh new file mode 100755 index 0000000000..5e01e5813a --- /dev/null +++ b/scripts/devDocker.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Check if the pro submodule is loaded +if [ ! -d "./packages/pro/src" ]; then + echo "[ERROR] Submodule is not loaded. This is only allowed with loaded submodules." + exit 1 +fi + +yarn build --scope @budibase/server --scope @budibase/worker +docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0 + +