From 635f33f81e6a4a837a6a7c193fc2506df72de784 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 9 Oct 2023 18:28:09 +0100 Subject: [PATCH 1/4] Make sure that external tables preserve the BB_REFERNECE type, under the hood it is text/array, but it should continue to be treated correctly as its own indpendent type when updating Budibase tables (fetching). --- packages/server/src/integrations/utils.ts | 104 ++++++++++++---------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b6b6f60fee..db562473e3 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,7 +1,12 @@ -import { SqlQuery, Table, SearchFilters, Datasource } from "@budibase/types" +import { + SqlQuery, + Table, + SearchFilters, + Datasource, + FieldType, +} from "@budibase/types" import { DocumentType, SEPARATOR } from "../db/utils" import { - FieldTypes, BuildSchemaErrors, InvalidColumns, NoEmptyFilterStrings, @@ -13,57 +18,57 @@ const ROW_ID_REGEX = /^\[.*]$/g const ENCODED_SPACE = encodeURIComponent(" ") const SQL_NUMBER_TYPE_MAP = { - integer: FieldTypes.NUMBER, - int: FieldTypes.NUMBER, - decimal: FieldTypes.NUMBER, - smallint: FieldTypes.NUMBER, - real: FieldTypes.NUMBER, - float: FieldTypes.NUMBER, - numeric: FieldTypes.NUMBER, - mediumint: FieldTypes.NUMBER, - dec: FieldTypes.NUMBER, - double: FieldTypes.NUMBER, - fixed: FieldTypes.NUMBER, - "double precision": FieldTypes.NUMBER, - number: FieldTypes.NUMBER, - binary_float: FieldTypes.NUMBER, - binary_double: FieldTypes.NUMBER, - money: FieldTypes.NUMBER, - smallmoney: FieldTypes.NUMBER, + integer: FieldType.NUMBER, + int: FieldType.NUMBER, + decimal: FieldType.NUMBER, + smallint: FieldType.NUMBER, + real: FieldType.NUMBER, + float: FieldType.NUMBER, + numeric: FieldType.NUMBER, + mediumint: FieldType.NUMBER, + dec: FieldType.NUMBER, + double: FieldType.NUMBER, + fixed: FieldType.NUMBER, + "double precision": FieldType.NUMBER, + number: FieldType.NUMBER, + binary_float: FieldType.NUMBER, + binary_double: FieldType.NUMBER, + money: FieldType.NUMBER, + smallmoney: FieldType.NUMBER, } const SQL_DATE_TYPE_MAP = { - timestamp: FieldTypes.DATETIME, - time: FieldTypes.DATETIME, - datetime: FieldTypes.DATETIME, - smalldatetime: FieldTypes.DATETIME, - date: FieldTypes.DATETIME, + timestamp: FieldType.DATETIME, + time: FieldType.DATETIME, + datetime: FieldType.DATETIME, + smalldatetime: FieldType.DATETIME, + date: FieldType.DATETIME, } const SQL_DATE_ONLY_TYPES = ["date"] const SQL_TIME_ONLY_TYPES = ["time"] const SQL_STRING_TYPE_MAP = { - varchar: FieldTypes.STRING, - char: FieldTypes.STRING, - nchar: FieldTypes.STRING, - nvarchar: FieldTypes.STRING, - ntext: FieldTypes.STRING, - enum: FieldTypes.STRING, - blob: FieldTypes.STRING, - long: FieldTypes.STRING, - text: FieldTypes.STRING, + varchar: FieldType.STRING, + char: FieldType.STRING, + nchar: FieldType.STRING, + nvarchar: FieldType.STRING, + ntext: FieldType.STRING, + enum: FieldType.STRING, + blob: FieldType.STRING, + long: FieldType.STRING, + text: FieldType.STRING, } const SQL_BOOLEAN_TYPE_MAP = { - boolean: FieldTypes.BOOLEAN, - bit: FieldTypes.BOOLEAN, - tinyint: FieldTypes.BOOLEAN, + boolean: FieldType.BOOLEAN, + bit: FieldType.BOOLEAN, + tinyint: FieldType.BOOLEAN, } const SQL_MISC_TYPE_MAP = { - json: FieldTypes.JSON, - bigint: FieldTypes.BIGINT, + json: FieldType.JSON, + bigint: FieldType.BIGINT, } const SQL_TYPE_MAP = { @@ -154,7 +159,7 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { } export function convertSqlType(type: string) { - let foundType = FieldTypes.STRING + let foundType = FieldType.STRING const lcType = type.toLowerCase() let matchingTypes = [] for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) { @@ -169,7 +174,7 @@ export function convertSqlType(type: string) { }).internal } const schema: any = { type: foundType } - if (foundType === FieldTypes.DATETIME) { + if (foundType === FieldType.DATETIME) { schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType) schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType) } @@ -212,7 +217,7 @@ export function shouldCopyRelationship( tableIds: string[] ) { return ( - column.type === FieldTypes.LINK && + column.type === FieldType.LINK && column.tableId && tableIds.includes(column.tableId) ) @@ -230,22 +235,23 @@ export function shouldCopySpecialColumn( column: { type: string }, fetchedColumn: { type: string } | undefined ) { - const isFormula = column.type === FieldTypes.FORMULA + const isFormula = column.type === FieldType.FORMULA const specialTypes = [ - FieldTypes.OPTIONS, - FieldTypes.LONGFORM, - FieldTypes.ARRAY, - FieldTypes.FORMULA, + FieldType.OPTIONS, + FieldType.LONGFORM, + FieldType.ARRAY, + FieldType.FORMULA, + FieldType.BB_REFERENCE, ] // column has been deleted, remove - formulas will never exist, always copy if (!isFormula && column && !fetchedColumn) { return false } const fetchedIsNumber = - !fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER + !fetchedColumn || fetchedColumn.type === FieldType.NUMBER return ( - specialTypes.indexOf(column.type as FieldTypes) !== -1 || - (fetchedIsNumber && column.type === FieldTypes.BOOLEAN) + specialTypes.indexOf(column.type as FieldType) !== -1 || + (fetchedIsNumber && column.type === FieldType.BOOLEAN) ) } From 4b73f636fc32f689089a892756294e7ab9fe00d0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Oct 2023 11:34:14 +0100 Subject: [PATCH 2/4] Updating CI to only check pro when going to base branch. --- .github/workflows/budibase_ci.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 9d1131ed7f..55dedb6481 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -270,18 +270,23 @@ jobs: if [[ $branch == "master" ]]; then base_commit=$(git rev-parse origin/master) - else + else if [[ $branch == "develop" ]]; then base_commit=$(git rev-parse origin/develop) fi - echo "target_branch=$branch" - echo "target_branch=$branch" >> "$GITHUB_OUTPUT" - echo "pro_commit=$pro_commit" - echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" - echo "base_commit=$base_commit" - echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" + if [[ ! -z $base_commit ]]; then + echo "target_branch=$branch" + echo "target_branch=$branch" >> "$GITHUB_OUTPUT" + echo "pro_commit=$pro_commit" + echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" + echo "base_commit=$base_commit" + echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" + else + echo "Nothing to do - branch to branch merge." + fi - - name: Check submodule merged to develop + - name: Check submodule merged to base branch + if: ${{ base_commit != '' }} uses: actions/github-script@v4 with: github-token: ${{ secrets.GITHUB_TOKEN }} From cd5d438a96273e10d1c6d9a988cc81b80b37b80a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Oct 2023 11:35:07 +0100 Subject: [PATCH 3/4] Quick fix. --- .github/workflows/budibase_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 55dedb6481..77b864b8ea 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -286,7 +286,7 @@ jobs: fi - name: Check submodule merged to base branch - if: ${{ base_commit != '' }} + if: ${{ steps.get_pro_commits.outputs.base_commit != '' }} uses: actions/github-script@v4 with: github-token: ${{ secrets.GITHUB_TOKEN }} From 6c2eb693959cd7158cdeff763e99cda6bae97d7d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Oct 2023 11:58:30 +0100 Subject: [PATCH 4/4] Correcting syntax. --- .github/workflows/budibase_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 77b864b8ea..c9de51957b 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -270,7 +270,7 @@ jobs: if [[ $branch == "master" ]]; then base_commit=$(git rev-parse origin/master) - else if [[ $branch == "develop" ]]; then + elif [[ $branch == "develop" ]]; then base_commit=$(git rev-parse origin/develop) fi @@ -295,7 +295,7 @@ jobs: const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}'; if (submoduleCommit !== baseCommit) { - console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.'); + console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'); console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md') process.exit(1); } else {