diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index b7dbcae771..d6e0432e83 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -179,13 +179,6 @@ jobs: - run: yarn --frozen-lockfile - name: Test server - env: - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: "${{ secrets.DATADOG_API_KEY }}" - DD_SITE: "datadoghq.eu" - NODE_OPTIONS: "-r dd-trace/ci/init" - DD_ENV: "ci" - DD_SERVICE: "budibase/packages/server" run: | if ${{ env.USE_NX_AFFECTED }}; then yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} @@ -233,10 +226,11 @@ jobs: if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }} run: | cd packages/pro + base_commit='${{ steps.get_pro_commits.outputs.base_commit }}' base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}' - any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit) + any_commit=$(git log --no-merges $base_commit...$pro_commit) if [ -n "$any_commit" ]; then echo $any_commit diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json index 9303874a77..481197b26c 100644 --- a/examples/nextjs-api-sales/package.json +++ b/examples/nextjs-api-sales/package.json @@ -22,6 +22,6 @@ "@types/react": "17.0.39", "eslint": "8.10.0", "eslint-config-next": "12.1.0", - "typescript": "5.2.2" + "typescript": "5.5.2" } } diff --git a/package.json b/package.json index e05eb795bc..d4a51f2e62 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "proper-lockfile": "^4.1.2", "svelte": "^4.2.10", "svelte-eslint-parser": "^0.33.1", - "typescript": "5.2.2", + "typescript": "5.5.2", "typescript-eslint": "^7.3.1", "yargs": "^17.7.2" }, diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index a6598365f7..9142da1823 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -16,7 +16,7 @@ "prepack": "cp package.json dist", "build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", - "check:types": "tsc -p tsconfig.json --noEmit --paths null", + "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "test": "bash scripts/test.sh", "test:watch": "jest --watchAll" }, @@ -78,7 +78,7 @@ "pouchdb-adapter-memory": "7.2.2", "testcontainers": "^10.7.2", "timekeeper": "2.2.0", - "typescript": "5.2.2" + "typescript": "5.5.2" }, "nx": { "targets": { diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index bfa7595d62..69c98fe569 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -1,14 +1,5 @@ -export const CONSTANT_INTERNAL_ROW_COLS = [ - "_id", - "_rev", - "type", - "createdAt", - "updatedAt", - "tableId", -] as const - -export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const - -export function isInternalColumnName(name: string): boolean { - return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name) -} +export { + CONSTANT_INTERNAL_ROW_COLS, + CONSTANT_EXTERNAL_ROW_COLS, + isInternalColumnName, +} from "@budibase/shared-core" diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index cdc5f3d3c8..34b950bf2c 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -397,9 +397,9 @@ class InternalBuilder { contains(filters.containsAny, true) } + const tableRef = opts?.aliases?.[table._id!] || table._id // when searching internal tables make sure long looking for rows - if (filters.documentType && !isExternalTable(table)) { - const tableRef = opts?.aliases?.[table._id!] || table._id + if (filters.documentType && !isExternalTable(table) && tableRef) { // has to be its own option, must always be AND onto the search query.andWhereLike( `${tableRef}._id`, diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 17ecd8f844..d79eedd194 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -17,6 +17,8 @@ SWITCHABLE_TYPES, ValidColumnNameRegex, helpers, + CONSTANT_INTERNAL_ROW_COLS, + CONSTANT_EXTERNAL_ROW_COLS, } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" @@ -52,7 +54,6 @@ const DATE_TYPE = FieldType.DATETIME const dispatch = createEventDispatcher() - const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] const { dispatch: gridDispatch, rows } = getContext("grid") export let field @@ -487,20 +488,27 @@ }) } const newError = {} + const prohibited = externalTable + ? CONSTANT_EXTERNAL_ROW_COLS + : CONSTANT_INTERNAL_ROW_COLS if (!externalTable && fieldInfo.name?.startsWith("_")) { newError.name = `Column name cannot start with an underscore.` } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) { newError.name = `Illegal character; must be alpha-numeric.` - } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) { - newError.name = `${PROHIBITED_COLUMN_NAMES.join( + } else if ( + prohibited.some( + name => fieldInfo?.name?.toLowerCase() === name.toLowerCase() + ) + ) { + newError.name = `${prohibited.join( ", " - )} are not allowed as column names` + )} are not allowed as column names - case insensitive.` } else if (inUse($tables.selected, fieldInfo.name, originalName)) { newError.name = `Column name already in use.` } if (fieldInfo.type === FieldType.AUTO && !fieldInfo.subtype) { - newError.subtype = `Auto Column requires a type` + newError.subtype = `Auto Column requires a type.` } if (fieldInfo.fieldName && fieldInfo.tableId) { diff --git a/packages/cli/package.json b/packages/cli/package.json index 88d5926ae3..1722d45730 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -11,7 +11,7 @@ "scripts": { "tsc": "node ../../scripts/build.js", "build": "yarn tsc", - "check:types": "tsc -p tsconfig.json --noEmit --paths null", + "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "start": "ts-node ./src/index.ts" }, "dependencies": { @@ -40,6 +40,6 @@ "@types/node-fetch": "2.6.4", "@types/pouchdb": "^6.4.0", "ts-node": "10.8.1", - "typescript": "5.2.2" + "typescript": "5.5.2" } } diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 00b503626f..7a9d1a5695 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -23,17 +23,21 @@ { "type": "bigint", "message": "stringAsNumber" }, { "type": "options", "message": "stringAsNumber" }, { "type": "formula", "message": "stringAsNumber" }, - { "type": "datetime", "message": "dateAsNumber"} + { "type": "datetime", "message": "dateAsNumber" } ], - "unsupported": [ - { "type": "json", "message": "jsonPrimitivesOnly" } - ] + "unsupported": [{ "type": "json", "message": "jsonPrimitivesOnly" }] }, "stringLike": { - "supported": ["string", "number", "bigint", "options", "longform", "boolean", "datetime"], - "unsupported": [ - { "type": "json", "message": "jsonPrimitivesOnly" } - ] + "supported": [ + "string", + "number", + "bigint", + "options", + "longform", + "boolean", + "datetime" + ], + "unsupported": [{ "type": "json", "message": "jsonPrimitivesOnly" }] }, "datetimeLike": { "supported": ["datetime"], @@ -43,11 +47,9 @@ { "type": "options", "message": "stringAsDate" }, { "type": "formula", "message": "stringAsDate" }, { "type": "bigint", "message": "stringAsDate" }, - { "type": "number", "message": "numberAsDate"} + { "type": "number", "message": "numberAsDate" } ], - "unsupported": [ - { "type": "json", "message": "jsonPrimitivesOnly" } - ] + "unsupported": [{ "type": "json", "message": "jsonPrimitivesOnly" }] } }, "layout": { diff --git a/packages/client/src/components/preview/SettingsBar.svelte b/packages/client/src/components/preview/SettingsBar.svelte index b69b8ce050..c5109c6bca 100644 --- a/packages/client/src/components/preview/SettingsBar.svelte +++ b/packages/client/src/components/preview/SettingsBar.svelte @@ -41,7 +41,7 @@ allSettings.push(setting) } }) - return allSettings.filter(setting => setting.showInBar) + return allSettings.filter(setting => setting.showInBar && !setting.hidden) } const updatePosition = () => { diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte index 20cfdb1ec5..ead2c67787 100644 --- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte +++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte @@ -3,6 +3,7 @@ import { Button } from "@budibase/bbui" import GridCell from "../cells/GridCell.svelte" import GridScrollWrapper from "./GridScrollWrapper.svelte" + import { BlankRowID } from "../lib/constants" const { renderedRows, @@ -17,6 +18,7 @@ isDragging, buttonColumnWidth, showVScrollbar, + dispatch, } = getContext("grid") let container @@ -89,6 +91,17 @@ {/each} +
($hoveredRowId = BlankRowID)} + on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)} + > + dispatch("add-row-inline")} + /> +
@@ -129,8 +142,11 @@ align-items: center; gap: 4px; } + .blank :global(.cell:hover) { + cursor: pointer; + } - /* Add left cell border */ + /* Add left cell border to all cells */ .button-column :global(.cell) { border-left: var(--cell-border); } diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index 8a82209162..8ea9e2264d 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -26,7 +26,7 @@ MaxCellRenderOverflow, GutterWidth, DefaultRowHeight, - Padding, + VPadding, SmallRowHeight, ControlsHeight, ScrollBarSize, @@ -119,7 +119,7 @@ // Derive min height and make available in context const minHeight = derived(rowHeight, $height => { const heightForControls = showControls ? ControlsHeight : 0 - return Padding + SmallRowHeight + $height + heightForControls + return VPadding + SmallRowHeight + $height + heightForControls }) context = { ...context, minHeight } @@ -354,8 +354,13 @@ transition: none; } - /* Overrides */ - .grid.quiet :global(.grid-data-content .row > .cell:not(:last-child)) { + /* Overrides for quiet */ + .grid.quiet :global(.grid-data-content .row > .cell:not(:last-child)), + .grid.quiet :global(.sticky-column .row > .cell), + .grid.quiet :global(.new-row .row > .cell:not(:last-child)) { border-right: none; } + .grid.quiet :global(.sticky-column:before) { + display: none; + } diff --git a/packages/frontend-core/src/components/grid/layout/GridBody.svelte b/packages/frontend-core/src/components/grid/layout/GridBody.svelte index 8be56674be..cf93f3004e 100644 --- a/packages/frontend-core/src/components/grid/layout/GridBody.svelte +++ b/packages/frontend-core/src/components/grid/layout/GridBody.svelte @@ -2,6 +2,7 @@ import { getContext, onMount } from "svelte" import GridScrollWrapper from "./GridScrollWrapper.svelte" import GridRow from "./GridRow.svelte" + import GridCell from "../cells/GridCell.svelte" import { BlankRowID } from "../lib/constants" import ButtonColumn from "./ButtonColumn.svelte" @@ -46,7 +47,6 @@ -
{#each $renderedRows as row, idx} @@ -54,13 +54,16 @@ {/each} {#if $config.canAddRows}
($hoveredRowId = BlankRowID)} on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)} - on:click={() => dispatch("add-row-inline")} - /> + > + dispatch("add-row-inline")} + /> +
{/if} {#if $props.buttons?.length} @@ -76,15 +79,13 @@ overflow: hidden; flex: 1 1 auto; } - .blank { - height: var(--row-height); - background: var(--cell-background); - border-bottom: var(--cell-border); - border-right: var(--cell-border); - position: absolute; + .row { + display: flex; + flex-direction: row; + justify-content: flex-start; + align-items: stretch; } - .blank.highlighted { - background: var(--cell-background-hover); + .blank :global(.cell:hover) { cursor: pointer; } diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte index 68ace8a5b2..1da8f7a63e 100644 --- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte @@ -31,6 +31,7 @@ filter, inlineFilters, columnRenderMap, + scrollTop, } = getContext("grid") let visible = false @@ -43,6 +44,21 @@ $: $datasource, (visible = false) $: selectedRowCount = Object.values($selectedRows).length $: hasNoRows = !$rows.length + $: renderedRowCount = $renderedRows.length + $: offset = getOffset($hasNextPage, renderedRowCount, $rowHeight, $scrollTop) + + const getOffset = (hasNextPage, rowCount, rowHeight, scrollTop) => { + // If we have a next page of data then we aren't truly at the bottom, so we + // render the add row component at the top + if (hasNextPage) { + return 0 + } + offset = rowCount * rowHeight - (scrollTop % rowHeight) + if (rowCount !== 0) { + offset -= 1 + } + return offset + } const addRow = async () => { // Blur the active cell and tick to let final value updates propagate @@ -85,12 +101,6 @@ return } - // If we have a next page of data then we aren't truly at the bottom, so we - // render the add row component at the top - if ($hasNextPage) { - offset = 0 - } - // If we don't have a next page then we're at the bottom and can scroll to // the max available offset else { @@ -98,10 +108,6 @@ ...state, top: $maxScrollTop, })) - offset = $renderedRows.length * $rowHeight - ($maxScrollTop % $rowHeight) - if ($renderedRows.length !== 0) { - offset -= 1 - } } // Update state and select initial cell @@ -171,39 +177,41 @@ {#if visible}
0} style="--offset:{offset}px; --sticky-width:{width}px;" >
- - - {#if isAdding} -
- {/if} - - {#if $stickyColumn} - {@const cellId = getCellID(NewRowID, $stickyColumn.name)} - - {#if $stickyColumn?.schema?.autocolumn} -
Can't edit auto column
- {/if} +
+ + {#if isAdding}
{/if} - - {/if} + + {#if $stickyColumn} + {@const cellId = getCellID(NewRowID, $stickyColumn.name)} + + {#if $stickyColumn?.schema?.autocolumn} +
Can't edit auto column
+ {/if} + {#if isAdding} +
+ {/if} + + {/if} +
@@ -270,7 +278,7 @@ margin-left: -6px; } - .container { + .new-row { position: absolute; top: var(--default-row-height); left: 0; @@ -280,10 +288,10 @@ flex-direction: row; align-items: stretch; } - .container :global(.cell) { + .new-row :global(.cell) { --cell-background: var(--spectrum-global-color-gray-75) !important; } - .container.floating :global(.cell) { + .new-row.floating :global(.cell) { height: calc(var(--row-height) + 1px); border-top: var(--cell-border); } @@ -312,8 +320,10 @@ pointer-events: all; z-index: 3; position: absolute; - top: calc(var(--row-height) + var(--offset) + 24px); - left: 18px; + top: calc( + var(--row-height) + var(--offset) + var(--default-row-height) / 2 + ); + left: calc(var(--default-row-height) / 2); } .button-with-keys { display: flex; diff --git a/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte b/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte index b57c89ee4f..85c1eb2897 100644 --- a/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte +++ b/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte @@ -66,62 +66,58 @@ -
- - {#each $renderedRows as row, idx} - {@const rowSelected = !!$selectedRows[row._id]} - {@const rowHovered = $hoveredRowId === row._id} - {@const rowFocused = $focusedRow?._id === row._id} - {@const cellId = getCellID(row._id, $stickyColumn?.name)} -
($hoveredRowId = row._id)} - on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)} - on:click={() => dispatch("rowclick", rows.actions.cleanRow(row))} - > - - {#if $stickyColumn} - - {/if} -
- {/each} - {#if $config.canAddRows} -
($hoveredRowId = BlankRowID)} - on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)} - on:click={() => dispatch("add-row-inline")} - > - - - - {#if $stickyColumn} - - - - {/if} -
- {/if} - -
+ + {#each $renderedRows as row, idx} + {@const rowSelected = !!$selectedRows[row._id]} + {@const rowHovered = $hoveredRowId === row._id} + {@const rowFocused = $focusedRow?._id === row._id} + {@const cellId = getCellID(row._id, $stickyColumn?.name)} +
($hoveredRowId = row._id)} + on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)} + on:click={() => dispatch("rowclick", rows.actions.cleanRow(row))} + > + + {#if $stickyColumn} + + {/if} +
+ {/each} + {#if $config.canAddRows} +
($hoveredRowId = BlankRowID)} + on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)} + on:click={() => dispatch("add-row-inline")} + > + + + + {#if $stickyColumn} + + + + {/if} +
+ {/if} +
diff --git a/packages/frontend-core/src/components/grid/lib/constants.js b/packages/frontend-core/src/components/grid/lib/constants.js index 4b5d04894a..6ea7a98178 100644 --- a/packages/frontend-core/src/components/grid/lib/constants.js +++ b/packages/frontend-core/src/components/grid/lib/constants.js @@ -1,12 +1,13 @@ -export const Padding = 100 -export const ScrollBarSize = 8 -export const GutterWidth = 72 -export const DefaultColumnWidth = 200 -export const MinColumnWidth = 80 export const SmallRowHeight = 36 export const MediumRowHeight = 64 export const LargeRowHeight = 92 export const DefaultRowHeight = SmallRowHeight +export const VPadding = SmallRowHeight * 2 +export const HPadding = 40 +export const ScrollBarSize = 8 +export const GutterWidth = 72 +export const DefaultColumnWidth = 200 +export const MinColumnWidth = 80 export const NewRowID = "new" export const BlankRowID = "blank" export const RowPageSize = 100 diff --git a/packages/frontend-core/src/components/grid/stores/scroll.js b/packages/frontend-core/src/components/grid/stores/scroll.js index e7114cd00c..814d4cdc8c 100644 --- a/packages/frontend-core/src/components/grid/stores/scroll.js +++ b/packages/frontend-core/src/components/grid/stores/scroll.js @@ -1,6 +1,12 @@ import { writable, derived, get } from "svelte/store" import { tick } from "svelte" -import { Padding, GutterWidth, FocusedCellMinOffset } from "../lib/constants" +import { + GutterWidth, + FocusedCellMinOffset, + ScrollBarSize, + HPadding, + VPadding, +} from "../lib/constants" import { parseCellID } from "../lib/utils" export const createStores = () => { @@ -34,28 +40,15 @@ export const deriveStores = context => { // Memoize store primitives const stickyColumnWidth = derived(stickyColumn, $col => $col?.width || 0, 0) - // Derive vertical limits - const contentHeight = derived( - [rows, rowHeight], - ([$rows, $rowHeight]) => ($rows.length + 1) * $rowHeight + Padding, - 0 - ) - const maxScrollTop = derived( - [height, contentHeight], - ([$height, $contentHeight]) => Math.max($contentHeight - $height, 0), - 0 - ) - // Derive horizontal limits const contentWidth = derived( [visibleColumns, stickyColumnWidth, buttonColumnWidth], ([$visibleColumns, $stickyColumnWidth, $buttonColumnWidth]) => { - const space = Math.max(Padding, $buttonColumnWidth - 1) - let width = GutterWidth + space + $stickyColumnWidth + let width = GutterWidth + $buttonColumnWidth + $stickyColumnWidth $visibleColumns.forEach(col => { width += col.width }) - return width + return width + HPadding }, 0 ) @@ -71,14 +64,6 @@ export const deriveStores = context => { }, 0 ) - - // Derive whether to show scrollbars or not - const showVScrollbar = derived( - [contentHeight, height], - ([$contentHeight, $height]) => { - return $contentHeight > $height - } - ) const showHScrollbar = derived( [contentWidth, screenWidth], ([$contentWidth, $screenWidth]) => { @@ -86,6 +71,30 @@ export const deriveStores = context => { } ) + // Derive vertical limits + const contentHeight = derived( + [rows, rowHeight, showHScrollbar], + ([$rows, $rowHeight, $showHScrollbar]) => { + let height = ($rows.length + 1) * $rowHeight + VPadding + if ($showHScrollbar) { + height += ScrollBarSize * 2 + } + return height + }, + 0 + ) + const maxScrollTop = derived( + [height, contentHeight], + ([$height, $contentHeight]) => Math.max($contentHeight - $height, 0), + 0 + ) + const showVScrollbar = derived( + [contentHeight, height], + ([$contentHeight, $height]) => { + return $contentHeight > $height + } + ) + return { contentHeight, contentWidth, diff --git a/packages/server/package.json b/packages/server/package.json index e146bd081c..94bbb6fc6b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -12,7 +12,7 @@ "prebuild": "rimraf dist/", "build": "node ./scripts/build.js", "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/", - "check:types": "tsc -p tsconfig.json --noEmit --paths null", + "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "build:isolated-vm-lib:snippets": "esbuild --minify --bundle src/jsRunner/bundles/snippets.ts --outfile=src/jsRunner/bundles/snippets.ivm.bundle.js --platform=node --format=iife --global-name=snippets", "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers", "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson", @@ -99,7 +99,7 @@ "mysql2": "3.9.8", "node-fetch": "2.6.7", "object-sizeof": "2.6.1", - "openai": "^3.2.1", + "openai": "^4.52.1", "openapi-types": "9.3.1", "pg": "8.10.0", "pouchdb": "7.3.0", @@ -152,7 +152,7 @@ "timekeeper": "2.2.0", "ts-node": "10.8.1", "tsconfig-paths": "4.0.0", - "typescript": "5.2.2", + "typescript": "5.5.2", "update-dotenv": "1.1.1", "yargs": "13.2.4" }, diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index f5c759ec72..e84f475148 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -279,6 +279,31 @@ describe.each([ }) }) + isInternal && + it("shouldn't allow duplicate column names", async () => { + const saveTableRequest: SaveTableRequest = { + ...basicTable(), + } + saveTableRequest.schema["Type"] = { + type: FieldType.STRING, + name: "Type", + } + // allow the "Type" column - internal columns aren't case sensitive + await config.api.table.save(saveTableRequest, { + status: 200, + }) + saveTableRequest.schema.foo = { type: FieldType.STRING, name: "foo" } + saveTableRequest.schema.FOO = { type: FieldType.STRING, name: "FOO" } + + await config.api.table.save(saveTableRequest, { + status: 400, + body: { + message: + 'Column(s) "foo" are duplicated - check for other columns with these name (case in-sensitive)', + }, + }) + }) + it("should add a new column for an internal DB table", async () => { const saveTableRequest: SaveTableRequest = { ...basicTable(), diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index bc926de7b7..45ef5ef703 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -1,4 +1,5 @@ -import { Configuration, OpenAIApi } from "openai" +import { OpenAI } from "openai" + import { AutomationActionStepId, AutomationStepSchema, @@ -75,13 +76,11 @@ export async function run({ inputs }: AutomationStepInput) { } try { - const configuration = new Configuration({ + const openai = new OpenAI({ apiKey: environment.OPENAI_API_KEY, }) - const openai = new OpenAIApi(configuration) - - const completion = await openai.createChatCompletion({ + const completion = await openai.chat.completions.create({ model: inputs.model, messages: [ { @@ -90,8 +89,7 @@ export async function run({ inputs }: AutomationStepInput) { }, ], }) - - const response = completion?.data?.choices[0]?.message?.content + const response = completion?.choices[0]?.message?.content return { response, diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 2d23c4ad32..3e3df371ab 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -1,17 +1,15 @@ import { runStep } from "./utilities" import environment from "../../environment" -import openai from "openai" +import { OpenAI } from "openai" import TestConfiguration from "../../../src/tests/utilities/TestConfiguration" import { AutomationActionStepId } from "@budibase/types" -jest.mock( - "openai", - jest.fn(() => ({ - Configuration: jest.fn(), - OpenAIApi: jest.fn(() => ({ - createChatCompletion: jest.fn(() => ({ - data: { +jest.mock("openai", () => ({ + OpenAI: jest.fn().mockImplementation(() => ({ + chat: { + completions: { + create: jest.fn(() => ({ choices: [ { message: { @@ -19,15 +17,13 @@ jest.mock( }, }, ], - }, - })), - })), - })) -) + })), + }, + }, + })), +})) -const mockedOpenAIApi = openai.OpenAIApi as jest.MockedClass< - typeof openai.OpenAIApi -> +const mockedOpenAI = OpenAI as jest.MockedClass const OPENAI_PROMPT = "What is the meaning of life?" @@ -77,14 +73,18 @@ describe("test the openai action", () => { }) it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => { - mockedOpenAIApi.mockImplementation( + mockedOpenAI.mockImplementation( () => ({ - createChatCompletion: jest.fn(() => { - throw new Error( - "An error occurred while calling createChatCompletion" - ) - }), + chat: { + completions: { + create: jest.fn(() => { + throw new Error( + "An error occurred while calling createChatCompletion" + ) + }), + }, + }, } as any) ) diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index ea40d2bfe9..fc32708708 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -17,6 +17,7 @@ import { cloneDeep } from "lodash/fp" import isEqual from "lodash/isEqual" import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula" import { context } from "@budibase/backend-core" +import { findDuplicateInternalColumns } from "@budibase/shared-core" import { getTable } from "../getters" import { checkAutoColumns } from "./utils" import * as viewsSdk from "../../views" @@ -44,6 +45,17 @@ export async function save( if (hasTypeChanged(table, oldTable)) { throw new Error("A column type has changed.") } + + // check for case sensitivity - we don't want to allow duplicated columns + const duplicateColumn = findDuplicateInternalColumns(table) + if (duplicateColumn.length) { + throw new Error( + `Column(s) "${duplicateColumn.join( + ", " + )}" are duplicated - check for other columns with these name (case in-sensitive)` + ) + } + // check that subtypes have been maintained table = checkAutoColumns(table, oldTable) diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts index e473675633..4bd4e8f583 100644 --- a/packages/server/src/utilities/schema.ts +++ b/packages/server/src/utilities/schema.ts @@ -79,7 +79,6 @@ export function validate(rows: Rows, schema: TableSchema): ValidationResults { } else if ( // If there's no data for this field don't bother with further checks // If the field is already marked as invalid there's no need for further checks - results.schemaValidation[columnName] === false || columnData == null || isAutoColumn ) { diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index 3049afdb95..da74d090b6 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -11,7 +11,7 @@ "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "dev": "tsc -p tsconfig.json --watch --preserveWatchOutput", - "check:types": "tsc -p tsconfig.json --noEmit --paths null", + "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "test": "jest", "test:watch": "yarn test --watchAll" }, @@ -21,7 +21,7 @@ }, "devDependencies": { "rimraf": "3.0.2", - "typescript": "5.2.2" + "typescript": "5.5.2" }, "nx": { "targets": { diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index 82ac0d51c3..c9d1a8fc8f 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,5 +1,6 @@ export * from "./api" export * from "./fields" +export * from "./rows" export const OperatorOptions = { Equals: { diff --git a/packages/shared-core/src/constants/rows.ts b/packages/shared-core/src/constants/rows.ts new file mode 100644 index 0000000000..bfa7595d62 --- /dev/null +++ b/packages/shared-core/src/constants/rows.ts @@ -0,0 +1,14 @@ +export const CONSTANT_INTERNAL_ROW_COLS = [ + "_id", + "_rev", + "type", + "createdAt", + "updatedAt", + "tableId", +] as const + +export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const + +export function isInternalColumnName(name: string): boolean { + return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name) +} diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index bd75406e26..443dbbce8d 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -250,12 +250,16 @@ export const buildQuery = (filter: SearchFilter[]) => { query.equal = query.equal || {} query.equal[field] = true } else { - query[queryOperator] = query[queryOperator] || {} - query[queryOperator]![field] = value + query[queryOperator] = { + ...query[queryOperator], + [field]: value, + } } } else { - query[queryOperator] = query[queryOperator] || {} - query[queryOperator]![field] = value + query[queryOperator] = { + ...query[queryOperator], + [field]: value, + } } } }) diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts index 7706b78037..8fd7909b18 100644 --- a/packages/shared-core/src/table.ts +++ b/packages/shared-core/src/table.ts @@ -1,4 +1,5 @@ -import { FieldType } from "@budibase/types" +import { FieldType, Table } from "@budibase/types" +import { CONSTANT_INTERNAL_ROW_COLS } from "./constants" const allowDisplayColumnByType: Record = { [FieldType.STRING]: true, @@ -51,3 +52,27 @@ export function canBeDisplayColumn(type: FieldType): boolean { export function canBeSortColumn(type: FieldType): boolean { return !!allowSortColumnByType[type] } + +export function findDuplicateInternalColumns(table: Table): string[] { + // maintains the case of keys + const casedKeys = Object.keys(table.schema) + // get the column names + const uncasedKeys = casedKeys.map(colName => colName.toLowerCase()) + // there are duplicates + const set = new Set(uncasedKeys) + let duplicates: string[] = [] + if (set.size !== uncasedKeys.length) { + for (let key of set.keys()) { + const count = uncasedKeys.filter(name => name === key).length + if (count > 1) { + duplicates.push(key) + } + } + } + for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) { + if (casedKeys.find(key => key === internalColumn)) { + duplicates.push(internalColumn) + } + } + return duplicates +} diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 8cf8d92692..238182a5a4 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -21,7 +21,7 @@ "scripts": { "build": "tsc --emitDeclarationOnly && rollup -c", "dev": "rollup -cw", - "check:types": "tsc -p tsconfig.json --noEmit --paths null", + "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "test": "jest", "manifest": "ts-node ./scripts/gen-collection-info.ts" }, @@ -45,6 +45,6 @@ "rollup-plugin-node-resolve": "^5.2.0", "rollup-plugin-terser": "^7.0.2", "ts-jest": "29.1.1", - "typescript": "5.2.2" + "typescript": "5.5.2" } } diff --git a/packages/types/package.json b/packages/types/package.json index f4c7b13344..c44fff971e 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "dev": "tsc -p tsconfig.json --watch --preserveWatchOutput", - "check:types": "tsc -p tsconfig.json --noEmit --paths null" + "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020" }, "jest": {}, "devDependencies": { @@ -20,7 +20,7 @@ "@types/pouchdb": "6.4.0", "@types/redlock": "4.0.7", "rimraf": "3.0.2", - "typescript": "5.2.2" + "typescript": "5.5.2" }, "dependencies": { "scim-patch": "^0.8.1" diff --git a/packages/worker/package.json b/packages/worker/package.json index 95410668da..02f1181e91 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -15,7 +15,7 @@ "prebuild": "rimraf dist/", "build": "node ../../scripts/build.js", "postbuild": "copyfiles -f ../../yarn.lock ./dist/", - "check:types": "tsc -p tsconfig.json --noEmit --paths null", + "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", @@ -91,7 +91,7 @@ "rimraf": "3.0.2", "supertest": "6.3.3", "timekeeper": "2.2.0", - "typescript": "5.2.2", + "typescript": "5.5.2", "update-dotenv": "1.1.1" }, "nx": { diff --git a/yarn.lock b/yarn.lock index d60a9895f9..1af9729165 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5906,6 +5906,14 @@ "@types/node" "*" form-data "^3.0.0" +"@types/node-fetch@^2.6.4": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + "@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.13.4", "@types/node@>=13.7.0", "@types/node@^20.4.5": version "20.12.10" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.10.tgz#8f0c3f12b0f075eee1fe20c1afb417e9765bef76" @@ -7471,7 +7479,7 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^0.26.0, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0: +axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11461,6 +11469,11 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== +form-data-encoder@1.7.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040" + integrity sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A== + form-data@4.0.0, form-data@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" @@ -11497,6 +11510,14 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +formdata-node@^4.3.2: + version "4.4.1" + resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-4.4.1.tgz#23f6a5cb9cb55315912cbec4ff7b0f59bbd191e2" + integrity sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ== + dependencies: + node-domexception "1.0.0" + web-streams-polyfill "4.0.0-beta.3" + formidable@^1.1.1: version "1.2.6" resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168" @@ -16232,6 +16253,11 @@ node-addon-api@^6.1.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== +node-domexception@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" + integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== + node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" @@ -16861,13 +16887,19 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -openai@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/openai/-/openai-3.2.1.tgz#1fa35bdf979cbde8453b43f2dd3a7d401ee40866" - integrity sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A== +openai@^4.52.1: + version "4.52.1" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65" + integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg== dependencies: - axios "^0.26.0" - form-data "^4.0.0" + "@types/node" "^18.11.18" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + agentkeepalive "^4.2.1" + form-data-encoder "1.7.2" + formdata-node "^4.3.2" + node-fetch "^2.6.7" + web-streams-polyfill "^3.2.1" openapi-response-validator@^9.2.0: version "9.3.1" @@ -22299,6 +22331,16 @@ wcwidth@^1.0.0, wcwidth@^1.0.1: dependencies: defaults "^1.0.3" +web-streams-polyfill@4.0.0-beta.3: + version "4.0.0-beta.3" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" + integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== + +web-streams-polyfill@^3.2.1: + version "3.3.3" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" + integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw== + webfinger@^0.4.2: version "0.4.2" resolved "https://registry.yarnpkg.com/webfinger/-/webfinger-0.4.2.tgz#3477a6d97799461896039fcffc650b73468ee76d"