diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index 61ab9a4eb2..f7f87f6e4c 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -20,8 +20,8 @@ jobs: with: root-reserve-mb: 30000 swap-size-mb: 1024 - remove-android: 'true' - remove-dotnet: 'true' + remove-android: "true" + remove-dotnet: "true" - name: Fail if not a tag run: | if [[ $GITHUB_REF != refs/tags/* ]]; then @@ -48,7 +48,7 @@ jobs: - name: Update versions run: ./scripts/updateVersions.sh - name: Run Yarn Build - run: yarn build:docker:pre + run: yarn build - name: Login to Docker Hub uses: docker/login-action@v2 with: diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 95e383edb0..c7b90dbdc4 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -12,14 +12,14 @@ RUN chmod +x /cleanup.sh WORKDIR /app ADD packages/server . COPY yarn.lock . -RUN yarn install --production=true --network-timeout 100000 +RUN yarn install --production=true --network-timeout 1000000 RUN /cleanup.sh # build worker WORKDIR /worker ADD packages/worker . COPY yarn.lock . -RUN yarn install --production=true --network-timeout 100000 +RUN yarn install --production=true --network-timeout 1000000 RUN /cleanup.sh FROM budibase/couchdb diff --git a/lerna.json b/lerna.json index 62a762ec71..34faefc099 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.35", + "version": "2.11.36", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index ba61ede746..7b51e6c839 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -36,7 +36,7 @@ import { FieldType, FieldSubtype, SourceName } from "@budibase/types" import RelationshipSelector from "components/common/RelationshipSelector.svelte" - const AUTO_TYPE = "auto" + const AUTO_TYPE = FIELDS.AUTO.type const FORMULA_TYPE = FIELDS.FORMULA.type const LINK_TYPE = FIELDS.LINK.type const STRING_TYPE = FIELDS.STRING.type @@ -60,8 +60,13 @@ {} ) - function makeFieldId(type, subtype) { - return `${type}${subtype || ""}`.toUpperCase() + function makeFieldId(type, subtype, autocolumn) { + // don't make field IDs for auto types + if (type === AUTO_TYPE || autocolumn) { + return type.toUpperCase() + } else { + return `${type}${subtype || ""}`.toUpperCase() + } } let originalName @@ -183,7 +188,8 @@ if (!savingColumn) { editableColumn.fieldId = makeFieldId( editableColumn.type, - editableColumn.subtype + editableColumn.subtype, + editableColumn.autocolumn ) allowedTypes = getAllowedTypes().map(t => ({ @@ -419,7 +425,7 @@ FIELDS.FORMULA, FIELDS.JSON, isUsers ? FIELDS.USERS : FIELDS.USER, - { name: "Auto Column", type: AUTO_TYPE }, + FIELDS.AUTO, ] } else { let fields = [ @@ -538,7 +544,7 @@ getOptionValue={field => field.fieldId} getOptionIcon={field => field.icon} isOptionEnabled={option => { - if (option.type == AUTO_TYPE) { + if (option.type === AUTO_TYPE) { return availableAutoColumnKeys?.length > 0 } return true diff --git a/packages/builder/src/components/backend/Datasources/TableImportSelection/index.svelte b/packages/builder/src/components/backend/Datasources/TableImportSelection/index.svelte index 1fc83d4978..3bc2457c99 100644 --- a/packages/builder/src/components/backend/Datasources/TableImportSelection/index.svelte +++ b/packages/builder/src/components/backend/Datasources/TableImportSelection/index.svelte @@ -57,7 +57,7 @@ {#if $store.error} {/if} diff --git a/packages/builder/src/components/backend/Datasources/TableImportSelection/tableSelectionStore.js b/packages/builder/src/components/backend/Datasources/TableImportSelection/tableSelectionStore.js index 6235ea397a..5c2ea9767c 100644 --- a/packages/builder/src/components/backend/Datasources/TableImportSelection/tableSelectionStore.js +++ b/packages/builder/src/components/backend/Datasources/TableImportSelection/tableSelectionStore.js @@ -1,6 +1,6 @@ import { derived, writable, get } from "svelte/store" import { keepOpen, notifications } from "@budibase/bbui" -import { datasources, ImportTableError, tables } from "stores/backend" +import { datasources, tables } from "stores/backend" export const createTableSelectionStore = (integration, datasource) => { const tableNamesStore = writable([]) @@ -30,12 +30,7 @@ export const createTableSelectionStore = (integration, datasource) => { notifications.success(`Tables fetched successfully.`) await onComplete() } catch (err) { - if (err instanceof ImportTableError) { - errorStore.set(err) - } else { - notifications.error("Error fetching tables.") - } - + errorStore.set(err) return keepOpen } } diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index a81b33c2d3..ac4079b69e 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -1,5 +1,21 @@ import { FieldType, FieldSubtype } from "@budibase/types" +export const AUTO_COLUMN_SUB_TYPES = { + AUTO_ID: "autoID", + CREATED_BY: "createdBy", + CREATED_AT: "createdAt", + UPDATED_BY: "updatedBy", + UPDATED_AT: "updatedAt", +} + +export const AUTO_COLUMN_DISPLAY_NAMES = { + AUTO_ID: "Auto ID", + CREATED_BY: "Created By", + CREATED_AT: "Created At", + UPDATED_BY: "Updated By", + UPDATED_AT: "Updated At", +} + export const FIELDS = { STRING: { name: "Text", @@ -107,6 +123,12 @@ export const FIELDS = { presence: false, }, }, + AUTO: { + name: "Auto Column", + type: FieldType.AUTO, + icon: "MagicWand", + constraints: {}, + }, FORMULA: { name: "Formula", type: FieldType.FORMULA, @@ -139,22 +161,6 @@ export const FIELDS = { }, } -export const AUTO_COLUMN_SUB_TYPES = { - AUTO_ID: "autoID", - CREATED_BY: "createdBy", - CREATED_AT: "createdAt", - UPDATED_BY: "updatedBy", - UPDATED_AT: "updatedAt", -} - -export const AUTO_COLUMN_DISPLAY_NAMES = { - AUTO_ID: "Auto ID", - CREATED_BY: "Created By", - CREATED_AT: "Created At", - UPDATED_BY: "Updated By", - UPDATED_AT: "Updated At", -} - export const FILE_TYPES = { IMAGE: ["png", "tiff", "gif", "raw", "jpg", "jpeg"], CODE: ["js", "rs", "py", "java", "rb", "hs", "yml"], diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 00384a6b1c..11184f2caa 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -9,15 +9,19 @@ import { API } from "api" import { DatasourceFeature } from "@budibase/types" import { TableNames } from "constants" -export class ImportTableError extends Error { - constructor(message) { - super(message) - const [title, description] = message.split(" - ") +class TableImportError extends Error { + constructor(errors) { + super() + this.name = "TableImportError" + this.errors = errors + } - this.name = "TableSelectionError" - // Capitalize the first character of both the title and description - this.title = title[0].toUpperCase() + title.substr(1) - this.description = description[0].toUpperCase() + description.substr(1) + get description() { + let message = "" + for (const key in this.errors) { + message += `${key}: ${this.errors[key]}\n` + } + return message } } @@ -25,7 +29,6 @@ export function createDatasourcesStore() { const store = writable({ list: [], selectedDatasourceId: null, - schemaError: null, }) const derivedStore = derived([store, tables], ([$store, $tables]) => { @@ -75,18 +78,13 @@ export function createDatasourcesStore() { store.update(state => ({ ...state, selectedDatasourceId: id, - // Remove any possible schema error - schemaError: null, })) } const updateDatasource = response => { - const { datasource, error } = response - if (error) { - store.update(state => ({ - ...state, - schemaError: error, - })) + const { datasource, errors } = response + if (errors && Object.keys(errors).length > 0) { + throw new TableImportError(errors) } replaceDatasource(datasource._id, datasource) select(datasource._id) @@ -94,20 +92,11 @@ export function createDatasourcesStore() { } const updateSchema = async (datasource, tablesFilter) => { - try { - const response = await API.buildDatasourceSchema({ - datasourceId: datasource?._id, - tablesFilter, - }) - updateDatasource(response) - } catch (e) { - // buildDatasourceSchema call returns user presentable errors with two parts divided with a " - ". - if (e.message.split(" - ").length === 2) { - throw new ImportTableError(e.message) - } else { - throw e - } - } + const response = await API.buildDatasourceSchema({ + datasourceId: datasource?._id, + tablesFilter, + }) + updateDatasource(response) } const sourceCount = source => { @@ -172,12 +161,6 @@ export function createDatasourcesStore() { replaceDatasource(datasource._id, null) } - const removeSchemaError = () => { - store.update(state => { - return { ...state, schemaError: null } - }) - } - const replaceDatasource = (datasourceId, datasource) => { if (!datasourceId) { return @@ -230,7 +213,6 @@ export function createDatasourcesStore() { create, update, delete: deleteDatasource, - removeSchemaError, replaceDatasource, getTableNames, } diff --git a/packages/builder/src/stores/backend/index.js b/packages/builder/src/stores/backend/index.js index 278e43c1ed..3781e2ab92 100644 --- a/packages/builder/src/stores/backend/index.js +++ b/packages/builder/src/stores/backend/index.js @@ -4,7 +4,7 @@ export { views } from "./views" export { viewsV2 } from "./viewsV2" export { permissions } from "./permissions" export { roles } from "./roles" -export { datasources, ImportTableError } from "./datasources" +export { datasources } from "./datasources" export { integrations } from "./integrations" export { sortedIntegrations } from "./sortedIntegrations" export { queries } from "./queries" diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 399d5f1d0c..8e6a0620da 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -5,7 +5,6 @@ import { getTableParams, } from "../../db/utils" import { destroy as tableDestroy } from "./table/internal" -import { BuildSchemaErrors, InvalidColumns } from "../../constants" import { getIntegration } from "../../integrations" import { invalidateDynamicVariables } from "../../threads/utils" import { context, db as dbCore, events } from "@budibase/backend-core" @@ -14,10 +13,13 @@ import { CreateDatasourceResponse, Datasource, DatasourcePlus, + ExternalTable, FetchDatasourceInfoRequest, FetchDatasourceInfoResponse, IntegrationBase, + Schema, SourceName, + Table, UpdateDatasourceResponse, UserCtx, VerifyDatasourceRequest, @@ -27,23 +29,6 @@ import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets" -function getErrorTables(errors: any, errorType: string) { - return Object.entries(errors) - .filter(entry => entry[1] === errorType) - .map(([name]) => name) -} - -function updateError(error: any, newError: any, tables: string[]) { - if (!error) { - error = "" - } - if (error.length > 0) { - error += "\n" - } - error += `${newError} ${tables.join(", ")}` - return error -} - async function getConnector( datasource: Datasource ): Promise { @@ -71,48 +56,36 @@ async function getAndMergeDatasource(datasource: Datasource) { return await sdk.datasources.enrich(enrichedDatasource) } -async function buildSchemaHelper(datasource: Datasource) { +async function buildSchemaHelper(datasource: Datasource): Promise { const connector = (await getConnector(datasource)) as DatasourcePlus - await connector.buildSchema(datasource._id!, datasource.entities!) - - const errors = connector.schemaErrors - let error = null - if (errors && Object.keys(errors).length > 0) { - const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY) - const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN) - if (noKey.length) { - error = updateError( - error, - "No primary key constraint found for the following:", - noKey - ) - } - if (invalidCol.length) { - const invalidCols = Object.values(InvalidColumns).join(", ") - error = updateError( - error, - `Cannot use columns ${invalidCols} found in following:`, - invalidCol - ) - } - } - return { tables: connector.tables, error } + return await connector.buildSchema( + datasource._id!, + datasource.entities! as Record + ) } -async function buildFilteredSchema(datasource: Datasource, filter?: string[]) { - let { tables, error } = await buildSchemaHelper(datasource) - let finalTables = tables - if (filter) { - finalTables = {} - for (let key in tables) { - if ( - filter.some((filter: any) => filter.toLowerCase() === key.toLowerCase()) - ) { - finalTables[key] = tables[key] - } +async function buildFilteredSchema( + datasource: Datasource, + filter?: string[] +): Promise { + let schema = await buildSchemaHelper(datasource) + if (!filter) { + return schema + } + + let filteredSchema: Schema = { tables: {}, errors: {} } + for (let key in schema.tables) { + if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { + filteredSchema.tables[key] = schema.tables[key] } } - return { tables: finalTables, error } + + for (let key in schema.errors) { + if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { + filteredSchema.errors[key] = schema.errors[key] + } + } + return filteredSchema } export async function fetch(ctx: UserCtx) { @@ -156,7 +129,7 @@ export async function buildSchemaFromDb(ctx: UserCtx) { const tablesFilter = ctx.request.body.tablesFilter const datasource = await sdk.datasources.get(ctx.params.datasourceId) - const { tables, error } = await buildFilteredSchema(datasource, tablesFilter) + const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter) datasource.entities = tables setDefaultDisplayColumns(datasource) @@ -164,13 +137,11 @@ export async function buildSchemaFromDb(ctx: UserCtx) { sdk.tables.populateExternalTableSchemas(datasource) ) datasource._rev = dbResp.rev - const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource) - const res: any = { datasource: cleanedDatasource } - if (error) { - res.error = error + ctx.body = { + datasource: await sdk.datasources.removeSecretSingle(datasource), + errors, } - ctx.body = res } /** @@ -298,15 +269,12 @@ export async function save( type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE, } - let schemaError = null + let errors: Record = {} if (fetchSchema) { - const { tables, error } = await buildFilteredSchema( - datasource, - tablesFilter - ) - schemaError = error - datasource.entities = tables + const schema = await buildFilteredSchema(datasource, tablesFilter) + datasource.entities = schema.tables setDefaultDisplayColumns(datasource) + errors = schema.errors } if (preSaveAction[datasource.source]) { @@ -327,13 +295,10 @@ export async function save( } } - const response: CreateDatasourceResponse = { + ctx.body = { datasource: await sdk.datasources.removeSecretSingle(datasource), + errors, } - if (schemaError) { - response.error = schemaError - } - ctx.body = response builderSocket?.emitDatasourceUpdate(ctx, datasource) } diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 5019073db4..3c1d7839e8 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -37,7 +37,7 @@ describe("/datasources", () => { .expect(200) expect(res.body.datasource.name).toEqual("Test") - expect(res.body.errors).toBeUndefined() + expect(res.body.errors).toEqual({}) expect(events.datasource.created).toBeCalledTimes(1) }) }) diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 326389996d..b37a4b36c1 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -159,11 +159,6 @@ export enum InvalidColumns { TABLE_ID = "tableId", } -export enum BuildSchemaErrors { - NO_KEY = "no_key", - INVALID_COLUMN = "invalid_column", -} - export enum AutomationErrors { INCORRECT_TYPE = "INCORRECT_TYPE", MAX_ITERATIONS = "MAX_ITERATIONS_REACHED", diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 84c19f8bbc..90f0fc9f2c 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -18,6 +18,7 @@ import _ from "lodash" import { generator } from "@budibase/backend-core/tests" import { utils } from "@budibase/backend-core" import { databaseTestProviders } from "../integrations/tests/utils" +import { Client } from "pg" const config = setup.getConfig()! @@ -1055,4 +1056,46 @@ describe("postgres integrations", () => { expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1) }) }) + + describe("POST /api/datasources/:datasourceId/schema", () => { + let client: Client + + beforeEach(async () => { + client = new Client( + (await databaseTestProviders.postgres.getDsConfig()).config! + ) + await client.connect() + }) + + afterEach(async () => { + await client.query(`DROP TABLE IF EXISTS "table"`) + await client.end() + }) + + it("recognises when a table has no primary key", async () => { + await client.query(`CREATE TABLE "table" (id SERIAL)`) + + const response = await makeRequest( + "post", + `/api/datasources/${postgresDatasource._id}/schema` + ) + + expect(response.body.errors).toEqual({ + table: "Table must have a primary key.", + }) + }) + + it("recognises when a table is using a reserved column name", async () => { + await client.query(`CREATE TABLE "table" (_id SERIAL PRIMARY KEY) `) + + const response = await makeRequest( + "post", + `/api/datasources/${postgresDatasource._id}/schema` + ) + + expect(response.body.errors).toEqual({ + table: "Table contains invalid columns.", + }) + }) + }) }) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 5360d6b319..57b6682cc8 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -14,9 +14,14 @@ import { SortJson, ExternalTable, TableRequest, + Schema, } from "@budibase/types" import { OAuth2Client } from "google-auth-library" -import { buildExternalTableId, finaliseExternalTables } from "./utils" +import { + buildExternalTableId, + checkExternalTables, + finaliseExternalTables, +} from "./utils" import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet" import fetch from "node-fetch" import { cache, configs, context, HTTPError } from "@budibase/backend-core" @@ -138,8 +143,6 @@ const SCHEMA: Integration = { class GoogleSheetsIntegration implements DatasourcePlus { private readonly config: GoogleSheetsConfig private client: GoogleSpreadsheet - public tables: Record = {} - public schemaErrors: Record = {} constructor(config: GoogleSheetsConfig) { this.config = config @@ -281,19 +284,37 @@ class GoogleSheetsIntegration implements DatasourcePlus { async buildSchema( datasourceId: string, entities: Record - ) { + ): Promise { // not fully configured yet if (!this.config.auth) { - return + return { tables: {}, errors: {} } } await this.connect() const sheets = this.client.sheetsByIndex const tables: Record = {} + let errors: Record = {} await utils.parallelForeach( sheets, async sheet => { // must fetch rows to determine schema - await sheet.getRows() + try { + await sheet.getRows() + } catch (err) { + // We expect this to always be an Error so if it's not, rethrow it to + // make sure we don't fail quietly. + if (!(err instanceof Error)) { + throw err + } + + if (err.message.startsWith("No values in the header row")) { + errors[sheet.title] = err.message + } else { + // If we get an error we don't expect, rethrow to avoid failing + // quietly. + throw err + } + return + } const id = buildExternalTableId(datasourceId, sheet.title) tables[sheet.title] = this.getTableSchema( @@ -305,9 +326,9 @@ class GoogleSheetsIntegration implements DatasourcePlus { }, 10 ) - const final = finaliseExternalTables(tables, entities) - this.tables = final.tables - this.schemaErrors = final.errors + let externalTables = finaliseExternalTables(tables, entities) + errors = { ...errors, ...checkExternalTables(externalTables) } + return { tables: externalTables, errors } } async query(json: QueryJson) { diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index cd62e590d8..06ffaf955d 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -11,6 +11,7 @@ import { DatasourceFeature, ConnectionInfo, SourceName, + Schema, } from "@budibase/types" import { getSqlQuery, @@ -18,6 +19,7 @@ import { convertSqlType, finaliseExternalTables, SqlClient, + checkExternalTables, } from "./utils" import Sql from "./base/sql" import { MSSQLTablesResponse, MSSQLColumn } from "./base/types" @@ -190,8 +192,6 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { private readonly config: MSSQLConfig private index: number = 0 private client?: sqlServer.ConnectionPool - public tables: Record = {} - public schemaErrors: Record = {} MASTER_TABLES = [ "spt_fallback_db", @@ -381,7 +381,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { async buildSchema( datasourceId: string, entities: Record - ) { + ): Promise { await this.connect() let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL) if (tableInfo == null || !Array.isArray(tableInfo)) { @@ -445,9 +445,12 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { schema, } } - const final = finaliseExternalTables(tables, entities) - this.tables = final.tables - this.schemaErrors = final.errors + let externalTables = finaliseExternalTables(tables, entities) + let errors = checkExternalTables(externalTables) + return { + tables: externalTables, + errors, + } } async queryTableNames() { diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 8a688c5f3b..3a954da9bd 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -10,6 +10,7 @@ import { DatasourceFeature, ConnectionInfo, SourceName, + Schema, } from "@budibase/types" import { getSqlQuery, @@ -17,6 +18,7 @@ import { buildExternalTableId, convertSqlType, finaliseExternalTables, + checkExternalTables, } from "./utils" import dayjs from "dayjs" import { NUMBER_REGEX } from "../utilities" @@ -140,8 +142,6 @@ export function bindingTypeCoerce(bindings: any[]) { class MySQLIntegration extends Sql implements DatasourcePlus { private config: MySQLConfig private client?: mysql.Connection - public tables: Record = {} - public schemaErrors: Record = {} constructor(config: MySQLConfig) { super(SqlClient.MY_SQL) @@ -279,7 +279,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus { async buildSchema( datasourceId: string, entities: Record - ) { + ): Promise { const tables: { [key: string]: ExternalTable } = {} await this.connect() @@ -328,9 +328,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus { } finally { await this.disconnect() } - const final = finaliseExternalTables(tables, entities) - this.tables = final.tables - this.schemaErrors = final.errors + + let externalTables = finaliseExternalTables(tables, entities) + let errors = checkExternalTables(tables) + return { tables: externalTables, errors } } async queryTableNames() { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 38f0a9d5ac..28d8fdd84d 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -9,9 +9,11 @@ import { DatasourcePlus, DatasourceFeature, ConnectionInfo, + Schema, } from "@budibase/types" import { buildExternalTableId, + checkExternalTables, convertSqlType, finaliseExternalTables, getSqlQuery, @@ -108,9 +110,6 @@ class OracleIntegration extends Sql implements DatasourcePlus { private readonly config: OracleConfig private index: number = 1 - public tables: Record = {} - public schemaErrors: Record = {} - private readonly COLUMNS_SQL = ` SELECT tabs.table_name, @@ -265,7 +264,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { async buildSchema( datasourceId: string, entities: Record - ) { + ): Promise { const columnsResponse = await this.internalQuery({ sql: this.COLUMNS_SQL, }) @@ -326,9 +325,9 @@ class OracleIntegration extends Sql implements DatasourcePlus { }) }) - const final = finaliseExternalTables(tables, entities) - this.tables = final.tables - this.schemaErrors = final.errors + let externalTables = finaliseExternalTables(tables, entities) + let errors = checkExternalTables(externalTables) + return { tables: externalTables, errors } } async getTableNames() { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index c4b7c2bb65..ef63f39d87 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -10,6 +10,7 @@ import { DatasourceFeature, ConnectionInfo, SourceName, + Schema, } from "@budibase/types" import { getSqlQuery, @@ -17,6 +18,7 @@ import { convertSqlType, finaliseExternalTables, SqlClient, + checkExternalTables, } from "./utils" import Sql from "./base/sql" import { PostgresColumn } from "./base/types" @@ -145,8 +147,6 @@ class PostgresIntegration extends Sql implements DatasourcePlus { private readonly config: PostgresConfig private index: number = 1 private open: boolean - public tables: Record = {} - public schemaErrors: Record = {} COLUMNS_SQL!: string @@ -274,7 +274,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { async buildSchema( datasourceId: string, entities: Record - ) { + ): Promise { let tableKeys: { [key: string]: string[] } = {} await this.openConnection() try { @@ -342,9 +342,9 @@ class PostgresIntegration extends Sql implements DatasourcePlus { } } - const final = finaliseExternalTables(tables, entities) - this.tables = final.tables - this.schemaErrors = final.errors + let finalizedTables = finaliseExternalTables(tables, entities) + let errors = checkExternalTables(finalizedTables) + return { tables: finalizedTables, errors } } catch (err) { // @ts-ignore throw new Error(err) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index db562473e3..79b18e767c 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -4,13 +4,10 @@ import { SearchFilters, Datasource, FieldType, + ExternalTable, } from "@budibase/types" import { DocumentType, SEPARATOR } from "../db/utils" -import { - BuildSchemaErrors, - InvalidColumns, - NoEmptyFilterStrings, -} from "../constants" +import { InvalidColumns, NoEmptyFilterStrings } from "../constants" import { helpers } from "@budibase/shared-core" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` @@ -266,9 +263,9 @@ export function shouldCopySpecialColumn( function copyExistingPropsOver( tableName: string, table: Table, - entities: { [key: string]: any }, - tableIds: [string] -) { + entities: Record, + tableIds: string[] +): Table { if (entities && entities[tableName]) { if (entities[tableName]?.primaryDisplay) { table.primaryDisplay = entities[tableName].primaryDisplay @@ -295,42 +292,41 @@ function copyExistingPropsOver( /** * Look through the final table definitions to see if anything needs to be - * copied over from the old and if any errors have occurred mark them so - * that the user can be made aware. + * copied over from the old. * @param tables The list of tables that have been retrieved from the external database. * @param entities The old list of tables, if there was any to look for definitions in. */ export function finaliseExternalTables( - tables: { [key: string]: any }, - entities: { [key: string]: any } -) { - const invalidColumns = Object.values(InvalidColumns) - let finalTables: { [key: string]: any } = {} - const errors: { [key: string]: string } = {} - // @ts-ignore - const tableIds: [string] = Object.values(tables).map(table => table._id) + tables: Record, + entities: Record +): Record { + let finalTables: Record = {} + const tableIds = Object.values(tables).map(table => table._id!) for (let [name, table] of Object.entries(tables)) { - const schemaFields = Object.keys(table.schema) - // make sure every table has a key - if (table.primary == null || table.primary.length === 0) { - errors[name] = BuildSchemaErrors.NO_KEY - continue - } else if ( - schemaFields.find(field => - invalidColumns.includes(field as InvalidColumns) - ) - ) { - errors[name] = BuildSchemaErrors.INVALID_COLUMN - continue - } - // make sure all previous props have been added back finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds) } - // sort the tables by name - finalTables = Object.entries(finalTables) + // sort the tables by name, this is for the UI to display them in alphabetical order + return Object.entries(finalTables) .sort(([a], [b]) => a.localeCompare(b)) .reduce((r, [k, v]) => ({ ...r, [k]: v }), {}) - return { tables: finalTables, errors } +} + +export function checkExternalTables( + tables: Record +): Record { + const invalidColumns = Object.values(InvalidColumns) as string[] + const errors: Record = {} + for (let [name, table] of Object.entries(tables)) { + if (!table.primary || table.primary.length === 0) { + errors[name] = "Table must have a primary key." + } + + const schemaFields = Object.keys(table.schema) + if (schemaFields.find(f => invalidColumns.includes(f))) { + errors[name] = "Table contains invalid columns." + } + } + return errors } /** diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index d0688a24d3..9cd3c8f4bb 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -2,7 +2,7 @@ import { Datasource } from "../../../documents" export interface CreateDatasourceResponse { datasource: Datasource - error?: any + errors: Record } export interface UpdateDatasourceResponse { diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 0e06b8fae0..39a10961de 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -1,4 +1,4 @@ -import { Table } from "../documents" +import { ExternalTable, Table } from "../documents" export const PASSWORD_REPLACEMENT = "--secret-value--" @@ -175,14 +175,19 @@ export interface IntegrationBase { }): void } -export interface DatasourcePlus extends IntegrationBase { - tables: Record - schemaErrors: Record +export interface Schema { + tables: Record + errors: Record +} +export interface DatasourcePlus extends IntegrationBase { // if the datasource supports the use of bindings directly (to protect against SQL injection) // this returns the format of the identifier getBindingIdentifier(): string getStringConcat(parts: string[]): string - buildSchema(datasourceId: string, entities: Record): any + buildSchema( + datasourceId: string, + entities: Record + ): Promise getTableNames(): Promise }