From ce6c5bfa683ad1124ed513b7e4dca6c7982e812c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 May 2023 15:47:55 +0100 Subject: [PATCH 1/2] Base connection work - extending the base integration to include the option of a connection check function. --- packages/server/src/integrations/airtable.ts | 10 ++++++++-- packages/server/src/integrations/arangodb.ts | 6 ++++++ packages/server/src/integrations/couchdb.ts | 6 ++++++ packages/server/src/integrations/dynamodb.ts | 2 ++ packages/server/src/integrations/elasticsearch.ts | 2 ++ packages/server/src/integrations/firebase.ts | 2 ++ packages/server/src/integrations/googlesheets.ts | 2 ++ .../server/src/integrations/microsoftSqlServer.ts | 2 ++ packages/server/src/integrations/mongodb.ts | 2 ++ packages/server/src/integrations/mysql.ts | 2 ++ packages/server/src/integrations/oracle.ts | 2 ++ packages/server/src/integrations/postgres.ts | 2 ++ packages/server/src/integrations/redis.ts | 11 +++++++++-- packages/server/src/integrations/s3.ts | 2 ++ packages/server/src/integrations/snowflake.ts | 8 +++++++- packages/types/src/sdk/datasources.ts | 10 ++++++++++ 16 files changed, 66 insertions(+), 5 deletions(-) diff --git a/packages/server/src/integrations/airtable.ts b/packages/server/src/integrations/airtable.ts index 1f56f0619b..58232ad56c 100644 --- a/packages/server/src/integrations/airtable.ts +++ b/packages/server/src/integrations/airtable.ts @@ -1,8 +1,9 @@ import { - Integration, + DatasourceFeature, DatasourceFieldType, - QueryType, + Integration, IntegrationBase, + QueryType, } from "@budibase/types" const Airtable = require("airtable") @@ -18,6 +19,7 @@ const SCHEMA: Integration = { "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.", friendlyName: "Airtable", type: "Spreadsheet", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { apiKey: { type: DatasourceFieldType.PASSWORD, @@ -88,6 +90,10 @@ class AirtableIntegration implements IntegrationBase { this.client = new Airtable(config).base(config.base) } + async connection() { + return { connected: true } + } + async create(query: { table: any; json: any }) { const { table, json } = query diff --git a/packages/server/src/integrations/arangodb.ts b/packages/server/src/integrations/arangodb.ts index e28940f36e..3a61193577 100644 --- a/packages/server/src/integrations/arangodb.ts +++ b/packages/server/src/integrations/arangodb.ts @@ -3,6 +3,7 @@ import { DatasourceFieldType, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" const { Database, aql } = require("arangojs") @@ -21,6 +22,7 @@ const SCHEMA: Integration = { type: "Non-relational", description: "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { url: { type: DatasourceFieldType.STRING, @@ -74,6 +76,10 @@ class ArangoDBIntegration implements IntegrationBase { this.client = new Database(newConfig) } + async connection() { + return { connected: true } + } + async read(query: { sql: any }) { try { const result = await this.client.query(query.sql) diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index 257b84ca13..dfb0daa2e8 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -1,4 +1,5 @@ import { + DatasourceFeature, DatasourceFieldType, Document, Integration, @@ -18,6 +19,7 @@ const SCHEMA: Integration = { type: "Non-relational", description: "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { url: { type: DatasourceFieldType.STRING, @@ -69,6 +71,10 @@ class CouchDBIntegration implements IntegrationBase { this.client = dbCore.DatabaseWithConnection(config.database, config.url) } + async connection() { + return { connected: true } + } + async query( command: string, errorMsg: string, diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index 28c1c7b52b..c682f355f9 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -3,6 +3,7 @@ import { DatasourceFieldType, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" import AWS from "aws-sdk" @@ -22,6 +23,7 @@ const SCHEMA: Integration = { "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.", friendlyName: "DynamoDB", type: "Non-relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { region: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index aeba628d30..475ddc0bd1 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -3,6 +3,7 @@ import { DatasourceFieldType, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" import { Client, ClientOptions } from "@elastic/elasticsearch" @@ -20,6 +21,7 @@ const SCHEMA: Integration = { "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.", friendlyName: "ElasticSearch", type: "Non-relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { url: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/firebase.ts b/packages/server/src/integrations/firebase.ts index a82b3be782..0646c98eba 100644 --- a/packages/server/src/integrations/firebase.ts +++ b/packages/server/src/integrations/firebase.ts @@ -3,6 +3,7 @@ import { Integration, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" import { Firestore, WhereFilterOp } from "@google-cloud/firestore" @@ -18,6 +19,7 @@ const SCHEMA: Integration = { type: "Non-relational", description: "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { email: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 476a6511e9..8884c8de13 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -1,4 +1,5 @@ import { + DatasourceFeature, DatasourceFieldType, DatasourcePlus, FieldType, @@ -64,6 +65,7 @@ const SCHEMA: Integration = { "Create and collaborate on online spreadsheets in real-time and from any device. ", friendlyName: "Google Sheets", type: "Spreadsheet", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { spreadsheetId: { display: "Google Sheet URL", diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index eb87c1ccf1..b6f360ce32 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -8,6 +8,7 @@ import { QueryType, SqlQuery, DatasourcePlus, + DatasourceFeature, } from "@budibase/types" import { getSqlQuery, @@ -39,6 +40,7 @@ const SCHEMA: Integration = { "Microsoft SQL Server is a relational database management system developed by Microsoft. ", friendlyName: "MS SQL Server", type: "Relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { user: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 38b3891fe4..20ba2acada 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -3,6 +3,7 @@ import { DatasourceFieldType, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" import { MongoClient, @@ -38,6 +39,7 @@ const getSchema = () => { type: "Non-relational", description: "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { connectionString: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 8d984ed402..6083223042 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -7,6 +7,7 @@ import { Table, TableSchema, DatasourcePlus, + DatasourceFeature, } from "@budibase/types" import { getSqlQuery, @@ -41,6 +42,7 @@ const SCHEMA: Integration = { type: "Relational", description: "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 65e0829905..bbe2189bdc 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -7,6 +7,7 @@ import { SqlQuery, Table, DatasourcePlus, + DatasourceFeature, } from "@budibase/types" import { buildExternalTableId, @@ -53,6 +54,7 @@ const SCHEMA: Integration = { type: "Relational", description: "Oracle Database is an object-relational database management system developed by Oracle Corporation", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index c981c3acc5..a968b71064 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -6,6 +6,7 @@ import { SqlQuery, Table, DatasourcePlus, + DatasourceFeature, } from "@budibase/types" import { getSqlQuery, @@ -50,6 +51,7 @@ const SCHEMA: Integration = { type: "Relational", description: "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index 73ef2bb55c..f596b6eb0d 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -1,4 +1,9 @@ -import { DatasourceFieldType, Integration, QueryType } from "@budibase/types" +import { + DatasourceFeature, + DatasourceFieldType, + Integration, + QueryType, +} from "@budibase/types" import Redis from "ioredis" interface RedisConfig { @@ -11,9 +16,11 @@ interface RedisConfig { const SCHEMA: Integration = { docs: "https://redis.io/docs/", - description: "", + description: + "Redis is a caching tool, providing powerful key-value store capabilities.", friendlyName: "Redis", type: "Non-relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { host: { type: "string", diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index ad3bb09109..cc368760f8 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -3,6 +3,7 @@ import { QueryType, IntegrationBase, DatasourceFieldType, + DatasourceFeature, } from "@budibase/types" const AWS = require("aws-sdk") @@ -22,6 +23,7 @@ const SCHEMA: Integration = { "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.", friendlyName: "Amazon S3", type: "Object store", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { region: { type: "string", diff --git a/packages/server/src/integrations/snowflake.ts b/packages/server/src/integrations/snowflake.ts index db702520f9..877405d447 100644 --- a/packages/server/src/integrations/snowflake.ts +++ b/packages/server/src/integrations/snowflake.ts @@ -1,4 +1,9 @@ -import { Integration, QueryType, SqlQuery } from "@budibase/types" +import { + DatasourceFeature, + Integration, + QueryType, + SqlQuery, +} from "@budibase/types" import { Snowflake } from "snowflake-promise" interface SnowflakeConfig { @@ -16,6 +21,7 @@ const SCHEMA: Integration = { "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.", friendlyName: "Snowflake", type: "Relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { account: { type: "string", diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 605b431d9e..ccd6f7b9c6 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -74,6 +74,10 @@ export enum FilterType { ONE_OF = "oneOf", } +export enum DatasourceFeature { + CONNECTION_CHECKING = "connection", +} + export interface StepDefinition { key: string template: string @@ -112,6 +116,7 @@ export interface Integration { docs: string plus?: boolean auth?: { type: string } + features?: DatasourceFeature[] relationships?: boolean description: string friendlyName: string @@ -124,11 +129,16 @@ export interface Integration { extra?: ExtraQueryConfig } +export interface ConnectionInformation { + connected: boolean +} + export interface IntegrationBase { create?(query: any): Promise read?(query: any): Promise update?(query: any): Promise delete?(query: any): Promise + connection?(): Promise } export interface DatasourcePlus extends IntegrationBase { From 57a633b926f5fbcedc53e549bd63314a1e8d6870 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 May 2023 17:40:39 +0100 Subject: [PATCH 2/2] Implementing main body of connection verification endpoint. --- .../server/src/api/controllers/datasource.ts | 121 +++++++++++------- packages/server/src/api/routes/datasource.ts | 5 + packages/types/src/api/web/app/datasource.ts | 8 ++ 3 files changed, 85 insertions(+), 49 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index b61b168980..e6866d528c 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -18,11 +18,68 @@ import { Row, CreateDatasourceResponse, UpdateDatasourceResponse, - UpdateDatasourceRequest, CreateDatasourceRequest, + VerifyDatasourceRequest, + VerifyDatasourceResponse, + IntegrationBase, + DatasourcePlus, } from "@budibase/types" import sdk from "../../sdk" +function getErrorTables(errors: any, errorType: string) { + return Object.entries(errors) + .filter(entry => entry[1] === errorType) + .map(([name]) => name) +} + +function updateError(error: any, newError: any, tables: string[]) { + if (!error) { + error = "" + } + if (error.length > 0) { + error += "\n" + } + error += `${newError} ${tables.join(", ")}` + return error +} + +async function getConnector( + datasource: Datasource +): Promise { + const Connector = await getIntegration(datasource.source) + datasource = await sdk.datasources.enrich(datasource) + // Connect to the DB and build the schema + return new Connector(datasource.config) +} + +async function buildSchemaHelper(datasource: Datasource) { + const connector = (await getConnector(datasource)) as DatasourcePlus + await connector.buildSchema(datasource._id!, datasource.entities!) + + const errors = connector.schemaErrors + let error = null + if (errors && Object.keys(errors).length > 0) { + const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY) + const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN) + if (noKey.length) { + error = updateError( + error, + "No primary key constraint found for the following:", + noKey + ) + } + if (invalidCol.length) { + const invalidCols = Object.values(InvalidColumns).join(", ") + error = updateError( + error, + `Cannot use columns ${invalidCols} found in following:`, + invalidCol + ) + } + } + return { tables: connector.tables, error } +} + export async function fetch(ctx: UserCtx) { // Get internal tables const db = context.getAppDB() @@ -66,6 +123,20 @@ export async function fetch(ctx: UserCtx) { ctx.body = [bbInternalDb, ...datasources] } +export async function verify( + ctx: UserCtx +) { + const datasource = ctx.request.body.datasource + const connector = (await getConnector(datasource)) as IntegrationBase + if (!connector.connection) { + ctx.throw(400, "Connection information verification not supported") + } + const connectionInfo = await connector.connection() + ctx.body = { + connected: connectionInfo.connected, + } +} + export async function buildSchemaFromDb(ctx: UserCtx) { const db = context.getAppDB() const datasource = await sdk.datasources.get(ctx.params.datasourceId) @@ -311,51 +382,3 @@ export async function query(ctx: UserCtx) { ctx.throw(400, err) } } - -function getErrorTables(errors: any, errorType: string) { - return Object.entries(errors) - .filter(entry => entry[1] === errorType) - .map(([name]) => name) -} - -function updateError(error: any, newError: any, tables: string[]) { - if (!error) { - error = "" - } - if (error.length > 0) { - error += "\n" - } - error += `${newError} ${tables.join(", ")}` - return error -} - -async function buildSchemaHelper(datasource: Datasource) { - const Connector = await getIntegration(datasource.source) - datasource = await sdk.datasources.enrich(datasource) - // Connect to the DB and build the schema - const connector = new Connector(datasource.config) - await connector.buildSchema(datasource._id, datasource.entities) - - const errors = connector.schemaErrors - let error = null - if (errors && Object.keys(errors).length > 0) { - const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY) - const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN) - if (noKey.length) { - error = updateError( - error, - "No primary key constraint found for the following:", - noKey - ) - } - if (invalidCol.length) { - const invalidCols = Object.values(InvalidColumns).join(", ") - error = updateError( - error, - `Cannot use columns ${invalidCols} found in following:`, - invalidCol - ) - } - } - return { tables: connector.tables, error } -} diff --git a/packages/server/src/api/routes/datasource.ts b/packages/server/src/api/routes/datasource.ts index 85929d2180..654fb794e3 100644 --- a/packages/server/src/api/routes/datasource.ts +++ b/packages/server/src/api/routes/datasource.ts @@ -15,6 +15,11 @@ router authorized(permissions.BUILDER), datasourceController.fetch ) + .post( + "/api/datasources/verify", + authorized(permissions.BUILDER), + datasourceController.verify + ) .get( "/api/datasources/:datasourceId", authorized( diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index d54259eab5..36e081d9f6 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -14,6 +14,14 @@ export interface CreateDatasourceRequest { fetchSchema?: boolean } +export interface VerifyDatasourceRequest { + datasource: Datasource +} + +export interface VerifyDatasourceResponse { + connected: boolean +} + export interface UpdateDatasourceRequest extends Datasource { datasource: Datasource }