Merge branch 'feature/datasource-connections' of github.com:Budibase/budibase into feature/datasource-conns

packages/server/src/api/controllers/datasource.ts

@@ -18,11 +18,68 @@ import {
   Row,
   CreateDatasourceResponse,
   UpdateDatasourceResponse,
-  UpdateDatasourceRequest,
   CreateDatasourceRequest,
+  VerifyDatasourceRequest,
+  VerifyDatasourceResponse,
+  IntegrationBase,
+  DatasourcePlus,
 } from "@budibase/types"
 import sdk from "../../sdk"
 
+function getErrorTables(errors: any, errorType: string) {
+  return Object.entries(errors)
+    .filter(entry => entry[1] === errorType)
+    .map(([name]) => name)
+}
+
+function updateError(error: any, newError: any, tables: string[]) {
+  if (!error) {
+    error = ""
+  }
+  if (error.length > 0) {
+    error += "\n"
+  }
+  error += `${newError} ${tables.join(", ")}`
+  return error
+}
+
+async function getConnector(
+  datasource: Datasource
+): Promise<IntegrationBase | DatasourcePlus> {
+  const Connector = await getIntegration(datasource.source)
+  datasource = await sdk.datasources.enrich(datasource)
+  // Connect to the DB and build the schema
+  return new Connector(datasource.config)
+}
+
+async function buildSchemaHelper(datasource: Datasource) {
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  await connector.buildSchema(datasource._id!, datasource.entities!)
+
+  const errors = connector.schemaErrors
+  let error = null
+  if (errors && Object.keys(errors).length > 0) {
+    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
+    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
+    if (noKey.length) {
+      error = updateError(
+        error,
+        "No primary key constraint found for the following:",
+        noKey
+      )
+    }
+    if (invalidCol.length) {
+      const invalidCols = Object.values(InvalidColumns).join(", ")
+      error = updateError(
+        error,
+        `Cannot use columns ${invalidCols} found in following:`,
+        invalidCol
+      )
+    }
+  }
+  return { tables: connector.tables, error }
+}
+
 export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
@@ -66,6 +123,20 @@ export async function fetch(ctx: UserCtx) {
   ctx.body = [bbInternalDb, ...datasources]
 }
 
+export async function verify(
+  ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
+) {
+  const datasource = ctx.request.body.datasource
+  const connector = (await getConnector(datasource)) as IntegrationBase
+  if (!connector.connection) {
+    ctx.throw(400, "Connection information verification not supported")
+  }
+  const connectionInfo = await connector.connection()
+  ctx.body = {
+    connected: connectionInfo.connected,
+  }
+}
+
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
@@ -311,51 +382,3 @@ export async function query(ctx: UserCtx) {
     ctx.throw(400, err)
   }
 }
-
-function getErrorTables(errors: any, errorType: string) {
-  return Object.entries(errors)
-    .filter(entry => entry[1] === errorType)
-    .map(([name]) => name)
-}
-
-function updateError(error: any, newError: any, tables: string[]) {
-  if (!error) {
-    error = ""
-  }
-  if (error.length > 0) {
-    error += "\n"
-  }
-  error += `${newError} ${tables.join(", ")}`
-  return error
-}
-
-async function buildSchemaHelper(datasource: Datasource) {
-  const Connector = await getIntegration(datasource.source)
-  datasource = await sdk.datasources.enrich(datasource)
-  // Connect to the DB and build the schema
-  const connector = new Connector(datasource.config)
-  await connector.buildSchema(datasource._id, datasource.entities)
-
-  const errors = connector.schemaErrors
-  let error = null
-  if (errors && Object.keys(errors).length > 0) {
-    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
-    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
-    if (noKey.length) {
-      error = updateError(
-        error,
-        "No primary key constraint found for the following:",
-        noKey
-      )
-    }
-    if (invalidCol.length) {
-      const invalidCols = Object.values(InvalidColumns).join(", ")
-      error = updateError(
-        error,
-        `Cannot use columns ${invalidCols} found in following:`,
-        invalidCol
-      )
-    }
-  }
-  return { tables: connector.tables, error }
-}

packages/server/src/api/routes/datasource.ts

@@ -15,6 +15,11 @@ router
     authorized(permissions.BUILDER),
     datasourceController.fetch
   )
+  .post(
+    "/api/datasources/verify",
+    authorized(permissions.BUILDER),
+    datasourceController.verify
+  )
   .get(
     "/api/datasources/:datasourceId",
     authorized(

packages/server/src/integrations/airtable.ts

@@ -1,8 +1,9 @@
 import {
-  Integration,
+  DatasourceFeature,
   DatasourceFieldType,
-  QueryType,
+  Integration,
   IntegrationBase,
+  QueryType,
 } from "@budibase/types"
 
 const Airtable = require("airtable")
@@ -18,6 +19,7 @@ const SCHEMA: Integration = {
     "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.",
   friendlyName: "Airtable",
   type: "Spreadsheet",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     apiKey: {
       type: DatasourceFieldType.PASSWORD,

packages/server/src/integrations/arangodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
 } from "@budibase/types"
 
 const { Database, aql } = require("arangojs")
@@ -21,6 +22,7 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/couchdb.ts

@@ -1,4 +1,5 @@
 import {
+  DatasourceFeature,
   DatasourceFieldType,
   Document,
   Integration,
@@ -18,6 +19,7 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/dynamodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
 } from "@budibase/types"
 
 import AWS from "aws-sdk"
@@ -22,6 +23,7 @@ const SCHEMA: Integration = {
     "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.",
   friendlyName: "DynamoDB",
   type: "Non-relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     region: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/elasticsearch.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
 } from "@budibase/types"
 
 import { Client, ClientOptions } from "@elastic/elasticsearch"
@@ -20,6 +21,7 @@ const SCHEMA: Integration = {
     "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
   friendlyName: "ElasticSearch",
   type: "Non-relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/firebase.ts

@@ -3,6 +3,7 @@ import {
   Integration,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
 } from "@budibase/types"
 import { Firestore, WhereFilterOp } from "@google-cloud/firestore"
 
@@ -18,6 +19,7 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     email: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/googlesheets.ts

@@ -1,4 +1,5 @@
 import {
+  DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
   FieldType,
@@ -64,6 +65,7 @@ const SCHEMA: Integration = {
     "Create and collaborate on online spreadsheets in real-time and from any device. ",
   friendlyName: "Google Sheets",
   type: "Spreadsheet",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     spreadsheetId: {
       display: "Google Sheet URL",

packages/server/src/integrations/microsoftSqlServer.ts

@@ -8,6 +8,7 @@ import {
   QueryType,
   SqlQuery,
   DatasourcePlus,
+  DatasourceFeature,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -39,6 +40,7 @@ const SCHEMA: Integration = {
     "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
   friendlyName: "MS SQL Server",
   type: "Relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     user: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/mongodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
 } from "@budibase/types"
 import {
   MongoClient,
@@ -38,6 +39,7 @@ const getSchema = () => {
     type: "Non-relational",
     description:
       "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
+    features: [DatasourceFeature.CONNECTION_CHECKING],
     datasource: {
       connectionString: {
         type: DatasourceFieldType.STRING,

packages/server/src/integrations/mysql.ts

@@ -7,6 +7,7 @@ import {
   Table,
   TableSchema,
   DatasourcePlus,
+  DatasourceFeature,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -41,6 +42,7 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/oracle.ts

@@ -7,6 +7,7 @@ import {
   SqlQuery,
   Table,
   DatasourcePlus,
+  DatasourceFeature,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -53,6 +54,7 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "Oracle Database is an object-relational database management system developed by Oracle Corporation",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/postgres.ts

@@ -6,6 +6,7 @@ import {
   SqlQuery,
   Table,
   DatasourcePlus,
+  DatasourceFeature,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -50,6 +51,7 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/redis.ts

@@ -1,4 +1,9 @@
-import { DatasourceFieldType, Integration, QueryType } from "@budibase/types"
+import {
+  DatasourceFeature,
+  DatasourceFieldType,
+  Integration,
+  QueryType,
+} from "@budibase/types"
 import Redis from "ioredis"
 
 interface RedisConfig {
@@ -11,9 +16,11 @@ interface RedisConfig {
 
 const SCHEMA: Integration = {
   docs: "https://redis.io/docs/",
-  description: "",
+  description:
+    "Redis is a caching tool, providing powerful key-value store capabilities.",
   friendlyName: "Redis",
   type: "Non-relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: "string",

packages/server/src/integrations/s3.ts

@@ -3,6 +3,7 @@ import {
   QueryType,
   IntegrationBase,
   DatasourceFieldType,
+  DatasourceFeature,
 } from "@budibase/types"
 
 const AWS = require("aws-sdk")
@@ -22,6 +23,7 @@ const SCHEMA: Integration = {
     "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
   friendlyName: "Amazon S3",
   type: "Object store",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     region: {
       type: "string",

packages/server/src/integrations/snowflake.ts

@@ -1,4 +1,9 @@
-import { Integration, QueryType, SqlQuery } from "@budibase/types"
+import {
+  DatasourceFeature,
+  Integration,
+  QueryType,
+  SqlQuery,
+} from "@budibase/types"
 import { Snowflake } from "snowflake-promise"
 
 interface SnowflakeConfig {
@@ -16,6 +21,7 @@ const SCHEMA: Integration = {
     "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.",
   friendlyName: "Snowflake",
   type: "Relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     account: {
       type: "string",

packages/types/src/api/web/app/datasource.ts

@@ -14,6 +14,14 @@ export interface CreateDatasourceRequest {
   fetchSchema?: boolean
 }
 
+export interface VerifyDatasourceRequest {
+  datasource: Datasource
+}
+
+export interface VerifyDatasourceResponse {
+  connected: boolean
+}
+
 export interface UpdateDatasourceRequest extends Datasource {
   datasource: Datasource
 }

packages/types/src/sdk/datasources.ts

@@ -73,6 +73,10 @@ export enum FilterType {
   ONE_OF = "oneOf",
 }
 
+export enum DatasourceFeature {
+  CONNECTION_CHECKING = "connection",
+}
+
 export interface StepDefinition {
   key: string
   template: string
@@ -111,6 +115,7 @@ export interface Integration {
   docs: string
   plus?: boolean
   auth?: { type: string }
+  features?: DatasourceFeature[]
   relationships?: boolean
   description: string
   friendlyName: string