From ce6c5bfa683ad1124ed513b7e4dca6c7982e812c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 May 2023 15:47:55 +0100 Subject: [PATCH 01/76] Base connection work - extending the base integration to include the option of a connection check function. --- packages/server/src/integrations/airtable.ts | 10 ++++++++-- packages/server/src/integrations/arangodb.ts | 6 ++++++ packages/server/src/integrations/couchdb.ts | 6 ++++++ packages/server/src/integrations/dynamodb.ts | 2 ++ packages/server/src/integrations/elasticsearch.ts | 2 ++ packages/server/src/integrations/firebase.ts | 2 ++ packages/server/src/integrations/googlesheets.ts | 2 ++ .../server/src/integrations/microsoftSqlServer.ts | 2 ++ packages/server/src/integrations/mongodb.ts | 2 ++ packages/server/src/integrations/mysql.ts | 2 ++ packages/server/src/integrations/oracle.ts | 2 ++ packages/server/src/integrations/postgres.ts | 2 ++ packages/server/src/integrations/redis.ts | 11 +++++++++-- packages/server/src/integrations/s3.ts | 2 ++ packages/server/src/integrations/snowflake.ts | 8 +++++++- packages/types/src/sdk/datasources.ts | 10 ++++++++++ 16 files changed, 66 insertions(+), 5 deletions(-) diff --git a/packages/server/src/integrations/airtable.ts b/packages/server/src/integrations/airtable.ts index 1f56f0619b..58232ad56c 100644 --- a/packages/server/src/integrations/airtable.ts +++ b/packages/server/src/integrations/airtable.ts @@ -1,8 +1,9 @@ import { - Integration, + DatasourceFeature, DatasourceFieldType, - QueryType, + Integration, IntegrationBase, + QueryType, } from "@budibase/types" const Airtable = require("airtable") @@ -18,6 +19,7 @@ const SCHEMA: Integration = { "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.", friendlyName: "Airtable", type: "Spreadsheet", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { apiKey: { type: DatasourceFieldType.PASSWORD, @@ -88,6 +90,10 @@ class AirtableIntegration implements IntegrationBase { this.client = new Airtable(config).base(config.base) } + async connection() { + return { connected: true } + } + async create(query: { table: any; json: any }) { const { table, json } = query diff --git a/packages/server/src/integrations/arangodb.ts b/packages/server/src/integrations/arangodb.ts index e28940f36e..3a61193577 100644 --- a/packages/server/src/integrations/arangodb.ts +++ b/packages/server/src/integrations/arangodb.ts @@ -3,6 +3,7 @@ import { DatasourceFieldType, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" const { Database, aql } = require("arangojs") @@ -21,6 +22,7 @@ const SCHEMA: Integration = { type: "Non-relational", description: "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { url: { type: DatasourceFieldType.STRING, @@ -74,6 +76,10 @@ class ArangoDBIntegration implements IntegrationBase { this.client = new Database(newConfig) } + async connection() { + return { connected: true } + } + async read(query: { sql: any }) { try { const result = await this.client.query(query.sql) diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index 257b84ca13..dfb0daa2e8 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -1,4 +1,5 @@ import { + DatasourceFeature, DatasourceFieldType, Document, Integration, @@ -18,6 +19,7 @@ const SCHEMA: Integration = { type: "Non-relational", description: "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { url: { type: DatasourceFieldType.STRING, @@ -69,6 +71,10 @@ class CouchDBIntegration implements IntegrationBase { this.client = dbCore.DatabaseWithConnection(config.database, config.url) } + async connection() { + return { connected: true } + } + async query( command: string, errorMsg: string, diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index 28c1c7b52b..c682f355f9 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -3,6 +3,7 @@ import { DatasourceFieldType, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" import AWS from "aws-sdk" @@ -22,6 +23,7 @@ const SCHEMA: Integration = { "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.", friendlyName: "DynamoDB", type: "Non-relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { region: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index aeba628d30..475ddc0bd1 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -3,6 +3,7 @@ import { DatasourceFieldType, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" import { Client, ClientOptions } from "@elastic/elasticsearch" @@ -20,6 +21,7 @@ const SCHEMA: Integration = { "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.", friendlyName: "ElasticSearch", type: "Non-relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { url: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/firebase.ts b/packages/server/src/integrations/firebase.ts index a82b3be782..0646c98eba 100644 --- a/packages/server/src/integrations/firebase.ts +++ b/packages/server/src/integrations/firebase.ts @@ -3,6 +3,7 @@ import { Integration, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" import { Firestore, WhereFilterOp } from "@google-cloud/firestore" @@ -18,6 +19,7 @@ const SCHEMA: Integration = { type: "Non-relational", description: "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { email: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 476a6511e9..8884c8de13 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -1,4 +1,5 @@ import { + DatasourceFeature, DatasourceFieldType, DatasourcePlus, FieldType, @@ -64,6 +65,7 @@ const SCHEMA: Integration = { "Create and collaborate on online spreadsheets in real-time and from any device. ", friendlyName: "Google Sheets", type: "Spreadsheet", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { spreadsheetId: { display: "Google Sheet URL", diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index eb87c1ccf1..b6f360ce32 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -8,6 +8,7 @@ import { QueryType, SqlQuery, DatasourcePlus, + DatasourceFeature, } from "@budibase/types" import { getSqlQuery, @@ -39,6 +40,7 @@ const SCHEMA: Integration = { "Microsoft SQL Server is a relational database management system developed by Microsoft. ", friendlyName: "MS SQL Server", type: "Relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { user: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 38b3891fe4..20ba2acada 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -3,6 +3,7 @@ import { DatasourceFieldType, QueryType, IntegrationBase, + DatasourceFeature, } from "@budibase/types" import { MongoClient, @@ -38,6 +39,7 @@ const getSchema = () => { type: "Non-relational", description: "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { connectionString: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 8d984ed402..6083223042 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -7,6 +7,7 @@ import { Table, TableSchema, DatasourcePlus, + DatasourceFeature, } from "@budibase/types" import { getSqlQuery, @@ -41,6 +42,7 @@ const SCHEMA: Integration = { type: "Relational", description: "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 65e0829905..bbe2189bdc 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -7,6 +7,7 @@ import { SqlQuery, Table, DatasourcePlus, + DatasourceFeature, } from "@budibase/types" import { buildExternalTableId, @@ -53,6 +54,7 @@ const SCHEMA: Integration = { type: "Relational", description: "Oracle Database is an object-relational database management system developed by Oracle Corporation", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index c981c3acc5..a968b71064 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -6,6 +6,7 @@ import { SqlQuery, Table, DatasourcePlus, + DatasourceFeature, } from "@budibase/types" import { getSqlQuery, @@ -50,6 +51,7 @@ const SCHEMA: Integration = { type: "Relational", description: "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index 73ef2bb55c..f596b6eb0d 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -1,4 +1,9 @@ -import { DatasourceFieldType, Integration, QueryType } from "@budibase/types" +import { + DatasourceFeature, + DatasourceFieldType, + Integration, + QueryType, +} from "@budibase/types" import Redis from "ioredis" interface RedisConfig { @@ -11,9 +16,11 @@ interface RedisConfig { const SCHEMA: Integration = { docs: "https://redis.io/docs/", - description: "", + description: + "Redis is a caching tool, providing powerful key-value store capabilities.", friendlyName: "Redis", type: "Non-relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { host: { type: "string", diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index ad3bb09109..cc368760f8 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -3,6 +3,7 @@ import { QueryType, IntegrationBase, DatasourceFieldType, + DatasourceFeature, } from "@budibase/types" const AWS = require("aws-sdk") @@ -22,6 +23,7 @@ const SCHEMA: Integration = { "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.", friendlyName: "Amazon S3", type: "Object store", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { region: { type: "string", diff --git a/packages/server/src/integrations/snowflake.ts b/packages/server/src/integrations/snowflake.ts index db702520f9..877405d447 100644 --- a/packages/server/src/integrations/snowflake.ts +++ b/packages/server/src/integrations/snowflake.ts @@ -1,4 +1,9 @@ -import { Integration, QueryType, SqlQuery } from "@budibase/types" +import { + DatasourceFeature, + Integration, + QueryType, + SqlQuery, +} from "@budibase/types" import { Snowflake } from "snowflake-promise" interface SnowflakeConfig { @@ -16,6 +21,7 @@ const SCHEMA: Integration = { "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.", friendlyName: "Snowflake", type: "Relational", + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { account: { type: "string", diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 605b431d9e..ccd6f7b9c6 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -74,6 +74,10 @@ export enum FilterType { ONE_OF = "oneOf", } +export enum DatasourceFeature { + CONNECTION_CHECKING = "connection", +} + export interface StepDefinition { key: string template: string @@ -112,6 +116,7 @@ export interface Integration { docs: string plus?: boolean auth?: { type: string } + features?: DatasourceFeature[] relationships?: boolean description: string friendlyName: string @@ -124,11 +129,16 @@ export interface Integration { extra?: ExtraQueryConfig } +export interface ConnectionInformation { + connected: boolean +} + export interface IntegrationBase { create?(query: any): Promise read?(query: any): Promise update?(query: any): Promise delete?(query: any): Promise + connection?(): Promise } export interface DatasourcePlus extends IntegrationBase { From 57a633b926f5fbcedc53e549bd63314a1e8d6870 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 May 2023 17:40:39 +0100 Subject: [PATCH 02/76] Implementing main body of connection verification endpoint. --- .../server/src/api/controllers/datasource.ts | 121 +++++++++++------- packages/server/src/api/routes/datasource.ts | 5 + packages/types/src/api/web/app/datasource.ts | 8 ++ 3 files changed, 85 insertions(+), 49 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index b61b168980..e6866d528c 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -18,11 +18,68 @@ import { Row, CreateDatasourceResponse, UpdateDatasourceResponse, - UpdateDatasourceRequest, CreateDatasourceRequest, + VerifyDatasourceRequest, + VerifyDatasourceResponse, + IntegrationBase, + DatasourcePlus, } from "@budibase/types" import sdk from "../../sdk" +function getErrorTables(errors: any, errorType: string) { + return Object.entries(errors) + .filter(entry => entry[1] === errorType) + .map(([name]) => name) +} + +function updateError(error: any, newError: any, tables: string[]) { + if (!error) { + error = "" + } + if (error.length > 0) { + error += "\n" + } + error += `${newError} ${tables.join(", ")}` + return error +} + +async function getConnector( + datasource: Datasource +): Promise { + const Connector = await getIntegration(datasource.source) + datasource = await sdk.datasources.enrich(datasource) + // Connect to the DB and build the schema + return new Connector(datasource.config) +} + +async function buildSchemaHelper(datasource: Datasource) { + const connector = (await getConnector(datasource)) as DatasourcePlus + await connector.buildSchema(datasource._id!, datasource.entities!) + + const errors = connector.schemaErrors + let error = null + if (errors && Object.keys(errors).length > 0) { + const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY) + const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN) + if (noKey.length) { + error = updateError( + error, + "No primary key constraint found for the following:", + noKey + ) + } + if (invalidCol.length) { + const invalidCols = Object.values(InvalidColumns).join(", ") + error = updateError( + error, + `Cannot use columns ${invalidCols} found in following:`, + invalidCol + ) + } + } + return { tables: connector.tables, error } +} + export async function fetch(ctx: UserCtx) { // Get internal tables const db = context.getAppDB() @@ -66,6 +123,20 @@ export async function fetch(ctx: UserCtx) { ctx.body = [bbInternalDb, ...datasources] } +export async function verify( + ctx: UserCtx +) { + const datasource = ctx.request.body.datasource + const connector = (await getConnector(datasource)) as IntegrationBase + if (!connector.connection) { + ctx.throw(400, "Connection information verification not supported") + } + const connectionInfo = await connector.connection() + ctx.body = { + connected: connectionInfo.connected, + } +} + export async function buildSchemaFromDb(ctx: UserCtx) { const db = context.getAppDB() const datasource = await sdk.datasources.get(ctx.params.datasourceId) @@ -311,51 +382,3 @@ export async function query(ctx: UserCtx) { ctx.throw(400, err) } } - -function getErrorTables(errors: any, errorType: string) { - return Object.entries(errors) - .filter(entry => entry[1] === errorType) - .map(([name]) => name) -} - -function updateError(error: any, newError: any, tables: string[]) { - if (!error) { - error = "" - } - if (error.length > 0) { - error += "\n" - } - error += `${newError} ${tables.join(", ")}` - return error -} - -async function buildSchemaHelper(datasource: Datasource) { - const Connector = await getIntegration(datasource.source) - datasource = await sdk.datasources.enrich(datasource) - // Connect to the DB and build the schema - const connector = new Connector(datasource.config) - await connector.buildSchema(datasource._id, datasource.entities) - - const errors = connector.schemaErrors - let error = null - if (errors && Object.keys(errors).length > 0) { - const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY) - const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN) - if (noKey.length) { - error = updateError( - error, - "No primary key constraint found for the following:", - noKey - ) - } - if (invalidCol.length) { - const invalidCols = Object.values(InvalidColumns).join(", ") - error = updateError( - error, - `Cannot use columns ${invalidCols} found in following:`, - invalidCol - ) - } - } - return { tables: connector.tables, error } -} diff --git a/packages/server/src/api/routes/datasource.ts b/packages/server/src/api/routes/datasource.ts index 85929d2180..654fb794e3 100644 --- a/packages/server/src/api/routes/datasource.ts +++ b/packages/server/src/api/routes/datasource.ts @@ -15,6 +15,11 @@ router authorized(permissions.BUILDER), datasourceController.fetch ) + .post( + "/api/datasources/verify", + authorized(permissions.BUILDER), + datasourceController.verify + ) .get( "/api/datasources/:datasourceId", authorized( diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index d54259eab5..36e081d9f6 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -14,6 +14,14 @@ export interface CreateDatasourceRequest { fetchSchema?: boolean } +export interface VerifyDatasourceRequest { + datasource: Datasource +} + +export interface VerifyDatasourceResponse { + connected: boolean +} + export interface UpdateDatasourceRequest extends Datasource { datasource: Datasource } From 20aa0f50ab86c54ff6e2832df8257284d587ba3a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 10 May 2023 18:22:23 +0200 Subject: [PATCH 03/76] Clean code --- packages/server/src/integration-test/postgres.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 78075b4e54..79f6db5cd1 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -19,7 +19,6 @@ import _ from "lodash" import { generator } from "@budibase/backend-core/tests" import { utils } from "@budibase/backend-core" import { GenericContainer } from "testcontainers" -import { generateRowIdField } from "../integrations/utils" const config = setup.getConfig()! From 640911f1dd5a95f6c3453190f6bf3b51ed4192ef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 10 May 2023 18:22:39 +0200 Subject: [PATCH 04/76] Update pg deps --- packages/server/package.json | 3 ++- yarn.lock | 31 ++++++++++++++++++++----------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index f0ece87bed..c65469f5cb 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -99,7 +99,7 @@ "mysql2": "2.3.3", "node-fetch": "2.6.7", "open": "8.4.0", - "pg": "8.5.1", + "pg": "8.10.0", "posthog-node": "1.3.0", "pouchdb": "7.3.0", "pouchdb-adapter-memory": "7.2.2", @@ -141,6 +141,7 @@ "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", "@types/oracledb": "5.2.2", + "@types/pg": "8.6.6", "@types/pouchdb": "6.4.0", "@types/redis": "4.0.11", "@types/server-destroy": "1.0.1", diff --git a/yarn.lock b/yarn.lock index 04d9a7d140..ab59d36d12 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5211,6 +5211,15 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/pg@8.6.6": + version "8.6.6" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.6.6.tgz#21cdf873a3e345a6e78f394677e3b3b1b543cb80" + integrity sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw== + dependencies: + "@types/node" "*" + pg-protocol "*" + pg-types "^2.2.0" + "@types/pouchdb-adapter-cordova-sqlite@*": version "1.0.1" resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-cordova-sqlite/-/pouchdb-adapter-cordova-sqlite-1.0.1.tgz#49e5ee6df7cc0c23196fcb340f43a560e74eb1d6" @@ -19085,7 +19094,7 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== -pg-connection-string@2.5.0, pg-connection-string@^2.4.0: +pg-connection-string@2.5.0, pg-connection-string@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== @@ -19095,17 +19104,17 @@ pg-int8@1.0.1: resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== -pg-pool@^3.2.2: +pg-pool@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== -pg-protocol@^1.4.0: +pg-protocol@*, pg-protocol@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833" integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q== -pg-types@^2.1.0: +pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== @@ -19116,16 +19125,16 @@ pg-types@^2.1.0: postgres-date "~1.0.4" postgres-interval "^1.1.0" -pg@8.5.1: - version "8.5.1" - resolved "https://registry.yarnpkg.com/pg/-/pg-8.5.1.tgz#34dcb15f6db4a29c702bf5031ef2e1e25a06a120" - integrity sha512-9wm3yX9lCfjvA98ybCyw2pADUivyNWT/yIP4ZcDVpMN0og70BUWYEGXPCTAQdGTAqnytfRADb7NERrY1qxhIqw== +pg@8.10.0: + version "8.10.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.10.0.tgz#5b8379c9b4a36451d110fc8cd98fc325fe62ad24" + integrity sha512-ke7o7qSTMb47iwzOSaZMfeR7xToFdkE71ifIipOAAaLIM0DYzfOAXlgFFmYUIE2BcJtvnVlGCID84ZzCegE8CQ== dependencies: buffer-writer "2.0.0" packet-reader "1.0.0" - pg-connection-string "^2.4.0" - pg-pool "^3.2.2" - pg-protocol "^1.4.0" + pg-connection-string "^2.5.0" + pg-pool "^3.6.0" + pg-protocol "^1.6.0" pg-types "^2.1.0" pgpass "1.x" From 239a51a2c08f24525b1ae710c2aafcbb19b00db6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 10 May 2023 18:23:42 +0200 Subject: [PATCH 05/76] Typings --- packages/server/src/integrations/postgres.ts | 6 +++--- .../server/src/migrations/functions/backfill/app/queries.ts | 2 +- packages/types/src/sdk/datasources.ts | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index c981c3acc5..4fb786a409 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -18,7 +18,7 @@ import Sql from "./base/sql" import { PostgresColumn } from "./base/types" import { escapeDangerousCharacters } from "../utilities" -const { Client, types } = require("pg") +import { Client, types } from "pg" // Return "date" and "timestamp" types as plain strings. // This lets us reference the original stored timezone. @@ -114,7 +114,7 @@ const SCHEMA: Integration = { } class PostgresIntegration extends Sql implements DatasourcePlus { - private readonly client: any + private readonly client: Client private readonly config: PostgresConfig private index: number = 1 private open: boolean @@ -163,7 +163,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { if (!this.config.schema) { this.config.schema = "public" } - this.client.query(`SET search_path TO ${this.config.schema}`) + await this.client.query(`SET search_path TO ${this.config.schema}`) this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'` this.open = true } diff --git a/packages/server/src/migrations/functions/backfill/app/queries.ts b/packages/server/src/migrations/functions/backfill/app/queries.ts index e66c7af841..e028721bce 100644 --- a/packages/server/src/migrations/functions/backfill/app/queries.ts +++ b/packages/server/src/migrations/functions/backfill/app/queries.ts @@ -33,7 +33,7 @@ export const backfill = async (appDb: any, timestamp: string | number) => { datasource = { type: "unknown", _id: query.datasourceId, - source: SourceName.UNKNOWN, + source: "unknown" as SourceName, } } else { throw e diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 605b431d9e..938f4c9340 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -55,7 +55,6 @@ export enum SourceName { FIRESTORE = "FIRESTORE", REDIS = "REDIS", SNOWFLAKE = "SNOWFLAKE", - UNKNOWN = "unknown", } export enum IncludeRelationship { From f022a43065b5fd95bf486dc17be6dfdd0b76f522 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 10 May 2023 18:23:52 +0200 Subject: [PATCH 06/76] Test pg connection --- packages/server/src/integrations/index.ts | 23 ++++++-- packages/server/src/integrations/postgres.ts | 13 +++++ .../sdk/tests/datasources/validators.spec.ts | 58 +++++++++++++++++++ 3 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 packages/server/src/sdk/tests/datasources/validators.spec.ts diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index f3285e441f..82432dfa2c 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -20,7 +20,7 @@ import env from "../environment" import { cloneDeep } from "lodash" import sdk from "../sdk" -const DEFINITIONS: { [key: string]: Integration } = { +const DEFINITIONS: Record = { [SourceName.POSTGRES]: postgres.schema, [SourceName.DYNAMODB]: dynamodb.schema, [SourceName.MONGODB]: mongodb.schema, @@ -38,7 +38,7 @@ const DEFINITIONS: { [key: string]: Integration } = { [SourceName.SNOWFLAKE]: snowflake.schema, } -const INTEGRATIONS: { [key: string]: any } = { +const INTEGRATIONS: Record = { [SourceName.POSTGRES]: postgres.integration, [SourceName.DYNAMODB]: dynamodb.integration, [SourceName.MONGODB]: mongodb.integration, @@ -55,6 +55,7 @@ const INTEGRATIONS: { [key: string]: any } = { [SourceName.REDIS]: redis.integration, [SourceName.FIRESTORE]: firebase.integration, [SourceName.SNOWFLAKE]: snowflake.integration, + [SourceName.ORACLE]: undefined, } // optionally add oracle integration if the oracle binary can be installed @@ -69,8 +70,9 @@ if ( export async function getDefinition(source: SourceName): Promise { // check if its integrated, faster - if (DEFINITIONS[source]) { - return DEFINITIONS[source] + const definition = DEFINITIONS[source] + if (definition) { + return definition } const allDefinitions = await getDefinitions() return allDefinitions[source] @@ -98,7 +100,7 @@ export async function getDefinitions() { } } -export async function getIntegration(integration: string) { +export async function getIntegration(integration: SourceName) { if (INTEGRATIONS[integration]) { return INTEGRATIONS[integration] } @@ -119,7 +121,18 @@ export async function getIntegration(integration: string) { throw new Error("No datasource implementation found.") } +const VALIDATORS: Partial< + Record Promise> +> = { + [SourceName.POSTGRES]: postgres.validateConnection, +} + +function getValidators(integration: SourceName) { + return VALIDATORS[integration] +} + export default { getDefinitions, getIntegration, + getValidators, } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 4fb786a409..d84ea273c4 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -330,7 +330,20 @@ class PostgresIntegration extends Sql implements DatasourcePlus { } } +async function validateConnection(config: PostgresConfig) { + const integration = new PostgresIntegration(config) + try { + await integration.openConnection() + return true + } catch { + return false + } finally { + await integration.closeConnection() + } +} + export default { schema: SCHEMA, integration: PostgresIntegration, + validateConnection, } diff --git a/packages/server/src/sdk/tests/datasources/validators.spec.ts b/packages/server/src/sdk/tests/datasources/validators.spec.ts new file mode 100644 index 0000000000..700953feea --- /dev/null +++ b/packages/server/src/sdk/tests/datasources/validators.spec.ts @@ -0,0 +1,58 @@ +import { SourceName } from "@budibase/types" +import integrations from "../../../integrations" +import { GenericContainer } from "testcontainers" + +jest.unmock("pg") + +describe("datasource validators", () => { + describe("postgres", () => { + const validator = integrations.getValidators(SourceName.POSTGRES)! + + let host: string + let port: number + + beforeAll(async () => { + const container = await new GenericContainer("postgres") + .withExposedPorts(5432) + .withEnv("POSTGRES_PASSWORD", "password") + .start() + + host = container.getContainerIpAddress() + port = container.getMappedPort(5432) + }) + + it("test valid connection string", async () => { + const result = await validator({ + host, + port, + database: "postgres", + user: "postgres", + password: "password", + schema: "public", + ssl: false, + rejectUnauthorized: false, + ca: false, + }) + expect(result).toBeTruthy() + }) + + it("test invalid connection string", async () => { + try { + const result = await validator({ + host, + port, + database: "postgres", + user: "wrong", + password: "password", + schema: "public", + ssl: false, + rejectUnauthorized: false, + ca: false, + }) + expect(result).toBeFalsy() + } catch (e) { + console.error(e) + } + }) + }) +}) From 25233c5c9b3579f8a63cb66800562fd5a7cae58b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 10 May 2023 18:27:40 +0200 Subject: [PATCH 07/76] Add message response --- packages/server/src/integrations/index.ts | 2 +- packages/server/src/integrations/postgres.ts | 4 +-- .../sdk/tests/datasources/validators.spec.ts | 30 +++++++++---------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 82432dfa2c..5a2271de25 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -122,7 +122,7 @@ export async function getIntegration(integration: SourceName) { } const VALIDATORS: Partial< - Record Promise> + Record Promise> > = { [SourceName.POSTGRES]: postgres.validateConnection, } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index d84ea273c4..c452b8afc7 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -335,8 +335,8 @@ async function validateConnection(config: PostgresConfig) { try { await integration.openConnection() return true - } catch { - return false + } catch (e: any) { + return { error: e.message as string } } finally { await integration.closeConnection() } diff --git a/packages/server/src/sdk/tests/datasources/validators.spec.ts b/packages/server/src/sdk/tests/datasources/validators.spec.ts index 700953feea..677ff8b59c 100644 --- a/packages/server/src/sdk/tests/datasources/validators.spec.ts +++ b/packages/server/src/sdk/tests/datasources/validators.spec.ts @@ -37,22 +37,20 @@ describe("datasource validators", () => { }) it("test invalid connection string", async () => { - try { - const result = await validator({ - host, - port, - database: "postgres", - user: "wrong", - password: "password", - schema: "public", - ssl: false, - rejectUnauthorized: false, - ca: false, - }) - expect(result).toBeFalsy() - } catch (e) { - console.error(e) - } + const result = await validator({ + host, + port, + database: "postgres", + user: "wrong", + password: "password", + schema: "public", + ssl: false, + rejectUnauthorized: false, + ca: false, + }) + expect(result).toEqual({ + error: 'password authentication failed for user "wrong"', + }) }) }) }) From b022dcba77fbc56ee017fa36d54e98b505b50de0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 May 2023 10:58:50 +0200 Subject: [PATCH 08/76] Types --- packages/server/src/integrations/index.ts | 6 ++---- packages/server/src/integrations/postgres.ts | 2 +- .../server/src/sdk/tests/datasources/validators.spec.ts | 4 +--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 5a2271de25..076c9fdcb5 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -121,9 +121,7 @@ export async function getIntegration(integration: SourceName) { throw new Error("No datasource implementation found.") } -const VALIDATORS: Partial< - Record Promise> -> = { +const VALIDATORS = { [SourceName.POSTGRES]: postgres.validateConnection, } @@ -134,5 +132,5 @@ function getValidators(integration: SourceName) { export default { getDefinitions, getIntegration, - getValidators, + getValidator: VALIDATORS, } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index c452b8afc7..da678854b4 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -31,7 +31,7 @@ if (types) { const JSON_REGEX = /'{.*}'::json/s -interface PostgresConfig { +export interface PostgresConfig { host: string port: number database: string diff --git a/packages/server/src/sdk/tests/datasources/validators.spec.ts b/packages/server/src/sdk/tests/datasources/validators.spec.ts index 677ff8b59c..9664c6c8fc 100644 --- a/packages/server/src/sdk/tests/datasources/validators.spec.ts +++ b/packages/server/src/sdk/tests/datasources/validators.spec.ts @@ -6,7 +6,7 @@ jest.unmock("pg") describe("datasource validators", () => { describe("postgres", () => { - const validator = integrations.getValidators(SourceName.POSTGRES)! + const validator = integrations.getValidator[SourceName.POSTGRES]! let host: string let port: number @@ -31,7 +31,6 @@ describe("datasource validators", () => { schema: "public", ssl: false, rejectUnauthorized: false, - ca: false, }) expect(result).toBeTruthy() }) @@ -46,7 +45,6 @@ describe("datasource validators", () => { schema: "public", ssl: false, rejectUnauthorized: false, - ca: false, }) expect(result).toEqual({ error: 'password authentication failed for user "wrong"', From f5fb4f885049b539b50a57ce957110a6a7db7905 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 May 2023 20:43:01 +0200 Subject: [PATCH 09/76] Clean --- packages/server/src/integrations/index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 076c9fdcb5..b78955f38d 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -125,10 +125,6 @@ const VALIDATORS = { [SourceName.POSTGRES]: postgres.validateConnection, } -function getValidators(integration: SourceName) { - return VALIDATORS[integration] -} - export default { getDefinitions, getIntegration, From ffef2499cc1ee45d03138b7ada91304ca38d6da2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 12:19:05 +0200 Subject: [PATCH 10/76] Implement the check as part of the integration --- packages/server/src/integrations/index.ts | 14 +++++------ packages/server/src/integrations/postgres.ts | 24 +++++++++---------- .../sdk/tests/datasources/validators.spec.ts | 11 ++++----- packages/types/src/sdk/datasources.ts | 6 +++++ 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index b78955f38d..90dd7cfcd6 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -20,7 +20,7 @@ import env from "../environment" import { cloneDeep } from "lodash" import sdk from "../sdk" -const DEFINITIONS: Record = { +const DEFINITIONS: Record = { [SourceName.POSTGRES]: postgres.schema, [SourceName.DYNAMODB]: dynamodb.schema, [SourceName.MONGODB]: mongodb.schema, @@ -36,6 +36,7 @@ const DEFINITIONS: Record = { [SourceName.GOOGLE_SHEETS]: googlesheets.schema, [SourceName.REDIS]: redis.schema, [SourceName.SNOWFLAKE]: snowflake.schema, + [SourceName.ORACLE]: undefined, } const INTEGRATIONS: Record = { @@ -68,7 +69,9 @@ if ( INTEGRATIONS[SourceName.ORACLE] = oracle.integration } -export async function getDefinition(source: SourceName): Promise { +export async function getDefinition( + source: SourceName +): Promise { // check if its integrated, faster const definition = DEFINITIONS[source] if (definition) { @@ -109,7 +112,7 @@ export async function getIntegration(integration: SourceName) { for (let plugin of plugins) { if (plugin.name === integration) { // need to use commonJS require due to its dynamic runtime nature - const retrieved: any = await getDatasourcePlugin(plugin) + const retrieved = await getDatasourcePlugin(plugin) if (retrieved.integration) { return retrieved.integration } else { @@ -121,12 +124,7 @@ export async function getIntegration(integration: SourceName) { throw new Error("No datasource implementation found.") } -const VALIDATORS = { - [SourceName.POSTGRES]: postgres.validateConnection, -} - export default { getDefinitions, getIntegration, - getValidator: VALIDATORS, } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index da678854b4..f6a8f22441 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -150,6 +150,17 @@ class PostgresIntegration extends Sql implements DatasourcePlus { this.open = false } + async testConnection() { + try { + await this.openConnection() + return true + } catch (e: any) { + return { error: e.message as string } + } finally { + await this.closeConnection() + } + } + getBindingIdentifier(): string { return `$${this.index++}` } @@ -330,20 +341,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { } } -async function validateConnection(config: PostgresConfig) { - const integration = new PostgresIntegration(config) - try { - await integration.openConnection() - return true - } catch (e: any) { - return { error: e.message as string } - } finally { - await integration.closeConnection() - } -} - export default { schema: SCHEMA, integration: PostgresIntegration, - validateConnection, } diff --git a/packages/server/src/sdk/tests/datasources/validators.spec.ts b/packages/server/src/sdk/tests/datasources/validators.spec.ts index 9664c6c8fc..b1b4d83fc8 100644 --- a/packages/server/src/sdk/tests/datasources/validators.spec.ts +++ b/packages/server/src/sdk/tests/datasources/validators.spec.ts @@ -1,13 +1,10 @@ -import { SourceName } from "@budibase/types" -import integrations from "../../../integrations" +import postgres from "../../../integrations/postgres" import { GenericContainer } from "testcontainers" jest.unmock("pg") describe("datasource validators", () => { describe("postgres", () => { - const validator = integrations.getValidator[SourceName.POSTGRES]! - let host: string let port: number @@ -22,7 +19,7 @@ describe("datasource validators", () => { }) it("test valid connection string", async () => { - const result = await validator({ + const integration = new postgres.integration({ host, port, database: "postgres", @@ -32,11 +29,12 @@ describe("datasource validators", () => { ssl: false, rejectUnauthorized: false, }) + const result = await integration.testConnection() expect(result).toBeTruthy() }) it("test invalid connection string", async () => { - const result = await validator({ + const integration = new postgres.integration({ host, port, database: "postgres", @@ -46,6 +44,7 @@ describe("datasource validators", () => { ssl: false, rejectUnauthorized: false, }) + const result = await integration.testConnection() expect(result).toEqual({ error: 'password authentication failed for user "wrong"', }) diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 938f4c9340..536aaa1fb1 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -128,6 +128,12 @@ export interface IntegrationBase { read?(query: any): Promise update?(query: any): Promise delete?(query: any): Promise + testConnection?(): Promise< + | true + | { + error: string + } + > } export interface DatasourcePlus extends IntegrationBase { From eaf7e399e702e78a7881256834980efe35844f95 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 12:20:17 +0200 Subject: [PATCH 11/76] Remove unnecessary export --- packages/server/src/integrations/postgres.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index f6a8f22441..2c2b2341e5 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -31,7 +31,7 @@ if (types) { const JSON_REGEX = /'{.*}'::json/s -export interface PostgresConfig { +interface PostgresConfig { host: string port: number database: string From c3856a48aa3acc364251a1fa28cf8957a945a51d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 12:41:14 +0200 Subject: [PATCH 12/76] Fix types --- packages/server/src/api/controllers/integration.ts | 6 +++--- packages/server/src/sdk/app/datasources/datasources.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/integration.ts b/packages/server/src/api/controllers/integration.ts index 743d216da7..23defac831 100644 --- a/packages/server/src/api/controllers/integration.ts +++ b/packages/server/src/api/controllers/integration.ts @@ -1,4 +1,4 @@ -import { getDefinitions } from "../../integrations" +import { getDefinition, getDefinitions } from "../../integrations" import { BBContext } from "@budibase/types" export async function fetch(ctx: BBContext) { @@ -7,7 +7,7 @@ export async function fetch(ctx: BBContext) { } export async function find(ctx: BBContext) { - const defs = await getDefinitions() + const def = await getDefinition(ctx.params.type) + ctx.body = def ctx.status = 200 - ctx.body = defs[ctx.params.type] } diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index b3fe5bcdf1..dee0a221aa 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -41,7 +41,7 @@ async function enrichDatasourceWithValues(datasource: Datasource) { { onlyFound: true } ) as Datasource const definition = await getDefinition(processed.source) - processed.config = checkDatasourceTypes(definition, processed.config) + processed.config = checkDatasourceTypes(definition!, processed.config) return { datasource: processed, envVars: env as Record, From 99d525ad79f8226cbb84d4938db74ee13b3da1d8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 12:43:59 +0200 Subject: [PATCH 13/76] Move tests to qa-core --- qa-core/package.json | 2 + .../integrations/validators/postgres.spec.ts | 2 +- qa-core/yarn.lock | 249 +++++++++++++++++- 3 files changed, 247 insertions(+), 6 deletions(-) rename packages/server/src/sdk/tests/datasources/validators.spec.ts => qa-core/src/integrations/validators/postgres.spec.ts (94%) diff --git a/qa-core/package.json b/qa-core/package.json index 73fd59cab2..2940ee5d1e 100644 --- a/qa-core/package.json +++ b/qa-core/package.json @@ -16,10 +16,12 @@ "test:notify": "node scripts/testResultsWebhook", "test:smoke": "yarn run test --testPathIgnorePatterns=\\\"\\/dataSources\\/\\\"", "test:ci": "start-server-and-test dev:built http://localhost:4001/health test:smoke", + "serve": "start-server-and-test dev:built http://localhost:4001/health", "dev:built": "cd ../ && yarn dev:built" }, "devDependencies": { "@budibase/types": "^2.3.17", + "@trendyol/jest-testcontainers": "^2.1.1", "@types/jest": "29.0.0", "@types/node-fetch": "2.6.2", "chance": "1.1.8", diff --git a/packages/server/src/sdk/tests/datasources/validators.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts similarity index 94% rename from packages/server/src/sdk/tests/datasources/validators.spec.ts rename to qa-core/src/integrations/validators/postgres.spec.ts index b1b4d83fc8..a41ed51d9b 100644 --- a/packages/server/src/sdk/tests/datasources/validators.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -1,4 +1,4 @@ -import postgres from "../../../integrations/postgres" +import postgres from "../../../../packages/server/src/integrations/postgres" import { GenericContainer } from "testcontainers" jest.unmock("pg") diff --git a/qa-core/yarn.lock b/qa-core/yarn.lock index 42beb07108..272b4cc03a 100644 --- a/qa-core/yarn.lock +++ b/qa-core/yarn.lock @@ -304,6 +304,11 @@ "@babel/helper-validator-identifier" "^7.18.6" to-fast-properties "^2.0.0" +"@balena/dockerignore@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@balena/dockerignore/-/dockerignore-1.0.2.tgz#9ffe4726915251e8eb69f44ef3547e0da2c03e0d" + integrity sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q== + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -779,6 +784,15 @@ request "^2.88.0" webfinger "^0.4.2" +"@trendyol/jest-testcontainers@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@trendyol/jest-testcontainers/-/jest-testcontainers-2.1.1.tgz#dced95cf9c37b75efe0a65db9b75ae8912f2f14a" + integrity sha512-4iAc2pMsev4BTUzoA7jO1VvbTOU2N3juQUYa8TwiSPXPuQtxKwV9WB9ZEP+JQ+Pj15YqfGOXp5H0WNMPtapjiA== + dependencies: + cwd "^0.10.0" + node-duration "^1.0.4" + testcontainers "4.7.0" + "@tsconfig/node10@^1.0.7": version "1.0.9" resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" @@ -832,6 +846,13 @@ dependencies: "@babel/types" "^7.3.0" +"@types/dockerode@^2.5.34": + version "2.5.34" + resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-2.5.34.tgz#9adb884f7cc6c012a6eb4b2ad794cc5d01439959" + integrity sha512-LcbLGcvcBwBAvjH9UrUI+4qotY+A5WCer5r43DR5XHv2ZIEByNXFdPLo1XxR+v/BjkGjlggW8qUiXuVEhqfkpA== + dependencies: + "@types/node" "*" + "@types/graceful-fs@^4.1.3": version "4.1.5" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15" @@ -1006,6 +1027,11 @@ ansi-styles@^5.0.0: resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== +any-promise@^1.1.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" + integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A== + anymatch@^3.0.3: version "3.1.2" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" @@ -1044,7 +1070,7 @@ argsarray@0.0.1: resolved "https://registry.yarnpkg.com/argsarray/-/argsarray-0.0.1.tgz#6e7207b4ecdb39b0af88303fa5ae22bda8df61cb" integrity sha512-u96dg2GcAKtpTrBdDoFIM7PjcBA+6rSP0OR94MOReNRyUECL6MtQt5XXmRr4qrftYaef9+l5hcpO5te7sML1Cg== -asn1@~0.2.3: +asn1@^0.2.6, asn1@~0.2.3: version "0.2.6" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== @@ -1199,7 +1225,7 @@ base64url@3.x.x, base64url@^3.0.1: resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d" integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A== -bcrypt-pbkdf@^1.0.0: +bcrypt-pbkdf@^1.0.0, bcrypt-pbkdf@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" integrity sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w== @@ -1304,6 +1330,11 @@ buffer@^5.5.0, buffer@^5.6.0: base64-js "^1.3.1" ieee754 "^1.1.13" +buildcheck@~0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/buildcheck/-/buildcheck-0.0.6.tgz#89aa6e417cfd1e2196e3f8fe915eb709d2fe4238" + integrity sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A== + bull@4.10.1: version "4.10.1" resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.1.tgz#f14974b6089358b62b495a2cbf838aadc098e43f" @@ -1319,6 +1350,11 @@ bull@4.10.1: semver "^7.3.2" uuid "^8.3.0" +byline@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/byline/-/byline-5.0.0.tgz#741c5216468eadc457b03410118ad77de8c1ddb1" + integrity sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q== + cache-content-type@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/cache-content-type/-/cache-content-type-1.0.1.tgz#035cde2b08ee2129f4a8315ea8f00a00dba1453c" @@ -1546,6 +1582,14 @@ correlation-id@4.0.0: dependencies: uuid "^8.3.1" +cpu-features@~0.0.7: + version "0.0.7" + resolved "https://registry.yarnpkg.com/cpu-features/-/cpu-features-0.0.7.tgz#81ba93e1d0a729fd25132a54c3ff689c37b542f7" + integrity sha512-fjzFmsUKKCrC9GrM1eQTvQx18e+kjXFzjRLvJPNEDjk31+bJ6ZiV6uchv/hzbzXVIgbWdrEyyX1IFKwse65+8w== + dependencies: + buildcheck "~0.0.6" + nan "^2.17.0" + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" @@ -1572,6 +1616,14 @@ crypt@0.0.2: resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow== +cwd@^0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/cwd/-/cwd-0.10.0.tgz#172400694057c22a13b0cf16162c7e4b7a7fe567" + integrity sha512-YGZxdTTL9lmLkCUTpg4j0zQ7IhRB5ZmqNBbGCl3Tg6MP/d5/6sY7L5mmTjzbc6JKgVZYiqTQTNhPFsbXNGlRaA== + dependencies: + find-pkg "^0.1.2" + fs-exists-sync "^0.1.0" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -1676,6 +1728,32 @@ diff@^4.0.1: resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== +docker-compose@^0.23.5: + version "0.23.19" + resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.19.tgz#9947726e2fe67bdfa9e8efe1ff15aa0de2e10eb8" + integrity sha512-v5vNLIdUqwj4my80wxFDkNH+4S85zsRuH29SO7dCWVWPCMt/ohZBsGN6g6KXWifT0pzQ7uOxqEKCYCDPJ8Vz4g== + dependencies: + yaml "^1.10.2" + +docker-modem@^3.0.0: + version "3.0.8" + resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-3.0.8.tgz#ef62c8bdff6e8a7d12f0160988c295ea8705e77a" + integrity sha512-f0ReSURdM3pcKPNS30mxOHSbaFLcknGmQjwSfmbcdOw1XWKXVhukM3NJHhr7NpY9BIyyWQb0EBo3KQvvuU5egQ== + dependencies: + debug "^4.1.1" + readable-stream "^3.5.0" + split-ca "^1.0.1" + ssh2 "^1.11.0" + +dockerode@^3.2.1: + version "3.3.5" + resolved "https://registry.yarnpkg.com/dockerode/-/dockerode-3.3.5.tgz#7ae3f40f2bec53ae5e9a741ce655fff459745629" + integrity sha512-/0YNa3ZDNeLr/tSckmD69+Gq+qVNhvKfAHNeZJBnp7EOP6RGKV8ORrJHkUn20So5wU+xxT7+1n5u8PjHbfjbSA== + dependencies: + "@balena/dockerignore" "^1.0.2" + docker-modem "^3.0.0" + tar-fs "~2.0.1" + dotenv@16.0.1: version "16.0.1" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d" @@ -1844,6 +1922,13 @@ exit@^0.1.2: resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" integrity sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ== +expand-tilde@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/expand-tilde/-/expand-tilde-1.2.2.tgz#0b81eba897e5a3d31d1c3d102f8f01441e559449" + integrity sha512-rtmc+cjLZqnu9dSYosX9EWmSJhTwpACgJQTfj4hgg2JjOD/6SIQalZrt4a3aQeh++oNxkazcaxrhPUj6+g5G/Q== + dependencies: + os-homedir "^1.0.1" + expect@^29.0.0: version "29.0.2" resolved "https://registry.yarnpkg.com/expect/-/expect-29.0.2.tgz#22c7132400f60444b427211f1d6bb604a9ab2420" @@ -1919,6 +2004,21 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" +find-file-up@^0.1.2: + version "0.1.3" + resolved "https://registry.yarnpkg.com/find-file-up/-/find-file-up-0.1.3.tgz#cf68091bcf9f300a40da411b37da5cce5a2fbea0" + integrity sha512-mBxmNbVyjg1LQIIpgO8hN+ybWBgDQK8qjht+EbrTCGmmPV/sc7RF1i9stPTD6bpvXZywBdrwRYxhSdJv867L6A== + dependencies: + fs-exists-sync "^0.1.0" + resolve-dir "^0.1.0" + +find-pkg@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/find-pkg/-/find-pkg-0.1.2.tgz#1bdc22c06e36365532e2a248046854b9788da557" + integrity sha512-0rnQWcFwZr7eO0513HahrWafsc3CTFioEB7DRiEYCUM/70QXSY8f3mCST17HXLcPvEhzH/Ty/Bxd72ZZsr/yvw== + dependencies: + find-file-up "^0.1.2" + find-up@^4.0.0, find-up@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" @@ -1984,6 +2084,11 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== +fs-exists-sync@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz#982d6893af918e72d08dec9e8673ff2b5a8d6add" + integrity sha512-cR/vflFyPZtrN6b38ZyWxpWdhlXrzZEBawlpBQMq7033xVY7/kg0GDMBK5jg8lDYQckdJ5x/YC88lM3C7VMsLg== + fs-minipass@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" @@ -2062,7 +2167,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@^7.1.3, glob@^7.1.4: +glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== @@ -2074,6 +2179,24 @@ glob@^7.1.3, glob@^7.1.4: once "^1.3.0" path-is-absolute "^1.0.0" +global-modules@^0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-0.2.3.tgz#ea5a3bed42c6d6ce995a4f8a1269b5dae223828d" + integrity sha512-JeXuCbvYzYXcwE6acL9V2bAOeSIGl4dD+iwLY9iUx2VBJJ80R18HCn+JCwHM9Oegdfya3lEkGCdaRkSyc10hDA== + dependencies: + global-prefix "^0.1.4" + is-windows "^0.2.0" + +global-prefix@^0.1.4: + version "0.1.5" + resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-0.1.5.tgz#8d3bc6b8da3ca8112a160d8d496ff0462bfef78f" + integrity sha512-gOPiyxcD9dJGCEArAhF4Hd0BAqvAe/JzERP7tYumE4yIkmIedPUVXcJFWbV3/p/ovIIvKjkrTk+f1UVkq7vvbw== + dependencies: + homedir-polyfill "^1.0.0" + ini "^1.3.4" + is-windows "^0.2.0" + which "^1.2.12" + globals@^11.1.0: version "11.12.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" @@ -2131,6 +2254,13 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +homedir-polyfill@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz#743298cef4e5af3e194161fbadcc2151d3a058e8" + integrity sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA== + dependencies: + parse-passwd "^1.0.0" + html-escaper@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" @@ -2230,6 +2360,11 @@ inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, i resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== +ini@^1.3.4: + version "1.3.8" + resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" + integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== + ioredis@4.28.0: version "4.28.0" resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.0.tgz#5a2be3f37ff2075e2332f280eaeb02ab4d9ff0d3" @@ -2318,6 +2453,11 @@ is-typedarray@~1.0.0: resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== +is-windows@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-0.2.0.tgz#de1aa6d63ea29dd248737b69f1ff8b8002d2108c" + integrity sha512-n67eJYmXbniZB7RF4I/FTjK1s6RPOCTxhYrVYLRaCt3lF0mpWZPKr3T2LSZAqyjQsxR2qMmGYXXzK0YWwcPM1Q== + isarray@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" @@ -3332,6 +3472,11 @@ msgpackr@^1.5.2: optionalDependencies: msgpackr-extract "^3.0.0" +nan@^2.17.0: + version "2.17.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.17.0.tgz#c0150a2368a182f033e9aa5195ec76ea41a199cb" + integrity sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ== + napi-macros@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b" @@ -3367,6 +3512,11 @@ node-addon-api@^3.1.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A== +node-duration@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/node-duration/-/node-duration-1.0.4.tgz#3e94ecc0e473691c89c4560074503362071cecac" + integrity sha512-eUXYNSY7DL53vqfTosggWkvyIW3bhAcqBDIlolgNYlZhianXTrCL50rlUJWD1eRqkIxMppXTfiFbp+9SjpPrgA== + node-fetch@2, node-fetch@2.6.7, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" @@ -3490,6 +3640,11 @@ only@~0.0.2: resolved "https://registry.yarnpkg.com/only/-/only-0.0.2.tgz#2afde84d03e50b9a8edc444e30610a70295edfb4" integrity sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ== +os-homedir@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" + integrity sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ== + p-finally@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" @@ -3543,6 +3698,11 @@ parse-json@^5.2.0: json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" +parse-passwd@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6" + integrity sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q== + parseurl@^1.3.2, parseurl@^1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" @@ -3982,6 +4142,15 @@ readable-stream@1.1.14, readable-stream@^1.0.27-1: string_decoder "^1.1.1" util-deprecate "^1.0.1" +readable-stream@^3.5.0: + version "3.6.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" + integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + readable-stream@~0.0.2: version "0.0.4" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-0.0.4.tgz#f32d76e3fb863344a548d79923007173665b3b8d" @@ -4077,6 +4246,14 @@ resolve-cwd@^3.0.0: dependencies: resolve-from "^5.0.0" +resolve-dir@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/resolve-dir/-/resolve-dir-0.1.1.tgz#b219259a5602fac5c5c496ad894a6e8cc430261e" + integrity sha512-QxMPqI6le2u0dCLyiGzgy92kjkkL6zO0XyvHzjdTNH3zM6e5Hz3BwG6+aEyNgiQ5Xz6PwTwgQEj3U50dByPKIA== + dependencies: + expand-tilde "^1.2.2" + global-modules "^0.2.3" + resolve-from@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" @@ -4238,6 +4415,11 @@ spark-md5@3.0.2: resolved "https://registry.yarnpkg.com/spark-md5/-/spark-md5-3.0.2.tgz#7952c4a30784347abcee73268e473b9c0167e3fc" integrity sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw== +split-ca@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6" + integrity sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ== + split2@^2.1.0: version "2.2.0" resolved "https://registry.yarnpkg.com/split2/-/split2-2.2.0.tgz#186b2575bcf83e85b7d18465756238ee4ee42493" @@ -4257,6 +4439,17 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== +ssh2@^1.11.0: + version "1.13.0" + resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-1.13.0.tgz#9b53a07534fa72283ada471b82395a3b3c875934" + integrity sha512-CIZBFRRY1y9mAZSqBGFE4EB4dNJad2ysT2PqO8OpkiI3UTB/gUZwE5EaN16qVyQ6s/M7EgC/iaV/MnjdlvnuzA== + dependencies: + asn1 "^0.2.6" + bcrypt-pbkdf "^1.0.2" + optionalDependencies: + cpu-features "~0.0.7" + nan "^2.17.0" + sshpk@^1.7.0: version "1.17.0" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" @@ -4314,6 +4507,13 @@ stream-combiner@~0.0.4: dependencies: duplexer "~0.1.1" +stream-to-array@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" + integrity sha512-UsZtOYEn4tWU2RGLOXr/o/xjRBftZRlG3dEWoaHr8j4GuypJ3isitGbVyjQKAuMu+xbiop8q224TjiZWc4XTZA== + dependencies: + any-promise "^1.1.0" + string-length@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" @@ -4403,7 +4603,7 @@ supports-preserve-symlinks-flag@^1.0.0: resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== -tar-fs@2.1.1: +tar-fs@2.1.1, tar-fs@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== @@ -4413,7 +4613,17 @@ tar-fs@2.1.1: pump "^3.0.0" tar-stream "^2.1.4" -tar-stream@^2.1.4: +tar-fs@~2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" + integrity sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA== + dependencies: + chownr "^1.1.1" + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^2.0.0" + +tar-stream@^2.0.0, tar-stream@^2.1.4: version "2.2.0" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== @@ -4445,6 +4655,23 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" +testcontainers@4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/testcontainers/-/testcontainers-4.7.0.tgz#5a9a864b1b0cc86984086dcc737c2f5e73490cf3" + integrity sha512-5SrG9RMfDRRZig34fDZeMcGD5i3lHCOJzn0kjouyK4TiEWjZB3h7kCk8524lwNRHROFE1j6DGjceonv/5hl5ag== + dependencies: + "@types/dockerode" "^2.5.34" + byline "^5.0.0" + debug "^4.1.1" + docker-compose "^0.23.5" + dockerode "^3.2.1" + get-port "^5.1.1" + glob "^7.1.6" + node-duration "^1.0.4" + slash "^3.0.0" + stream-to-array "^2.3.0" + tar-fs "^2.1.0" + through2@3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4" @@ -4746,6 +4973,13 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" +which@^1.2.12: + version "1.3.1" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" + integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== + dependencies: + isexe "^2.0.0" + which@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" @@ -4824,6 +5058,11 @@ yallist@^4.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== +yaml@^1.10.2: + version "1.10.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" + integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== + yargs-parser@^21.0.0, yargs-parser@^21.0.1: version "21.1.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" From 1c2c00e62146f7f405d9250e0b2e49874f615407 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 13:13:27 +0200 Subject: [PATCH 14/76] Fix bool --- packages/types/src/sdk/datasources.ts | 2 +- qa-core/src/integrations/validators/postgres.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 536aaa1fb1..63091b7361 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -129,7 +129,7 @@ export interface IntegrationBase { update?(query: any): Promise delete?(query: any): Promise testConnection?(): Promise< - | true + | boolean | { error: string } diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index a41ed51d9b..b4d811ce18 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -30,7 +30,7 @@ describe("datasource validators", () => { rejectUnauthorized: false, }) const result = await integration.testConnection() - expect(result).toBeTruthy() + expect(result).toBe(true) }) it("test invalid connection string", async () => { From d9a38b090817076afb28ee0220ee646d68ec6f79 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 13:47:24 +0200 Subject: [PATCH 15/76] Update import --- qa-core/src/integrations/validators/postgres.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index b4d811ce18..5aa3250e2a 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -1,5 +1,5 @@ -import postgres from "../../../../packages/server/src/integrations/postgres" import { GenericContainer } from "testcontainers" +import postgres from "../../../../packages/server/src/integrations/postgres" jest.unmock("pg") From f10a66cc421057e773281e73543633a3f48933f2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 May 2023 11:29:02 +0200 Subject: [PATCH 16/76] Type mysql configs --- packages/server/src/integrations/mysql.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 8d984ed402..ceec64fac7 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -20,18 +20,11 @@ import { NUMBER_REGEX } from "../utilities" import Sql from "./base/sql" import { MySQLColumn } from "./base/types" -const mysql = require("mysql2/promise") +import mysql from "mysql2/promise" -interface MySQLConfig { - host: string - port: number - user: string - password: string +export interface MySQLConfig extends mysql.ConnectionOptions { database: string - ssl?: { [key: string]: any } rejectUnauthorized: boolean - typeCast: Function - multipleStatements: boolean } const SCHEMA: Integration = { @@ -134,7 +127,8 @@ class MySQLIntegration extends Sql implements DatasourcePlus { if ( config.rejectUnauthorized != null && !config.rejectUnauthorized && - config.ssl + config.ssl && + typeof config.ssl !== "string" ) { config.ssl.rejectUnauthorized = config.rejectUnauthorized } From 721492e76d215b075e2f8952b0b87ef2b6545392 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 May 2023 13:25:35 +0200 Subject: [PATCH 17/76] Validate mysql --- packages/server/src/integrations/mysql.ts | 24 +++++-- .../integrations/validators/postgres.spec.ts | 62 +++++++++++++++++++ 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index ceec64fac7..89a8297818 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -85,8 +85,6 @@ const SCHEMA: Integration = { }, } -const TimezoneAwareDateTypes = ["timestamp"] - function bindingTypeCoerce(bindings: any[]) { for (let i = 0; i < bindings.length; i++) { const binding = bindings[i] @@ -113,7 +111,7 @@ function bindingTypeCoerce(bindings: any[]) { class MySQLIntegration extends Sql implements DatasourcePlus { private config: MySQLConfig - private client: any + private client?: mysql.Connection public tables: Record = {} public schemaErrors: Record = {} @@ -167,7 +165,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus { } async disconnect() { - await this.client.end() + await this.client!.end() } async internalQuery( @@ -186,10 +184,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus { ? baseBindings : bindingTypeCoerce(baseBindings) // Node MySQL is callback based, so we must wrap our call in a promise - const response = await this.client.query(query.sql, bindings) + const response = await this.client!.query(query.sql, bindings) return response[0] } finally { - if (opts?.connect) { + if (opts?.connect && this.client) { await this.disconnect() } } @@ -288,7 +286,21 @@ class MySQLIntegration extends Sql implements DatasourcePlus { } } +async function validateConnection(config: MySQLConfig) { + const integration = new MySQLIntegration(config) + try { + const [result] = await integration.internalQuery( + { sql: "SELECT 1+1 AS checkRes" }, + { connect: true } + ) + return result?.checkRes == 2 + } catch (e: any) { + return { error: e.message as string } + } +} + export default { schema: SCHEMA, integration: MySQLIntegration, + validateConnection, } diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index 5aa3250e2a..c5f6fae1ad 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -2,6 +2,7 @@ import { GenericContainer } from "testcontainers" import postgres from "../../../../packages/server/src/integrations/postgres" jest.unmock("pg") +jest.unmock("mysql2/promise") describe("datasource validators", () => { describe("postgres", () => { @@ -50,4 +51,65 @@ describe("datasource validators", () => { }) }) }) + + describe("mysql", () => { + const validator = integrations.getValidator[SourceName.MYSQL]! + + let host: string + let port: number + + beforeAll(async () => { + const container = await new GenericContainer("mysql") + .withExposedPorts(3306) + .withEnv("MYSQL_ROOT_PASSWORD", "admin") + .withEnv("MYSQL_DATABASE", "db") + .withEnv("MYSQL_USER", "user") + .withEnv("MYSQL_PASSWORD", "password") + .start() + + host = container.getContainerIpAddress() + port = container.getMappedPort(3306) + }) + + it("test valid connection string", async () => { + const result = await validator({ + host, + port, + user: "user", + database: "db", + password: "password", + rejectUnauthorized: true, + }) + expect(result).toBe(true) + }) + + it("test invalid database", async () => { + const result = await validator({ + host, + port, + user: "user", + database: "test", + password: "password", + rejectUnauthorized: true, + }) + expect(result).toEqual({ + error: "Access denied for user 'user'@'%' to database 'test'", + }) + }) + + it("test invalid password", async () => { + const result = await validator({ + host, + port, + user: "root", + database: "test", + password: "wrong", + rejectUnauthorized: true, + }) + expect(result).toEqual({ + error: + "Access denied for user 'root'@'172.17.0.1' (using password: YES)", + }) + }) + }) }) From 8a1564ef076d0120dcb5a9d7e86031488c22d75b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 12:29:52 +0200 Subject: [PATCH 18/76] Implement the check as part of the integration --- packages/server/src/integrations/mysql.ts | 28 +++++++++---------- .../integrations/validators/postgres.spec.ts | 12 ++++---- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 89a8297818..199ed6fc7d 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -22,7 +22,7 @@ import { MySQLColumn } from "./base/types" import mysql from "mysql2/promise" -export interface MySQLConfig extends mysql.ConnectionOptions { +interface MySQLConfig extends mysql.ConnectionOptions { database: string rejectUnauthorized: boolean } @@ -152,6 +152,18 @@ class MySQLIntegration extends Sql implements DatasourcePlus { } } + async testConnection() { + try { + const [result] = await this.internalQuery( + { sql: "SELECT 1+1 AS checkRes" }, + { connect: true } + ) + return result?.checkRes == 2 + } catch (e: any) { + return { error: e.message as string } + } + } + getBindingIdentifier(): string { return "?" } @@ -286,21 +298,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus { } } -async function validateConnection(config: MySQLConfig) { - const integration = new MySQLIntegration(config) - try { - const [result] = await integration.internalQuery( - { sql: "SELECT 1+1 AS checkRes" }, - { connect: true } - ) - return result?.checkRes == 2 - } catch (e: any) { - return { error: e.message as string } - } -} - export default { schema: SCHEMA, integration: MySQLIntegration, - validateConnection, } diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index c5f6fae1ad..07f2b80f1a 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -1,5 +1,6 @@ import { GenericContainer } from "testcontainers" import postgres from "../../../../packages/server/src/integrations/postgres" +import mysql from "../../../../packages/server/src/integrations/mysql" jest.unmock("pg") jest.unmock("mysql2/promise") @@ -53,8 +54,6 @@ describe("datasource validators", () => { }) describe("mysql", () => { - const validator = integrations.getValidator[SourceName.MYSQL]! - let host: string let port: number @@ -72,7 +71,7 @@ describe("datasource validators", () => { }) it("test valid connection string", async () => { - const result = await validator({ + const integration = new mysql.integration({ host, port, user: "user", @@ -80,11 +79,12 @@ describe("datasource validators", () => { password: "password", rejectUnauthorized: true, }) + const result = await integration.testConnection() expect(result).toBe(true) }) it("test invalid database", async () => { - const result = await validator({ + const integration = new mysql.integration({ host, port, user: "user", @@ -92,13 +92,14 @@ describe("datasource validators", () => { password: "password", rejectUnauthorized: true, }) + const result = await integration.testConnection() expect(result).toEqual({ error: "Access denied for user 'user'@'%' to database 'test'", }) }) it("test invalid password", async () => { - const result = await validator({ + const integration = new mysql.integration({ host, port, user: "root", @@ -106,6 +107,7 @@ describe("datasource validators", () => { password: "wrong", rejectUnauthorized: true, }) + const result = await integration.testConnection() expect(result).toEqual({ error: "Access denied for user 'root'@'172.17.0.1' (using password: YES)", From e92b9a8abca73822747d330d8faa46756bb45207 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 13:39:31 +0200 Subject: [PATCH 19/76] Split testss --- .../src/integrations/validators/mysql.spec.ts | 68 +++++++++++++++++++ .../integrations/validators/postgres.spec.ts | 64 ----------------- 2 files changed, 68 insertions(+), 64 deletions(-) create mode 100644 qa-core/src/integrations/validators/mysql.spec.ts diff --git a/qa-core/src/integrations/validators/mysql.spec.ts b/qa-core/src/integrations/validators/mysql.spec.ts new file mode 100644 index 0000000000..166085fdd5 --- /dev/null +++ b/qa-core/src/integrations/validators/mysql.spec.ts @@ -0,0 +1,68 @@ +import { GenericContainer } from "testcontainers" +import mysql from "../../../../packages/server/src/integrations/mysql" + +jest.unmock("mysql2/promise") + +describe("datasource validators", () => { + describe("mysql", () => { + let host: string + let port: number + + beforeAll(async () => { + const container = await new GenericContainer("mysql") + .withExposedPorts(3306) + .withEnv("MYSQL_ROOT_PASSWORD", "admin") + .withEnv("MYSQL_DATABASE", "db") + .withEnv("MYSQL_USER", "user") + .withEnv("MYSQL_PASSWORD", "password") + .start() + + host = container.getContainerIpAddress() + port = container.getMappedPort(3306) + }) + + it("test valid connection string", async () => { + const integration = new mysql.integration({ + host, + port, + user: "user", + database: "db", + password: "password", + rejectUnauthorized: true, + }) + const result = await integration.testConnection() + expect(result).toBe(true) + }) + + it("test invalid database", async () => { + const integration = new mysql.integration({ + host, + port, + user: "user", + database: "test", + password: "password", + rejectUnauthorized: true, + }) + const result = await integration.testConnection() + expect(result).toEqual({ + error: "Access denied for user 'user'@'%' to database 'test'", + }) + }) + + it("test invalid password", async () => { + const integration = new mysql.integration({ + host, + port, + user: "root", + database: "test", + password: "wrong", + rejectUnauthorized: true, + }) + const result = await integration.testConnection() + expect(result).toEqual({ + error: + "Access denied for user 'root'@'172.17.0.1' (using password: YES)", + }) + }) + }) +}) diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index 07f2b80f1a..5aa3250e2a 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -1,9 +1,7 @@ import { GenericContainer } from "testcontainers" import postgres from "../../../../packages/server/src/integrations/postgres" -import mysql from "../../../../packages/server/src/integrations/mysql" jest.unmock("pg") -jest.unmock("mysql2/promise") describe("datasource validators", () => { describe("postgres", () => { @@ -52,66 +50,4 @@ describe("datasource validators", () => { }) }) }) - - describe("mysql", () => { - let host: string - let port: number - - beforeAll(async () => { - const container = await new GenericContainer("mysql") - .withExposedPorts(3306) - .withEnv("MYSQL_ROOT_PASSWORD", "admin") - .withEnv("MYSQL_DATABASE", "db") - .withEnv("MYSQL_USER", "user") - .withEnv("MYSQL_PASSWORD", "password") - .start() - - host = container.getContainerIpAddress() - port = container.getMappedPort(3306) - }) - - it("test valid connection string", async () => { - const integration = new mysql.integration({ - host, - port, - user: "user", - database: "db", - password: "password", - rejectUnauthorized: true, - }) - const result = await integration.testConnection() - expect(result).toBe(true) - }) - - it("test invalid database", async () => { - const integration = new mysql.integration({ - host, - port, - user: "user", - database: "test", - password: "password", - rejectUnauthorized: true, - }) - const result = await integration.testConnection() - expect(result).toEqual({ - error: "Access denied for user 'user'@'%' to database 'test'", - }) - }) - - it("test invalid password", async () => { - const integration = new mysql.integration({ - host, - port, - user: "root", - database: "test", - password: "wrong", - rejectUnauthorized: true, - }) - const result = await integration.testConnection() - expect(result).toEqual({ - error: - "Access denied for user 'root'@'172.17.0.1' (using password: YES)", - }) - }) - }) }) From 209ada0c389809c8edb464eb6bf1d04c71455832 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 May 2023 15:58:51 +0200 Subject: [PATCH 20/76] Validate couchdb --- .../backend-core/src/db/couch/DatabaseImpl.ts | 14 ++++++++++---- packages/backend-core/src/db/couch/utils.ts | 16 +++++++++++++++- packages/server/src/integrations/couchdb.ts | 19 ++++++++++++++----- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 94d78e94ff..29ca4123f5 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -12,7 +12,7 @@ import { isDocument, } from "@budibase/types" import { getCouchInfo } from "./connections" -import { directCouchCall } from "./utils" +import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" import { WriteStream, ReadStream } from "fs" import { newid } from "../../docIds/newid" @@ -46,6 +46,8 @@ export class DatabaseImpl implements Database { private readonly instanceNano?: Nano.ServerScope private readonly pouchOpts: DatabaseOpts + private readonly couchInfo = getCouchInfo() + constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) { if (dbName == null) { throw new Error("Database name cannot be undefined.") @@ -53,8 +55,8 @@ export class DatabaseImpl implements Database { this.name = dbName this.pouchOpts = opts || {} if (connection) { - const couchInfo = getCouchInfo(connection) - this.instanceNano = buildNano(couchInfo) + this.couchInfo = getCouchInfo(connection) + this.instanceNano = buildNano(this.couchInfo) } if (!DatabaseImpl.nano) { DatabaseImpl.init() @@ -67,7 +69,11 @@ export class DatabaseImpl implements Database { } async exists() { - let response = await directCouchCall(`/${this.name}`, "HEAD") + const response = await directCouchUrlCall({ + url: `${this.couchInfo.url}/${this.name}`, + method: "HEAD", + cookie: this.couchInfo.cookie, + }) return response.status === 200 } diff --git a/packages/backend-core/src/db/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts index 426bf92158..51b2a38998 100644 --- a/packages/backend-core/src/db/couch/utils.ts +++ b/packages/backend-core/src/db/couch/utils.ts @@ -9,6 +9,20 @@ export async function directCouchCall( ) { let { url, cookie } = getCouchInfo() const couchUrl = `${url}/${path}` + return await directCouchUrlCall({ url: couchUrl, cookie, method, body }) +} + +export async function directCouchUrlCall({ + url, + cookie, + method, + body, +}: { + url: string + cookie: string + method: string + body?: any +}) { const params: any = { method: method, headers: { @@ -19,7 +33,7 @@ export async function directCouchCall( params.body = JSON.stringify(body) params.headers["Content-Type"] = "application/json" } - return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) + return await fetch(checkSlashesInUrl(encodeURI(url)), params) } export async function directCouchQuery( diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index 257b84ca13..1c6c978612 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -7,7 +7,7 @@ import { } from "@budibase/types" import { db as dbCore } from "@budibase/backend-core" -interface CouchDBConfig { +export interface CouchDBConfig { url: string database: string } @@ -61,11 +61,9 @@ const SCHEMA: Integration = { } class CouchDBIntegration implements IntegrationBase { - private config: CouchDBConfig - private readonly client: any + private readonly client: dbCore.DatabaseImpl constructor(config: CouchDBConfig) { - this.config = config this.client = dbCore.DatabaseWithConnection(config.database, config.url) } @@ -75,7 +73,7 @@ class CouchDBIntegration implements IntegrationBase { query: { json?: object; id?: string } ) { try { - return await this.client[command](query.id || query.json) + return await (this.client as any)[command](query.id || query.json) } catch (err) { console.error(errorMsg, err) throw err @@ -127,7 +125,18 @@ class CouchDBIntegration implements IntegrationBase { } } +async function validateConnection(config: CouchDBConfig) { + const integration = new CouchDBIntegration(config) + try { + const result = await integration.query("exists", "validation error", {}) + return result === true + } catch (e: any) { + return { error: e.message as string } + } +} + export default { schema: SCHEMA, integration: CouchDBIntegration, + validateConnection, } From 5456866c45ed0f5d47ad022f1449f9e1e9fc03a1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 May 2023 17:14:51 +0200 Subject: [PATCH 21/76] Fix tests to use connection string instead of env --- .../backend-core/src/db/couch/connections.ts | 16 ++++++++-------- .../src/integrations/validators/postgres.spec.ts | 2 ++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/db/couch/connections.ts b/packages/backend-core/src/db/couch/connections.ts index 06c661f350..4214c7cdc6 100644 --- a/packages/backend-core/src/db/couch/connections.ts +++ b/packages/backend-core/src/db/couch/connections.ts @@ -4,21 +4,21 @@ export const getCouchInfo = (connection?: string) => { const urlInfo = getUrlInfo(connection) let username let password - if (env.COUCH_DB_USERNAME) { - // set from env - username = env.COUCH_DB_USERNAME - } else if (urlInfo.auth.username) { + if (urlInfo.auth?.username) { // set from url username = urlInfo.auth.username + } else if (env.COUCH_DB_USERNAME) { + // set from env + username = env.COUCH_DB_USERNAME } else if (!env.isTest()) { throw new Error("CouchDB username not set") } - if (env.COUCH_DB_PASSWORD) { - // set from env - password = env.COUCH_DB_PASSWORD - } else if (urlInfo.auth.password) { + if (urlInfo.auth?.password) { // set from url password = urlInfo.auth.password + } else if (env.COUCH_DB_PASSWORD) { + // set from env + password = env.COUCH_DB_PASSWORD } else if (!env.isTest()) { throw new Error("CouchDB password not set") } diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index 5aa3250e2a..6e1219540b 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -1,6 +1,8 @@ import { GenericContainer } from "testcontainers" import postgres from "../../../../packages/server/src/integrations/postgres" +import { generator } from "@budibase/backend-core/tests" + jest.unmock("pg") describe("datasource validators", () => { From d656edad4014014d259cf897b32c57734c40a4ea Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 12:35:07 +0200 Subject: [PATCH 22/76] Implement the check as part of the integration --- packages/server/src/integrations/couchdb.ts | 20 +++---- .../integrations/validators/postgres.spec.ts | 60 +++++++++++++++++++ 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index 1c6c978612..a4692c7cd6 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -67,6 +67,15 @@ class CouchDBIntegration implements IntegrationBase { this.client = dbCore.DatabaseWithConnection(config.database, config.url) } + async testConnection() { + try { + const result = await this.query("exists", "validation error", {}) + return result === true + } catch (e: any) { + return { error: e.message as string } + } + } + async query( command: string, errorMsg: string, @@ -125,18 +134,7 @@ class CouchDBIntegration implements IntegrationBase { } } -async function validateConnection(config: CouchDBConfig) { - const integration = new CouchDBIntegration(config) - try { - const result = await integration.query("exists", "validation error", {}) - return result === true - } catch (e: any) { - return { error: e.message as string } - } -} - export default { schema: SCHEMA, integration: CouchDBIntegration, - validateConnection, } diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index 6e1219540b..78bda902d2 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -3,6 +3,8 @@ import postgres from "../../../../packages/server/src/integrations/postgres" import { generator } from "@budibase/backend-core/tests" +import couchdb from "../../../../packages/server/src/integrations/couchdb" + jest.unmock("pg") describe("datasource validators", () => { @@ -52,4 +54,62 @@ describe("datasource validators", () => { }) }) }) + + describe("couchdb", () => { + let url: string + + beforeAll(async () => { + const user = generator.first() + const password = generator.hash() + + const container = await new GenericContainer("budibase/couchdb") + .withExposedPorts(5984) + .withEnv("COUCHDB_USER", user) + .withEnv("COUCHDB_PASSWORD", password) + .start() + + const host = container.getContainerIpAddress() + const port = container.getMappedPort(5984) + + await container.exec([ + `curl`, + `-u`, + `${user}:${password}`, + `-X`, + `PUT`, + `localhost:5984/db`, + ]) + url = `http://${user}:${password}@${host}:${port}` + }) + + it("test valid connection string", async () => { + const integration = new couchdb.integration({ + url, + database: "db", + }) + const result = await integration.testConnection() + expect(result).toBe(true) + }) + + it("test invalid database", async () => { + const integration = new couchdb.integration({ + url, + database: "random_db", + }) + const result = await integration.testConnection() + expect(result).toBe(false) + }) + + it("test invalid url", async () => { + const integration = new couchdb.integration({ + url: "http://invalid:123", + database: "any", + }) + const result = await integration.testConnection() + expect(result).toEqual({ + error: + "request to http://invalid:123/any failed, reason: getaddrinfo ENOTFOUND invalid", + }) + }) + }) }) From 33988428ea24894db285529c88821bc9b60b3297 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 13:52:55 +0200 Subject: [PATCH 23/76] Split tests --- .../src/integrations/validators/couch.spec.ts | 64 +++++++++++++++++++ .../integrations/validators/postgres.spec.ts | 62 ------------------ 2 files changed, 64 insertions(+), 62 deletions(-) create mode 100644 qa-core/src/integrations/validators/couch.spec.ts diff --git a/qa-core/src/integrations/validators/couch.spec.ts b/qa-core/src/integrations/validators/couch.spec.ts new file mode 100644 index 0000000000..18fbea8c43 --- /dev/null +++ b/qa-core/src/integrations/validators/couch.spec.ts @@ -0,0 +1,64 @@ +import { GenericContainer } from "testcontainers" +import { generator } from "@budibase/backend-core/tests" + +import couchdb from "../../../../packages/server/src/integrations/couchdb" + +describe("datasource validators", () => { + describe("couchdb", () => { + let url: string + + beforeAll(async () => { + const user = generator.first() + const password = generator.hash() + + const container = await new GenericContainer("budibase/couchdb") + .withExposedPorts(5984) + .withEnv("COUCHDB_USER", user) + .withEnv("COUCHDB_PASSWORD", password) + .start() + + const host = container.getContainerIpAddress() + const port = container.getMappedPort(5984) + + await container.exec([ + `curl`, + `-u`, + `${user}:${password}`, + `-X`, + `PUT`, + `localhost:5984/db`, + ]) + url = `http://${user}:${password}@${host}:${port}` + }) + + it("test valid connection string", async () => { + const integration = new couchdb.integration({ + url, + database: "db", + }) + const result = await integration.testConnection() + expect(result).toBe(true) + }) + + it("test invalid database", async () => { + const integration = new couchdb.integration({ + url, + database: "random_db", + }) + const result = await integration.testConnection() + expect(result).toBe(false) + }) + + it("test invalid url", async () => { + const integration = new couchdb.integration({ + url: "http://invalid:123", + database: "any", + }) + const result = await integration.testConnection() + expect(result).toEqual({ + error: + "request to http://invalid:123/any failed, reason: getaddrinfo ENOTFOUND invalid", + }) + }) + }) +}) diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index 78bda902d2..5aa3250e2a 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -1,10 +1,6 @@ import { GenericContainer } from "testcontainers" import postgres from "../../../../packages/server/src/integrations/postgres" -import { generator } from "@budibase/backend-core/tests" - -import couchdb from "../../../../packages/server/src/integrations/couchdb" - jest.unmock("pg") describe("datasource validators", () => { @@ -54,62 +50,4 @@ describe("datasource validators", () => { }) }) }) - - describe("couchdb", () => { - let url: string - - beforeAll(async () => { - const user = generator.first() - const password = generator.hash() - - const container = await new GenericContainer("budibase/couchdb") - .withExposedPorts(5984) - .withEnv("COUCHDB_USER", user) - .withEnv("COUCHDB_PASSWORD", password) - .start() - - const host = container.getContainerIpAddress() - const port = container.getMappedPort(5984) - - await container.exec([ - `curl`, - `-u`, - `${user}:${password}`, - `-X`, - `PUT`, - `localhost:5984/db`, - ]) - url = `http://${user}:${password}@${host}:${port}` - }) - - it("test valid connection string", async () => { - const integration = new couchdb.integration({ - url, - database: "db", - }) - const result = await integration.testConnection() - expect(result).toBe(true) - }) - - it("test invalid database", async () => { - const integration = new couchdb.integration({ - url, - database: "random_db", - }) - const result = await integration.testConnection() - expect(result).toBe(false) - }) - - it("test invalid url", async () => { - const integration = new couchdb.integration({ - url: "http://invalid:123", - database: "any", - }) - const result = await integration.testConnection() - expect(result).toEqual({ - error: - "request to http://invalid:123/any failed, reason: getaddrinfo ENOTFOUND invalid", - }) - }) - }) }) From bd736836f0267d8418de26ca6d3bb8b3514f195a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 13:55:22 +0200 Subject: [PATCH 24/76] Remove export --- packages/server/src/integrations/couchdb.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index a4692c7cd6..24100f9c9f 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -7,7 +7,7 @@ import { } from "@budibase/types" import { db as dbCore } from "@budibase/backend-core" -export interface CouchDBConfig { +interface CouchDBConfig { url: string database: string } From 32695018bf5bbec7b01790890a81b5bbef85096d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 May 2023 17:48:37 +0200 Subject: [PATCH 25/76] Validate and test microsoft sql --- .../src/integrations/microsoftSqlServer.ts | 17 ++++++- .../integrations/validators/postgres.spec.ts | 47 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index eb87c1ccf1..3f14360d57 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -22,7 +22,7 @@ import { MSSQLTablesResponse, MSSQLColumn } from "./base/types" const sqlServer = require("mssql") const DEFAULT_SCHEMA = "dbo" -interface MSSQLConfig { +export interface MSSQLConfig { user: string password: string server: string @@ -138,6 +138,10 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { } } + // async end(){ + // this.client!. + // } + async internalQuery( query: SqlQuery, operation: string | undefined = undefined @@ -306,7 +310,18 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { } } +async function validateConnection(config: MSSQLConfig) { + const integration = new SqlServerIntegration(config) + try { + await integration.connect() + return true + } catch (e: any) { + return { error: e.message as string } + } +} + export default { schema: SCHEMA, integration: SqlServerIntegration, + validateConnection, } diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index 5aa3250e2a..dcc0b4cb73 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -2,6 +2,7 @@ import { GenericContainer } from "testcontainers" import postgres from "../../../../packages/server/src/integrations/postgres" jest.unmock("pg") +jest.unmock("mssql") describe("datasource validators", () => { describe("postgres", () => { @@ -50,4 +51,50 @@ describe("datasource validators", () => { }) }) }) + + describe("mssql", () => { + const validator = integrations.getValidator[SourceName.SQL_SERVER]! + + let host: string, port: number + + beforeAll(async () => { + const container = await new GenericContainer( + "mcr.microsoft.com/mssql/server" + ) + .withExposedPorts(1433) + .withEnv("ACCEPT_EULA", "Y") + .withEnv("MSSQL_SA_PASSWORD", "Str0Ng_p@ssW0rd!") + .withEnv("MSSQL_PID", "Developer") + .start() + + host = container.getContainerIpAddress() + port = container.getMappedPort(1433) + }) + + it("test valid connection string", async () => { + const result = await validator({ + user: "sa", + password: "Str0Ng_p@ssW0rd!", + server: host, + port: port, + database: "master", + schema: "dbo", + }) + expect(result).toBe(true) + }) + + it("test invalid password", async () => { + const result = await validator({ + user: "sa", + password: "wrong_pwd", + server: host, + port: port, + database: "master", + schema: "dbo", + }) + expect(result).toEqual({ + error: "ConnectionError: Login failed for user 'sa'.", + }) + }) + }) }) From fd18529135e636cd77aaf22b1b0f18b3ddb497b6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 May 2023 20:32:44 +0200 Subject: [PATCH 26/76] Wait for readiness --- .../integrations/validators/postgres.spec.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index dcc0b4cb73..e36686fe64 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -1,5 +1,8 @@ import { GenericContainer } from "testcontainers" +import { Duration, TemporalUnit } from "node-duration" + import postgres from "../../../../packages/server/src/integrations/postgres" +import { SourceName } from "@budibase/types" jest.unmock("pg") jest.unmock("mssql") @@ -57,14 +60,24 @@ describe("datasource validators", () => { let host: string, port: number + const password = "Str0Ng_p@ssW0rd!" + beforeAll(async () => { const container = await new GenericContainer( "mcr.microsoft.com/mssql/server" ) .withExposedPorts(1433) .withEnv("ACCEPT_EULA", "Y") - .withEnv("MSSQL_SA_PASSWORD", "Str0Ng_p@ssW0rd!") + .withEnv("MSSQL_SA_PASSWORD", password) .withEnv("MSSQL_PID", "Developer") + .withWaitStrategy(Wait.forHealthCheck()) + .withHealthCheck({ + test: `/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${password}" -Q "SELECT 1" -b -o /dev/null`, + interval: new Duration(1000, TemporalUnit.MILLISECONDS), + timeout: new Duration(3, TemporalUnit.SECONDS), + retries: 20, + startPeriod: new Duration(100, TemporalUnit.MILLISECONDS), + }) .start() host = container.getContainerIpAddress() @@ -74,7 +87,7 @@ describe("datasource validators", () => { it("test valid connection string", async () => { const result = await validator({ user: "sa", - password: "Str0Ng_p@ssW0rd!", + password, server: host, port: port, database: "master", From 0418e90a37735d718002db28064a9db1761899e5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 12:49:08 +0200 Subject: [PATCH 27/76] Implement the check as part of the integration --- .../src/integrations/microsoftSqlServer.ts | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 3f14360d57..4391dbd07e 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -121,6 +121,15 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { } } + async testConnection() { + try { + await this.connect() + return true + } catch (e: any) { + return { error: e.message as string } + } + } + getBindingIdentifier(): string { return `@p${this.index++}` } @@ -310,18 +319,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { } } -async function validateConnection(config: MSSQLConfig) { - const integration = new SqlServerIntegration(config) - try { - await integration.connect() - return true - } catch (e: any) { - return { error: e.message as string } - } -} - export default { schema: SCHEMA, integration: SqlServerIntegration, - validateConnection, } From 2641a8c13552d6a032ec16edfb79b8b29b4e0ac8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 12:50:47 +0200 Subject: [PATCH 28/76] Clean code --- packages/server/src/integrations/microsoftSqlServer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 4391dbd07e..2650d777cc 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -22,7 +22,7 @@ import { MSSQLTablesResponse, MSSQLColumn } from "./base/types" const sqlServer = require("mssql") const DEFAULT_SCHEMA = "dbo" -export interface MSSQLConfig { +interface MSSQLConfig { user: string password: string server: string From d98bc11a20cb76d50953450d56874520f9812f53 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 12:51:19 +0200 Subject: [PATCH 29/76] Clean --- packages/server/src/integrations/microsoftSqlServer.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 2650d777cc..4c77bc2632 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -147,10 +147,6 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { } } - // async end(){ - // this.client!. - // } - async internalQuery( query: SqlQuery, operation: string | undefined = undefined From 0557c8efcc2b06f19cc0e49bbb54de53428381b0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 16:24:26 +0200 Subject: [PATCH 30/76] Split tests --- .../src/integrations/validators/mssql.spec.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 qa-core/src/integrations/validators/mssql.spec.ts diff --git a/qa-core/src/integrations/validators/mssql.spec.ts b/qa-core/src/integrations/validators/mssql.spec.ts new file mode 100644 index 0000000000..89af8e87c0 --- /dev/null +++ b/qa-core/src/integrations/validators/mssql.spec.ts @@ -0,0 +1,64 @@ +import { GenericContainer, Wait } from "testcontainers" +import { Duration, TemporalUnit } from "node-duration" + +import mssql from "../../../../packages/server/src/integrations/microsoftSqlServer" + +jest.unmock("mssql") + +describe("datasource validators", () => { + describe("mssql", () => { + let host: string, port: number + + const password = "Str0Ng_p@ssW0rd!" + + beforeAll(async () => { + const container = await new GenericContainer( + "mcr.microsoft.com/mssql/server" + ) + .withExposedPorts(1433) + .withEnv("ACCEPT_EULA", "Y") + .withEnv("MSSQL_SA_PASSWORD", password) + .withEnv("MSSQL_PID", "Developer") + .withWaitStrategy(Wait.forHealthCheck()) + .withHealthCheck({ + test: `/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${password}" -Q "SELECT 1" -b -o /dev/null`, + interval: new Duration(1000, TemporalUnit.MILLISECONDS), + timeout: new Duration(3, TemporalUnit.SECONDS), + retries: 20, + startPeriod: new Duration(100, TemporalUnit.MILLISECONDS), + }) + .start() + + host = container.getContainerIpAddress() + port = container.getMappedPort(1433) + }) + + it("test valid connection string", async () => { + const integration = new mssql.integration({ + user: "sa", + password, + server: host, + port: port, + database: "master", + schema: "dbo", + }) + const result = await integration.testConnection() + expect(result).toBe(true) + }) + + it("test invalid password", async () => { + const integration = new mssql.integration({ + user: "sa", + password: "wrong_pwd", + server: host, + port: port, + database: "master", + schema: "dbo", + }) + const result = await integration.testConnection() + expect(result).toEqual({ + error: "ConnectionError: Login failed for user 'sa'.", + }) + }) + }) +}) From b739e8da9cede51829150740641235b214df6063 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 16:25:12 +0200 Subject: [PATCH 31/76] Undo --- .../integrations/validators/postgres.spec.ts | 60 ------------------- 1 file changed, 60 deletions(-) diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index e36686fe64..5aa3250e2a 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -1,11 +1,7 @@ import { GenericContainer } from "testcontainers" -import { Duration, TemporalUnit } from "node-duration" - import postgres from "../../../../packages/server/src/integrations/postgres" -import { SourceName } from "@budibase/types" jest.unmock("pg") -jest.unmock("mssql") describe("datasource validators", () => { describe("postgres", () => { @@ -54,60 +50,4 @@ describe("datasource validators", () => { }) }) }) - - describe("mssql", () => { - const validator = integrations.getValidator[SourceName.SQL_SERVER]! - - let host: string, port: number - - const password = "Str0Ng_p@ssW0rd!" - - beforeAll(async () => { - const container = await new GenericContainer( - "mcr.microsoft.com/mssql/server" - ) - .withExposedPorts(1433) - .withEnv("ACCEPT_EULA", "Y") - .withEnv("MSSQL_SA_PASSWORD", password) - .withEnv("MSSQL_PID", "Developer") - .withWaitStrategy(Wait.forHealthCheck()) - .withHealthCheck({ - test: `/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${password}" -Q "SELECT 1" -b -o /dev/null`, - interval: new Duration(1000, TemporalUnit.MILLISECONDS), - timeout: new Duration(3, TemporalUnit.SECONDS), - retries: 20, - startPeriod: new Duration(100, TemporalUnit.MILLISECONDS), - }) - .start() - - host = container.getContainerIpAddress() - port = container.getMappedPort(1433) - }) - - it("test valid connection string", async () => { - const result = await validator({ - user: "sa", - password, - server: host, - port: port, - database: "master", - schema: "dbo", - }) - expect(result).toBe(true) - }) - - it("test invalid password", async () => { - const result = await validator({ - user: "sa", - password: "wrong_pwd", - server: host, - port: port, - database: "master", - schema: "dbo", - }) - expect(result).toEqual({ - error: "ConnectionError: Login failed for user 'sa'.", - }) - }) - }) }) From e8fa690566c9b4123d86d27ed334e1a0509eac7b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 10:07:59 +0200 Subject: [PATCH 32/76] Add mongo checks --- packages/server/src/integrations/mongodb.ts | 10 +++ .../src/integrations/validators/mongo.spec.ts | 88 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 qa-core/src/integrations/validators/mongo.spec.ts diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 38b3891fe4..77dc2ad7b2 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -631,8 +631,18 @@ class MongoIntegration implements IntegrationBase { } } } +async function validateConnection(config: MongoDBConfig) { + const integration = new MongoIntegration(config) + try { + await integration.connect() + return true + } catch (e: any) { + return { error: e.message as string } + } +} export default { schema: SCHEMA, integration: MongoIntegration, + validateConnection, } diff --git a/qa-core/src/integrations/validators/mongo.spec.ts b/qa-core/src/integrations/validators/mongo.spec.ts new file mode 100644 index 0000000000..1c4729513e --- /dev/null +++ b/qa-core/src/integrations/validators/mongo.spec.ts @@ -0,0 +1,88 @@ +import { GenericContainer } from "testcontainers" +import mongodb from "../../../../packages/server/src/integrations/mongodb" + +jest.unmock("mongodb") + +describe("datasource validators", () => { + describe("mongo", () => { + const validator = integrations.getValidator[SourceName.MONGODB] + + let connectionSettings: { + user: string + password: string + host: string + port: number + } + + function getConnectionString( + settings: Partial = {} + ) { + const { user, password, host, port } = { + ...connectionSettings, + ...settings, + } + return `mongodb://${user}:${password}@${host}:${port}` + } + + beforeAll(async () => { + const user = generator.name() + const password = generator.hash() + const container = await new GenericContainer("mongo") + .withExposedPorts(27017) + .withEnv("MONGO_INITDB_ROOT_USERNAME", user) + .withEnv("MONGO_INITDB_ROOT_PASSWORD", password) + .start() + + connectionSettings = { + user, + password, + host: container.getContainerIpAddress(), + port: container.getMappedPort(27017), + } + }) + + it("test valid connection string", async () => { + const result = await validator({ + connectionString: getConnectionString(), + db: "", + tlsCertificateFile: "", + tlsCertificateKeyFile: "", + tlsCAFile: "", + }) + expect(result).toBe(true) + }) + + it("test invalid password", async () => { + const result = await validator({ + connectionString: getConnectionString({ password: "wrong" }), + db: "", + tlsCertificateFile: "", + tlsCertificateKeyFile: "", + tlsCAFile: "", + }) + expect(result).toEqual({ error: "Authentication failed." }) + }) + + it("test invalid username", async () => { + const result = await validator({ + connectionString: getConnectionString({ user: "wrong" }), + db: "", + tlsCertificateFile: "", + tlsCertificateKeyFile: "", + tlsCAFile: "", + }) + expect(result).toEqual({ error: "Authentication failed." }) + }) + + it("test invalid connection", async () => { + const result = await validator({ + connectionString: getConnectionString({ host: "http://nothinghere" }), + db: "", + tlsCertificateFile: "", + tlsCertificateKeyFile: "", + tlsCAFile: "", + }) + expect(result).toEqual({ error: "Error: getaddrinfo ENOTFOUND http" }) + }) + }) +}) From 2f9b07638140473e45141ddad3c6da25a3b60fc6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 12:59:59 +0200 Subject: [PATCH 33/76] Implement the check as part of the integration --- packages/server/src/integrations/mongodb.ts | 19 +++++++++---------- .../src/integrations/validators/mongo.spec.ts | 17 ++++++++++------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 77dc2ad7b2..b8e4089411 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -358,6 +358,15 @@ class MongoIntegration implements IntegrationBase { this.client = new MongoClient(config.connectionString, options) } + async testConnection() { + try { + await this.connect() + return true + } catch (e: any) { + return { error: e.message as string } + } + } + async connect() { return this.client.connect() } @@ -631,18 +640,8 @@ class MongoIntegration implements IntegrationBase { } } } -async function validateConnection(config: MongoDBConfig) { - const integration = new MongoIntegration(config) - try { - await integration.connect() - return true - } catch (e: any) { - return { error: e.message as string } - } -} export default { schema: SCHEMA, integration: MongoIntegration, - validateConnection, } diff --git a/qa-core/src/integrations/validators/mongo.spec.ts b/qa-core/src/integrations/validators/mongo.spec.ts index 1c4729513e..dd8e188f27 100644 --- a/qa-core/src/integrations/validators/mongo.spec.ts +++ b/qa-core/src/integrations/validators/mongo.spec.ts @@ -1,12 +1,11 @@ import { GenericContainer } from "testcontainers" -import mongodb from "../../../../packages/server/src/integrations/mongodb" +import mongo from "../../../../packages/server/src/integrations/mongodb" +import { generator } from "../../shared" jest.unmock("mongodb") describe("datasource validators", () => { describe("mongo", () => { - const validator = integrations.getValidator[SourceName.MONGODB] - let connectionSettings: { user: string password: string @@ -42,46 +41,50 @@ describe("datasource validators", () => { }) it("test valid connection string", async () => { - const result = await validator({ + const integration = new mongo.integration({ connectionString: getConnectionString(), db: "", tlsCertificateFile: "", tlsCertificateKeyFile: "", tlsCAFile: "", }) + const result = await integration.testConnection() expect(result).toBe(true) }) it("test invalid password", async () => { - const result = await validator({ + const integration = new mongo.integration({ connectionString: getConnectionString({ password: "wrong" }), db: "", tlsCertificateFile: "", tlsCertificateKeyFile: "", tlsCAFile: "", }) + const result = await integration.testConnection() expect(result).toEqual({ error: "Authentication failed." }) }) it("test invalid username", async () => { - const result = await validator({ + const integration = new mongo.integration({ connectionString: getConnectionString({ user: "wrong" }), db: "", tlsCertificateFile: "", tlsCertificateKeyFile: "", tlsCAFile: "", }) + const result = await integration.testConnection() expect(result).toEqual({ error: "Authentication failed." }) }) it("test invalid connection", async () => { - const result = await validator({ + const integration = new mongo.integration({ connectionString: getConnectionString({ host: "http://nothinghere" }), db: "", tlsCertificateFile: "", tlsCertificateKeyFile: "", tlsCAFile: "", }) + const result = await integration.testConnection() expect(result).toEqual({ error: "Error: getaddrinfo ENOTFOUND http" }) }) }) From b41438c854df001b680ca1666ad4a7f799fa2fc6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 13:01:49 +0200 Subject: [PATCH 34/76] Clean code --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 14345384f7..c57a98d246 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 14345384f7a6755d1e2de327104741e0f208f55d +Subproject commit c57a98d246a50a43905d8572a88c901ec598390c From 8b73b768953ad34434c812eeed24306c85417631 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 16:39:33 +0200 Subject: [PATCH 35/76] Undo refs --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index c57a98d246..14345384f7 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit c57a98d246a50a43905d8572a88c901ec598390c +Subproject commit 14345384f7a6755d1e2de327104741e0f208f55d From 901bff5399ea8c54877d0814db2cc12a229183fa Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 11:17:14 +0200 Subject: [PATCH 36/76] Validate arango --- packages/server/src/integrations/arangodb.ts | 21 ++++++-- .../integrations/validators/arango.spec.ts | 48 +++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 qa-core/src/integrations/validators/arango.spec.ts diff --git a/packages/server/src/integrations/arangodb.ts b/packages/server/src/integrations/arangodb.ts index e28940f36e..31bbffe060 100644 --- a/packages/server/src/integrations/arangodb.ts +++ b/packages/server/src/integrations/arangodb.ts @@ -5,9 +5,9 @@ import { IntegrationBase, } from "@budibase/types" -const { Database, aql } = require("arangojs") +import { Database, aql } from "arangojs" -interface ArangodbConfig { +export interface ArangodbConfig { url: string username: string password: string @@ -58,7 +58,7 @@ const SCHEMA: Integration = { class ArangoDBIntegration implements IntegrationBase { private config: ArangodbConfig - private client: any + private client constructor(config: ArangodbConfig) { const newConfig = { @@ -102,9 +102,24 @@ class ArangoDBIntegration implements IntegrationBase { this.client.close() } } + + async check() { + await this.client.get() + } +} + +async function validateConnection(config: ArangodbConfig) { + const integration = new ArangoDBIntegration(config) + try { + await integration.check() + return true + } catch (e: any) { + return { error: e.message as string } + } } export default { schema: SCHEMA, integration: ArangoDBIntegration, + validateConnection, } diff --git a/qa-core/src/integrations/validators/arango.spec.ts b/qa-core/src/integrations/validators/arango.spec.ts new file mode 100644 index 0000000000..b865df6147 --- /dev/null +++ b/qa-core/src/integrations/validators/arango.spec.ts @@ -0,0 +1,48 @@ +import { GenericContainer, Wait } from "testcontainers" +import arangodb from "../../../../packages/server/src/integrations/arangodb" +import { generator } from "../../shared" + +jest.unmock("arangojs") + +describe("datasource validators", () => { + describe("arangodb", () => { + const validator = integrations.getValidator[SourceName.ARANGODB] + + let connectionSettings: { + user: string + password: string + url: string + } + + beforeAll(async () => { + const user = "root" + const password = generator.hash() + const container = await new GenericContainer("arangodb") + .withExposedPorts(8529) + .withEnv("ARANGO_ROOT_PASSWORD", password) + .withWaitStrategy( + Wait.forLogMessage("is ready for business. Have fun!") + ) + .start() + + connectionSettings = { + user, + password, + url: `http://${container.getContainerIpAddress()}:${container.getMappedPort( + 8529 + )}`, + } + }) + + it("test valid connection string", async () => { + const result = await validator({ + url: connectionSettings.url, + username: connectionSettings.user, + password: connectionSettings.password, + databaseName: "", + collection: "", + }) + expect(result).toBe(true) + }) + }) +}) From b8d11fa351e67adb2622f9986d1305d05ec17585 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 11:23:51 +0200 Subject: [PATCH 37/76] Test bad connections --- .../integrations/validators/arango.spec.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/qa-core/src/integrations/validators/arango.spec.ts b/qa-core/src/integrations/validators/arango.spec.ts index b865df6147..716fc19087 100644 --- a/qa-core/src/integrations/validators/arango.spec.ts +++ b/qa-core/src/integrations/validators/arango.spec.ts @@ -44,5 +44,31 @@ describe("datasource validators", () => { }) expect(result).toBe(true) }) + + it("test wrong password", async () => { + const result = await validator({ + url: connectionSettings.url, + username: connectionSettings.user, + password: "wrong", + databaseName: "", + collection: "", + }) + expect(result).toEqual({ + error: "not authorized to execute this request", + }) + }) + + it("test wrong url", async () => { + const result = await validator({ + url: "http://not.here", + username: connectionSettings.user, + password: connectionSettings.password, + databaseName: "", + collection: "", + }) + expect(result).toEqual({ + error: "getaddrinfo ENOTFOUND not.here", + }) + }) }) }) From fad57db634e0675740c6e61cc7b368b92253984b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 13:07:45 +0200 Subject: [PATCH 38/76] Implement the check as part of the integration --- packages/server/src/integrations/arangodb.ts | 26 +++++++------------ .../integrations/validators/arango.spec.ts | 11 ++++---- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/packages/server/src/integrations/arangodb.ts b/packages/server/src/integrations/arangodb.ts index 31bbffe060..297a164383 100644 --- a/packages/server/src/integrations/arangodb.ts +++ b/packages/server/src/integrations/arangodb.ts @@ -7,7 +7,7 @@ import { import { Database, aql } from "arangojs" -export interface ArangodbConfig { +interface ArangodbConfig { url: string username: string password: string @@ -74,6 +74,15 @@ class ArangoDBIntegration implements IntegrationBase { this.client = new Database(newConfig) } + async testConnection() { + try { + await this.client.get() + return true + } catch (e: any) { + return { error: e.message as string } + } + } + async read(query: { sql: any }) { try { const result = await this.client.query(query.sql) @@ -102,24 +111,9 @@ class ArangoDBIntegration implements IntegrationBase { this.client.close() } } - - async check() { - await this.client.get() - } -} - -async function validateConnection(config: ArangodbConfig) { - const integration = new ArangoDBIntegration(config) - try { - await integration.check() - return true - } catch (e: any) { - return { error: e.message as string } - } } export default { schema: SCHEMA, integration: ArangoDBIntegration, - validateConnection, } diff --git a/qa-core/src/integrations/validators/arango.spec.ts b/qa-core/src/integrations/validators/arango.spec.ts index 716fc19087..51282de13f 100644 --- a/qa-core/src/integrations/validators/arango.spec.ts +++ b/qa-core/src/integrations/validators/arango.spec.ts @@ -6,8 +6,6 @@ jest.unmock("arangojs") describe("datasource validators", () => { describe("arangodb", () => { - const validator = integrations.getValidator[SourceName.ARANGODB] - let connectionSettings: { user: string password: string @@ -35,37 +33,40 @@ describe("datasource validators", () => { }) it("test valid connection string", async () => { - const result = await validator({ + const integration = new arangodb.integration({ url: connectionSettings.url, username: connectionSettings.user, password: connectionSettings.password, databaseName: "", collection: "", }) + const result = await integration.testConnection() expect(result).toBe(true) }) it("test wrong password", async () => { - const result = await validator({ + const integration = new arangodb.integration({ url: connectionSettings.url, username: connectionSettings.user, password: "wrong", databaseName: "", collection: "", }) + const result = await integration.testConnection() expect(result).toEqual({ error: "not authorized to execute this request", }) }) it("test wrong url", async () => { - const result = await validator({ + const integration = new arangodb.integration({ url: "http://not.here", username: connectionSettings.user, password: connectionSettings.password, databaseName: "", collection: "", }) + const result = await integration.testConnection() expect(result).toEqual({ error: "getaddrinfo ENOTFOUND not.here", }) From 058ac416eac3950292c27e02fa4eb348cecb05f7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 10:16:06 +0200 Subject: [PATCH 39/76] Test redis --- packages/server/src/integrations/redis.ts | 13 +++- .../datasources/validators/redis.spec.ts | 73 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/sdk/tests/datasources/validators/redis.spec.ts diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index 73ef2bb55c..fd3c93125e 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -86,7 +86,7 @@ const SCHEMA: Integration = { class RedisIntegration { private readonly config: RedisConfig - private client: any + private client constructor(config: RedisConfig) { this.config = config @@ -99,6 +99,17 @@ class RedisIntegration { }) } + async testConnection() { + try { + await this.client.ping() + return true + } catch (e: any) { + return { error: e.message as string } + } finally { + await this.disconnect() + } + } + async disconnect() { return this.client.quit() } diff --git a/packages/server/src/sdk/tests/datasources/validators/redis.spec.ts b/packages/server/src/sdk/tests/datasources/validators/redis.spec.ts new file mode 100644 index 0000000000..48ae74779f --- /dev/null +++ b/packages/server/src/sdk/tests/datasources/validators/redis.spec.ts @@ -0,0 +1,73 @@ +import { generator } from "@budibase/backend-core/tests" +import redis from "../../../../integrations/redis" +import { GenericContainer } from "testcontainers" + +jest.unmock("pg") + +describe("datasource validators", () => { + describe("redis", () => { + describe("unsecured", () => { + let host: string + let port: number + + beforeAll(async () => { + const container = await new GenericContainer("redis") + .withExposedPorts(6379) + .start() + + host = container.getContainerIpAddress() + port = container.getMappedPort(6379) + }) + + it("test valid connection", async () => { + const integration = new redis.integration({ + host, + port, + username: "", + }) + const result = await integration.testConnection() + expect(result).toBe(true) + }) + + it("test invalid connection even with wrong user/password", async () => { + const integration = new redis.integration({ + host, + port, + username: generator.name(), + password: generator.hash(), + }) + const result = await integration.testConnection() + expect(result).toEqual({ + error: + "WRONGPASS invalid username-password pair or user is disabled.", + }) + }) + }) + + describe("secured", () => { + let host: string + let port: number + + beforeAll(async () => { + const container = await new GenericContainer("redis") + .withExposedPorts(6379) + .withCmd(["redis-server", "--requirepass", "P@ssW0rd!"]) + .start() + + host = container.getContainerIpAddress() + port = container.getMappedPort(6379) + }) + + it("test valid connection", async () => { + const integration = new redis.integration({ + host, + port, + username: "", + password: "P@ssW0rd!", + }) + const result = await integration.testConnection() + expect(result).toBe(true) + }) + }) + }) +}) From 47d8701578ed96a937248e7cb0b9a0c28987bffe Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 16:53:32 +0200 Subject: [PATCH 40/76] Move files --- .../src/integrations}/validators/redis.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename {packages/server/src/sdk/tests/datasources => qa-core/src/integrations}/validators/redis.spec.ts (96%) diff --git a/packages/server/src/sdk/tests/datasources/validators/redis.spec.ts b/qa-core/src/integrations/validators/redis.spec.ts similarity index 96% rename from packages/server/src/sdk/tests/datasources/validators/redis.spec.ts rename to qa-core/src/integrations/validators/redis.spec.ts index 48ae74779f..d51a6d695a 100644 --- a/packages/server/src/sdk/tests/datasources/validators/redis.spec.ts +++ b/qa-core/src/integrations/validators/redis.spec.ts @@ -1,5 +1,5 @@ import { generator } from "@budibase/backend-core/tests" -import redis from "../../../../integrations/redis" +import redis from "../../../../packages/server/src/integrations/redis" import { GenericContainer } from "testcontainers" jest.unmock("pg") From 5cd3f676ac80feace8ec557afbd94a5076bc0930 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 16:53:55 +0200 Subject: [PATCH 41/76] Fix import --- qa-core/src/integrations/validators/redis.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa-core/src/integrations/validators/redis.spec.ts b/qa-core/src/integrations/validators/redis.spec.ts index d51a6d695a..37ae648172 100644 --- a/qa-core/src/integrations/validators/redis.spec.ts +++ b/qa-core/src/integrations/validators/redis.spec.ts @@ -1,6 +1,6 @@ -import { generator } from "@budibase/backend-core/tests" import redis from "../../../../packages/server/src/integrations/redis" import { GenericContainer } from "testcontainers" +import { generator } from "../../shared" jest.unmock("pg") From 1c39946103e4b0e2eacce74582789a43cf378b4f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 17:01:25 +0200 Subject: [PATCH 42/76] Clean code --- qa-core/src/integrations/validators/redis.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/qa-core/src/integrations/validators/redis.spec.ts b/qa-core/src/integrations/validators/redis.spec.ts index 37ae648172..fd35f5d23e 100644 --- a/qa-core/src/integrations/validators/redis.spec.ts +++ b/qa-core/src/integrations/validators/redis.spec.ts @@ -2,8 +2,6 @@ import redis from "../../../../packages/server/src/integrations/redis" import { GenericContainer } from "testcontainers" import { generator } from "../../shared" -jest.unmock("pg") - describe("datasource validators", () => { describe("redis", () => { describe("unsecured", () => { From 1e505791c05d882a3f9ffdda71113259f28364b4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 17:37:28 +0200 Subject: [PATCH 43/76] Implement s3 checks --- packages/server/src/integrations/s3.ts | 15 ++++++-- .../src/integrations/validators/s3.spec.ts | 37 +++++++++++++++++++ qa-core/src/shared/generator.ts | 2 +- 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 qa-core/src/integrations/validators/s3.spec.ts diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index ad3bb09109..96b745ae08 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -5,8 +5,8 @@ import { DatasourceFieldType, } from "@budibase/types" -const AWS = require("aws-sdk") -const csv = require("csvtojson") +import AWS from "aws-sdk" +import csv from "csvtojson" interface S3Config { region: string @@ -152,7 +152,7 @@ const SCHEMA: Integration = { class S3Integration implements IntegrationBase { private readonly config: S3Config - private client: any + private client constructor(config: S3Config) { this.config = config @@ -165,6 +165,15 @@ class S3Integration implements IntegrationBase { this.client = new AWS.S3(this.config) } + async testConnection() { + try { + const buckets = await this.client.listBuckets().promise() + return true + } catch (e: any) { + return { error: e.message as string } + } + } + async create(query: { bucket: string location: string diff --git a/qa-core/src/integrations/validators/s3.spec.ts b/qa-core/src/integrations/validators/s3.spec.ts new file mode 100644 index 0000000000..bd76bea26a --- /dev/null +++ b/qa-core/src/integrations/validators/s3.spec.ts @@ -0,0 +1,37 @@ +import s3 from "../../../../packages/server/src/integrations/s3" +import { GenericContainer } from "testcontainers" +import { generator } from "../../shared" + +jest.unmock("aws-sdk") + +describe("datasource validators", () => { + describe("s3", () => { + let host: string + let port: number + + beforeAll(async () => { + const container = await new GenericContainer("localstack/localstack") + .withExposedPorts(4566) + .withEnv("SERVICES", "s3") + .withEnv("DEFAULT_REGION", "eu-west-1") + .withEnv("AWS_ACCESS_KEY_ID", "testkey") + .withEnv("AWS_SECRET_ACCESS_KEY", "testsecret") + .start() + + host = container.getContainerIpAddress() + port = container.getMappedPort(4566) + }) + + it("test valid connection", async () => { + const integration = new s3.integration({ + region: "eu-west-1", + accessKeyId: "testkey", + secretAccessKey: "testsecret", + s3ForcePathStyle: false, + endpoint: `http://${host}:${port}`, + }) + const result = await integration.testConnection() + expect(result).toBe(true) + }) + }) +}) diff --git a/qa-core/src/shared/generator.ts b/qa-core/src/shared/generator.ts index c9395f7e47..1789fc0f75 100644 --- a/qa-core/src/shared/generator.ts +++ b/qa-core/src/shared/generator.ts @@ -1,3 +1,3 @@ -const Chance = require("chance") +import Chance from "chance" export default new Chance() From d893fdf6cff661c1a0a6610a62783debba6a2791 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 17:40:40 +0200 Subject: [PATCH 44/76] Test wrong endpoint --- qa-core/src/integrations/validators/s3.spec.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/qa-core/src/integrations/validators/s3.spec.ts b/qa-core/src/integrations/validators/s3.spec.ts index bd76bea26a..57c2377c76 100644 --- a/qa-core/src/integrations/validators/s3.spec.ts +++ b/qa-core/src/integrations/validators/s3.spec.ts @@ -33,5 +33,20 @@ describe("datasource validators", () => { const result = await integration.testConnection() expect(result).toBe(true) }) + + it("test wrong endpoint", async () => { + const integration = new s3.integration({ + region: "eu-west-2", + accessKeyId: "testkey", + secretAccessKey: "testsecret", + s3ForcePathStyle: false, + endpoint: `http://wrong:123`, + }) + const result = await integration.testConnection() + expect(result).toEqual({ + error: + "Inaccessible host: `wrong' at port `undefined'. This service may not be available in the `eu-west-2' region.", + }) + }) }) }) From 79b30942f6666b9cbbe6054cbb4b83dd05e4c77d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 15 May 2023 17:52:10 +0200 Subject: [PATCH 45/76] Clean code --- qa-core/src/integrations/validators/s3.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/qa-core/src/integrations/validators/s3.spec.ts b/qa-core/src/integrations/validators/s3.spec.ts index 57c2377c76..6ed5c44449 100644 --- a/qa-core/src/integrations/validators/s3.spec.ts +++ b/qa-core/src/integrations/validators/s3.spec.ts @@ -1,6 +1,5 @@ import s3 from "../../../../packages/server/src/integrations/s3" import { GenericContainer } from "testcontainers" -import { generator } from "../../shared" jest.unmock("aws-sdk") From f2f0c2e708bad1bde75c565f298f636a599cb6f6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 18:28:25 +0200 Subject: [PATCH 46/76] Typing --- packages/server/src/integrations/dynamodb.ts | 23 +++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index 28c1c7b52b..d748622130 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -7,6 +7,7 @@ import { import AWS from "aws-sdk" import { AWS_REGION } from "../db/dynamoClient" +import { DocumentClient } from "aws-sdk/clients/dynamodb" interface DynamoDBConfig { region: string @@ -128,7 +129,7 @@ const SCHEMA: Integration = { class DynamoDBIntegration implements IntegrationBase { private config: DynamoDBConfig - private client: any + private client constructor(config: DynamoDBConfig) { this.config = config @@ -148,7 +149,10 @@ class DynamoDBIntegration implements IntegrationBase { this.client = new AWS.DynamoDB.DocumentClient(this.config) } - async create(query: { table: string; json: object }) { + async create(query: { + table: string + json: Omit + }) { const params = { TableName: query.table, ...query.json, @@ -189,7 +193,10 @@ class DynamoDBIntegration implements IntegrationBase { return new AWS.DynamoDB(this.config).describeTable(params).promise() } - async get(query: { table: string; json: object }) { + async get(query: { + table: string + json: Omit + }) { const params = { TableName: query.table, ...query.json, @@ -197,7 +204,10 @@ class DynamoDBIntegration implements IntegrationBase { return this.client.get(params).promise() } - async update(query: { table: string; json: object }) { + async update(query: { + table: string + json: Omit + }) { const params = { TableName: query.table, ...query.json, @@ -205,7 +215,10 @@ class DynamoDBIntegration implements IntegrationBase { return this.client.update(params).promise() } - async delete(query: { table: string; json: object }) { + async delete(query: { + table: string + json: Omit + }) { const params = { TableName: query.table, ...query.json, From 3933a3881bdbcff817eca649de5c3204fd10b439 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 May 2023 18:34:56 +0200 Subject: [PATCH 47/76] Test dynamodb connection --- packages/server/src/db/dynamoClient.ts | 2 +- packages/server/src/integrations/dynamodb.ts | 9 +++ .../integrations/validators/dynamodb.spec.ts | 62 +++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 qa-core/src/integrations/validators/dynamodb.spec.ts diff --git a/packages/server/src/db/dynamoClient.ts b/packages/server/src/db/dynamoClient.ts index cb045b7d6f..597be2b21e 100644 --- a/packages/server/src/db/dynamoClient.ts +++ b/packages/server/src/db/dynamoClient.ts @@ -140,7 +140,7 @@ export function init(endpoint: string) { docClient = new AWS.DynamoDB.DocumentClient(docClientParams) } -if (!env.isProd()) { +if (!env.isProd() && !env.isJest()) { env._set("AWS_ACCESS_KEY_ID", "KEY_ID") env._set("AWS_SECRET_ACCESS_KEY", "SECRET_KEY") init("http://localhost:8333") diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index d748622130..ca004ee11b 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -149,6 +149,15 @@ class DynamoDBIntegration implements IntegrationBase { this.client = new AWS.DynamoDB.DocumentClient(this.config) } + async testConnection() { + try { + const scanRes = await new AWS.DynamoDB(this.config).listTables().promise() + return !!scanRes.$response + } catch (e: any) { + return { error: e.message as string } + } + } + async create(query: { table: string json: Omit diff --git a/qa-core/src/integrations/validators/dynamodb.spec.ts b/qa-core/src/integrations/validators/dynamodb.spec.ts new file mode 100644 index 0000000000..53377ff06b --- /dev/null +++ b/qa-core/src/integrations/validators/dynamodb.spec.ts @@ -0,0 +1,62 @@ +import { GenericContainer } from "testcontainers" +import { env } from "@budibase/backend-core" + +import dynamodb from "../../../../packages/server/src/integrations/dynamodb" +import { generator } from "../../shared" + +jest.unmock("aws-sdk") + +describe("datasource validators", () => { + describe("dynamodb", () => { + let connectionSettings: { + user: string + password: string + url: string + } + + beforeAll(async () => { + const user = "root" + const password = generator.hash() + const container = await new GenericContainer("amazon/dynamodb-local") + .withExposedPorts(8000) + .start() + + connectionSettings = { + user, + password, + url: `http://${container.getContainerIpAddress()}:${container.getMappedPort( + 8000 + )}`, + } + env._set("AWS_ACCESS_KEY_ID", "mocked_key") + env._set("AWS_SECRET_ACCESS_KEY", "mocked_secret") + }) + + it("test valid connection string", async () => { + const integration = new dynamodb.integration({ + endpoint: connectionSettings.url, + region: "", + accessKeyId: "", + secretAccessKey: "", + }) + + const result = await integration.testConnection() + expect(result).toBe(true) + }) + + it("test wrong endpoint", async () => { + const integration = new dynamodb.integration({ + endpoint: "http://wrong.url:2880", + region: "", + accessKeyId: "", + secretAccessKey: "", + }) + + const result = await integration.testConnection() + expect(result).toEqual({ + error: + "Inaccessible host: `wrong.url' at port `undefined'. This service may not be available in the `eu-west-1' region.", + }) + }) + }) +}) From bd74f8eef74d9ac4c90d33e8194fe0bb40b7f9e8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 May 2023 17:08:23 +0100 Subject: [PATCH 48/76] Updating testConnection usage. --- .../server/src/api/controllers/datasource.ts | 16 ++++++++++++---- packages/types/src/api/web/app/datasource.ts | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index e6866d528c..363cde2e6f 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -128,12 +128,20 @@ export async function verify( ) { const datasource = ctx.request.body.datasource const connector = (await getConnector(datasource)) as IntegrationBase - if (!connector.connection) { + if (!connector.testConnection) { ctx.throw(400, "Connection information verification not supported") } - const connectionInfo = await connector.connection() - ctx.body = { - connected: connectionInfo.connected, + const response = await connector.testConnection() + + if (typeof response === "boolean") { + ctx.body = { + connected: response, + } + } else { + ctx.body = { + connected: false, + error: response.error + } } } diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index 36e081d9f6..983fd45b92 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -20,6 +20,7 @@ export interface VerifyDatasourceRequest { export interface VerifyDatasourceResponse { connected: boolean + error?: string } export interface UpdateDatasourceRequest extends Datasource { From 6b7269b9542bd4e5ac52d17d429527855c077a8e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 May 2023 17:36:16 +0100 Subject: [PATCH 49/76] Refactoring response from integrations to a simple type. --- packages/server/src/api/controllers/datasource.ts | 2 +- packages/server/src/integrations/airtable.ts | 2 +- packages/server/src/integrations/arangodb.ts | 9 +++++++-- packages/server/src/integrations/couchdb.ts | 9 +++++++-- packages/server/src/integrations/dynamodb.ts | 9 +++++++-- packages/server/src/integrations/elasticsearch.ts | 2 +- packages/server/src/integrations/firebase.ts | 2 +- packages/server/src/integrations/googlesheets.ts | 2 +- .../server/src/integrations/microsoftSqlServer.ts | 9 +++++++-- packages/server/src/integrations/mongodb.ts | 9 +++++++-- packages/server/src/integrations/mysql.ts | 9 +++++++-- packages/server/src/integrations/oracle.ts | 9 ++------- packages/server/src/integrations/postgres.ts | 9 +++++++-- packages/server/src/integrations/redis.ts | 9 +++++++-- packages/server/src/integrations/s3.ts | 15 ++++++++++----- packages/server/src/integrations/snowflake.ts | 2 +- packages/types/src/sdk/datasources.ts | 12 ++++++------ 17 files changed, 80 insertions(+), 40 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 363cde2e6f..3534fc3076 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -140,7 +140,7 @@ export async function verify( } else { ctx.body = { connected: false, - error: response.error + error: response.error, } } } diff --git a/packages/server/src/integrations/airtable.ts b/packages/server/src/integrations/airtable.ts index cb13de4f10..d606cd570b 100644 --- a/packages/server/src/integrations/airtable.ts +++ b/packages/server/src/integrations/airtable.ts @@ -19,7 +19,7 @@ const SCHEMA: Integration = { "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.", friendlyName: "Airtable", type: "Spreadsheet", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: [], datasource: { apiKey: { type: DatasourceFieldType.PASSWORD, diff --git a/packages/server/src/integrations/arangodb.ts b/packages/server/src/integrations/arangodb.ts index 544dc12377..b486748a68 100644 --- a/packages/server/src/integrations/arangodb.ts +++ b/packages/server/src/integrations/arangodb.ts @@ -4,6 +4,7 @@ import { QueryType, IntegrationBase, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import { Database, aql } from "arangojs" @@ -77,12 +78,16 @@ class ArangoDBIntegration implements IntegrationBase { } async testConnection() { + const response: ConnectionInfo = { + connected: false, + } try { await this.client.get() - return true + response.connected = true } catch (e: any) { - return { error: e.message as string } + response.error = e.message as string } + return response } async read(query: { sql: any }) { diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index 663081edd3..4ccbd5456d 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -1,4 +1,5 @@ import { + ConnectionInfo, DatasourceFeature, DatasourceFieldType, Document, @@ -70,12 +71,16 @@ class CouchDBIntegration implements IntegrationBase { } async testConnection() { + const response: ConnectionInfo = { + connected: false, + } try { const result = await this.query("exists", "validation error", {}) - return result === true + response.connected = result === true } catch (e: any) { - return { error: e.message as string } + response.error = e.message as string } + return response } async query( diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index 9f63edb465..28b42c7a54 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -4,6 +4,7 @@ import { QueryType, IntegrationBase, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import AWS from "aws-sdk" @@ -152,12 +153,16 @@ class DynamoDBIntegration implements IntegrationBase { } async testConnection() { + const response: ConnectionInfo = { + connected: false, + } try { const scanRes = await new AWS.DynamoDB(this.config).listTables().promise() - return !!scanRes.$response + response.connected = !!scanRes.$response } catch (e: any) { - return { error: e.message as string } + response.error = e.message as string } + return response } async create(query: { diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index 475ddc0bd1..621e81f367 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -21,7 +21,7 @@ const SCHEMA: Integration = { "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.", friendlyName: "ElasticSearch", type: "Non-relational", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: [], datasource: { url: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/firebase.ts b/packages/server/src/integrations/firebase.ts index 0646c98eba..e8b3c8d92c 100644 --- a/packages/server/src/integrations/firebase.ts +++ b/packages/server/src/integrations/firebase.ts @@ -19,7 +19,7 @@ const SCHEMA: Integration = { type: "Non-relational", description: "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: [], datasource: { email: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index fef4689fdf..0b2be54fb3 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -65,7 +65,7 @@ const SCHEMA: Integration = { "Create and collaborate on online spreadsheets in real-time and from any device. ", friendlyName: "Google Sheets", type: "Spreadsheet", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: [], datasource: { spreadsheetId: { display: "Google Sheet URL", diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index eb65be43da..47f36f60e9 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -9,6 +9,7 @@ import { SqlQuery, DatasourcePlus, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import { getSqlQuery, @@ -124,12 +125,16 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { } async testConnection() { + const response: ConnectionInfo = { + connected: false, + } try { await this.connect() - return true + response.connected = true } catch (e: any) { - return { error: e.message as string } + response.error = e.message as string } + return response } getBindingIdentifier(): string { diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 9e3d7344b4..ee7302c501 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -4,6 +4,7 @@ import { QueryType, IntegrationBase, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import { MongoClient, @@ -361,12 +362,16 @@ class MongoIntegration implements IntegrationBase { } async testConnection() { + const response: ConnectionInfo = { + connected: false, + } try { await this.connect() - return true + response.connected = true } catch (e: any) { - return { error: e.message as string } + response.error = e.message as string } + return response } async connect() { diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 1f3ea4ea1c..eb721a6e0f 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -8,6 +8,7 @@ import { TableSchema, DatasourcePlus, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import { getSqlQuery, @@ -155,15 +156,19 @@ class MySQLIntegration extends Sql implements DatasourcePlus { } async testConnection() { + const response: ConnectionInfo = { + connected: false, + } try { const [result] = await this.internalQuery( { sql: "SELECT 1+1 AS checkRes" }, { connect: true } ) - return result?.checkRes == 2 + response.connected = result?.checkRes == 2 } catch (e: any) { - return { error: e.message as string } + response.error = e.message as string } + return response } getBindingIdentifier(): string { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index bbe2189bdc..82d8382a31 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -25,12 +25,7 @@ import { ExecuteOptions, Result, } from "oracledb" -import { - OracleTable, - OracleColumn, - OracleColumnsResponse, - OracleConstraint, -} from "./base/types" +import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types" let oracledb: any try { oracledb = require("oracledb") @@ -54,7 +49,7 @@ const SCHEMA: Integration = { type: "Relational", description: "Oracle Database is an object-relational database management system developed by Oracle Corporation", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: [], datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 2ffa04c2e4..bf77ec08c6 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -7,6 +7,7 @@ import { Table, DatasourcePlus, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import { getSqlQuery, @@ -153,14 +154,18 @@ class PostgresIntegration extends Sql implements DatasourcePlus { } async testConnection() { + const response: ConnectionInfo = { + connected: false, + } try { await this.openConnection() - return true + response.connected = true } catch (e: any) { - return { error: e.message as string } + response.error = e.message as string } finally { await this.closeConnection() } + return response } getBindingIdentifier(): string { diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index 1d1d705026..d71f66edc1 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -1,4 +1,5 @@ import { + ConnectionInfo, DatasourceFeature, DatasourceFieldType, Integration, @@ -107,14 +108,18 @@ class RedisIntegration { } async testConnection() { + const response: ConnectionInfo = { + connected: false, + } try { await this.client.ping() - return true + response.connected = true } catch (e: any) { - return { error: e.message as string } + response.error = e.message as string } finally { await this.disconnect() } + return response } async disconnect() { diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index d09c7f890d..0f9848ed59 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -4,6 +4,7 @@ import { IntegrationBase, DatasourceFieldType, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import AWS from "aws-sdk" @@ -168,12 +169,16 @@ class S3Integration implements IntegrationBase { } async testConnection() { - try { - const buckets = await this.client.listBuckets().promise() - return true - } catch (e: any) { - return { error: e.message as string } + const response: ConnectionInfo = { + connected: false, } + try { + await this.client.listBuckets().promise() + response.connected = true + } catch (e: any) { + response.error = e.message as string + } + return response } async create(query: { diff --git a/packages/server/src/integrations/snowflake.ts b/packages/server/src/integrations/snowflake.ts index 877405d447..46a40b8b03 100644 --- a/packages/server/src/integrations/snowflake.ts +++ b/packages/server/src/integrations/snowflake.ts @@ -21,7 +21,7 @@ const SCHEMA: Integration = { "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.", friendlyName: "Snowflake", type: "Relational", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: [], datasource: { account: { type: "string", diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 000abb7590..9df9670877 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -128,17 +128,17 @@ export interface Integration { extra?: ExtraQueryConfig } +export type ConnectionInfo = { + connected: boolean + error?: string +} + export interface IntegrationBase { create?(query: any): Promise read?(query: any): Promise update?(query: any): Promise delete?(query: any): Promise - testConnection?(): Promise< - | boolean - | { - error: string - } - > + testConnection?(): Promise } export interface DatasourcePlus extends IntegrationBase { From b5890b788f842726d63cb3678b5cf721e294f19d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 09:50:23 +0200 Subject: [PATCH 50/76] Append auth on verification --- packages/server/src/api/controllers/datasource.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 3534fc3076..65c910bcc0 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -25,6 +25,7 @@ import { DatasourcePlus, } from "@budibase/types" import sdk from "../../sdk" +import _ from "lodash" function getErrorTables(errors: any, errorType: string) { return Object.entries(errors) @@ -126,8 +127,13 @@ export async function fetch(ctx: UserCtx) { export async function verify( ctx: UserCtx ) { - const datasource = ctx.request.body.datasource - const connector = (await getConnector(datasource)) as IntegrationBase + const { datasource } = ctx.request.body + + const { config: { auth } = {} } = await sdk.datasources.get(datasource._id!) + + const connector = await getConnector( + _.merge({ config: { auth } }, datasource) + ) if (!connector.testConnection) { ctx.throw(400, "Connection information verification not supported") } From 77102cea120ec32ebee1909b9d1d8cb88da14861 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 09:50:40 +0200 Subject: [PATCH 51/76] Typing --- packages/types/src/documents/app/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index 8dfdfe6d0f..89e7422cb4 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -9,7 +9,7 @@ export interface Datasource extends Document { // the config is defined by the schema config?: { [key: string]: string | number | boolean | any[] - } + } & { auth?: object } plus?: boolean entities?: { [key: string]: Table From cd93d327a5b1368abcb02a51ccf9325cb5876af9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 10:02:11 +0200 Subject: [PATCH 52/76] Test google sheets --- packages/server/src/integrations/googlesheets.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 0b2be54fb3..6cad048c83 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -141,6 +141,16 @@ class GoogleSheetsIntegration implements DatasourcePlus { this.client = new GoogleSpreadsheet(spreadsheetId) } + async testConnection() { + try { + await this.connect() + await this.client.loadInfo() + return true + } catch (e: any) { + return { error: e.message as string } + } + } + getBindingIdentifier() { return "" } From a676e42b1f356dc3fd563dd12a62ad20abf7eb40 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 10:44:41 +0200 Subject: [PATCH 53/76] Handle config --- packages/server/src/api/controllers/datasource.ts | 6 ++---- packages/types/src/documents/app/datasource.ts | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 65c910bcc0..686515b293 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -129,11 +129,9 @@ export async function verify( ) { const { datasource } = ctx.request.body - const { config: { auth } = {} } = await sdk.datasources.get(datasource._id!) + const { config } = await sdk.datasources.get(datasource._id!) - const connector = await getConnector( - _.merge({ config: { auth } }, datasource) - ) + const connector = await getConnector(_.merge(datasource, { config })) if (!connector.testConnection) { ctx.throw(400, "Connection information verification not supported") } diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index 89e7422cb4..8dfdfe6d0f 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -9,7 +9,7 @@ export interface Datasource extends Document { // the config is defined by the schema config?: { [key: string]: string | number | boolean | any[] - } & { auth?: object } + } plus?: boolean entities?: { [key: string]: Table From 5e7d839cb1c2fc8b35e44f268df2692b4615ebe6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 10:50:27 +0200 Subject: [PATCH 54/76] Use existing merge --- packages/server/src/api/controllers/datasource.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 686515b293..a87c8c68f6 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -25,7 +25,6 @@ import { DatasourcePlus, } from "@budibase/types" import sdk from "../../sdk" -import _ from "lodash" function getErrorTables(errors: any, errorType: string) { return Object.entries(errors) @@ -128,10 +127,14 @@ export async function verify( ctx: UserCtx ) { const { datasource } = ctx.request.body + const existingDatasource = await sdk.datasources.get(datasource._id!) - const { config } = await sdk.datasources.get(datasource._id!) + const enrichedDatasource = sdk.datasources.mergeConfigs( + datasource, + existingDatasource + ) - const connector = await getConnector(_.merge(datasource, { config })) + const connector = await getConnector(enrichedDatasource) if (!connector.testConnection) { ctx.throw(400, "Connection information verification not supported") } From a21039d5b4be12455f676df304e41ab0915f3b71 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 10:56:46 +0200 Subject: [PATCH 55/76] Fix auth checks when specified auth --- packages/server/src/sdk/app/datasources/datasources.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index dee0a221aa..c886e6a15f 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -13,6 +13,7 @@ import { import { cloneDeep } from "lodash/fp" import { getEnvironmentVariables } from "../../utils" import { getDefinitions, getDefinition } from "../../../integrations" +import _ from "lodash" const ENV_VAR_PREFIX = "env." @@ -147,6 +148,11 @@ export function mergeConfigs(update: Datasource, old: Datasource) { } } } + + if (old.config?.auth) { + update.config = _.merge(old.config, update.config) + } + // update back to actual passwords for everything else for (let [key, value] of Object.entries(update.config)) { if (value !== PASSWORD_REPLACEMENT) { From 911a0720694a54546e40502a2a4df2007f649f77 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 11:44:58 +0200 Subject: [PATCH 56/76] Update types --- packages/server/src/integrations/googlesheets.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 6cad048c83..a8fd966782 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -1,5 +1,5 @@ import { - DatasourceFeature, + ConnectionInfo, DatasourceFieldType, DatasourcePlus, FieldType, @@ -141,13 +141,16 @@ class GoogleSheetsIntegration implements DatasourcePlus { this.client = new GoogleSpreadsheet(spreadsheetId) } - async testConnection() { + async testConnection(): Promise { try { await this.connect() await this.client.loadInfo() - return true + return { connected: true } } catch (e: any) { - return { error: e.message as string } + return { + connected: false, + error: e.message as string, + } } } From d6473aab87b80be32aaae9c9810358e737d31355 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 09:50:23 +0200 Subject: [PATCH 57/76] Append auth on verification --- packages/server/src/api/controllers/datasource.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index a87c8c68f6..a6609abff2 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -25,6 +25,7 @@ import { DatasourcePlus, } from "@budibase/types" import sdk from "../../sdk" +import _ from "lodash" function getErrorTables(errors: any, errorType: string) { return Object.entries(errors) From b3b962534f692a4bab7cf8d460323f70a79eed95 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 11:40:29 +0200 Subject: [PATCH 58/76] Test snowflake connection --- packages/server/src/integrations/snowflake.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/server/src/integrations/snowflake.ts b/packages/server/src/integrations/snowflake.ts index 46a40b8b03..7b35968aaf 100644 --- a/packages/server/src/integrations/snowflake.ts +++ b/packages/server/src/integrations/snowflake.ts @@ -71,6 +71,15 @@ class SnowflakeIntegration { this.client = new Snowflake(config) } + async testConnection() { + try { + await this.client.connect() + return true + } catch (e: any) { + return { error: e.message as string } + } + } + async internalQuery(query: SqlQuery) { await this.client.connect() try { From 062127b1f10b4bf74ae6158d0922035ef441a180 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 11:48:40 +0200 Subject: [PATCH 59/76] Fix types --- packages/server/src/integrations/snowflake.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/snowflake.ts b/packages/server/src/integrations/snowflake.ts index 7b35968aaf..dc5fb66c55 100644 --- a/packages/server/src/integrations/snowflake.ts +++ b/packages/server/src/integrations/snowflake.ts @@ -1,4 +1,5 @@ import { + ConnectionInfo, DatasourceFeature, Integration, QueryType, @@ -71,12 +72,15 @@ class SnowflakeIntegration { this.client = new Snowflake(config) } - async testConnection() { + async testConnection(): Promise { try { await this.client.connect() - return true + return { connected: true } } catch (e: any) { - return { error: e.message as string } + return { + connected: false, + error: e.message as string, + } } } From 2176ac8da746b3b9512f73e0423831d8f6389118 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 11:49:51 +0200 Subject: [PATCH 60/76] Clean code --- packages/server/src/api/controllers/datasource.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index a6609abff2..a87c8c68f6 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -25,7 +25,6 @@ import { DatasourcePlus, } from "@budibase/types" import sdk from "../../sdk" -import _ from "lodash" function getErrorTables(errors: any, errorType: string) { return Object.entries(errors) From be6c398f53e5a8d745597e6e4de4174d9ed66a50 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 13:23:35 +0200 Subject: [PATCH 61/76] Remove bool ref --- packages/server/src/api/controllers/datasource.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 3534fc3076..283a2726d5 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -133,15 +133,9 @@ export async function verify( } const response = await connector.testConnection() - if (typeof response === "boolean") { - ctx.body = { - connected: response, - } - } else { - ctx.body = { - connected: false, - error: response.error, - } + ctx.body = { + connected: response.connected, + error: response.error, } } From 419e2de602571a5e864032e4b9490e03e25781ac Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 13:21:05 +0200 Subject: [PATCH 62/76] Test airtable connection --- packages/server/src/integrations/airtable.ts | 29 ++++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/airtable.ts b/packages/server/src/integrations/airtable.ts index d606cd570b..e513c8cfc1 100644 --- a/packages/server/src/integrations/airtable.ts +++ b/packages/server/src/integrations/airtable.ts @@ -1,12 +1,12 @@ import { - DatasourceFeature, + ConnectionInfo, DatasourceFieldType, Integration, IntegrationBase, QueryType, } from "@budibase/types" -const Airtable = require("airtable") +import Airtable from "airtable" interface AirtableConfig { apiKey: string @@ -83,13 +83,36 @@ const SCHEMA: Integration = { class AirtableIntegration implements IntegrationBase { private config: AirtableConfig - private client: any + private client constructor(config: AirtableConfig) { this.config = config this.client = new Airtable(config).base(config.base) } + async testConnection(): Promise { + const mockTable = Date.now().toString() + try { + await this.client.makeRequest({ + path: `/${mockTable}`, + }) + + return { connected: true } + } catch (e: any) { + if ( + e.message === + `Could not find table ${mockTable} in application ${this.config.base}` + ) { + return { connected: true } + } + + return { + connected: false, + error: e.message as string, + } + } + } + async create(query: { table: any; json: any }) { const { table, json } = query From 87497fee7373776dfe6ca83fddb3099868157761 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 13:22:02 +0200 Subject: [PATCH 63/76] Comments --- packages/server/src/integrations/airtable.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/integrations/airtable.ts b/packages/server/src/integrations/airtable.ts index e513c8cfc1..54a91132e3 100644 --- a/packages/server/src/integrations/airtable.ts +++ b/packages/server/src/integrations/airtable.ts @@ -103,6 +103,7 @@ class AirtableIntegration implements IntegrationBase { e.message === `Could not find table ${mockTable} in application ${this.config.base}` ) { + // The request managed to check the application, so the credentials are valid return { connected: true } } From ce357ecac84f0799094768e0f5f5890d178b5d14 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 13:37:30 +0200 Subject: [PATCH 64/76] Setup CONNECTION_CHECKING feature --- packages/server/src/integrations/airtable.ts | 3 ++- packages/server/src/integrations/googlesheets.ts | 3 ++- packages/server/src/integrations/snowflake.ts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/airtable.ts b/packages/server/src/integrations/airtable.ts index 54a91132e3..a102caab76 100644 --- a/packages/server/src/integrations/airtable.ts +++ b/packages/server/src/integrations/airtable.ts @@ -1,5 +1,6 @@ import { ConnectionInfo, + DatasourceFeature, DatasourceFieldType, Integration, IntegrationBase, @@ -19,7 +20,7 @@ const SCHEMA: Integration = { "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.", friendlyName: "Airtable", type: "Spreadsheet", - features: [], + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { apiKey: { type: DatasourceFieldType.PASSWORD, diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index a8fd966782..eea9cc4176 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -1,5 +1,6 @@ import { ConnectionInfo, + DatasourceFeature, DatasourceFieldType, DatasourcePlus, FieldType, @@ -65,7 +66,7 @@ const SCHEMA: Integration = { "Create and collaborate on online spreadsheets in real-time and from any device. ", friendlyName: "Google Sheets", type: "Spreadsheet", - features: [], + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { spreadsheetId: { display: "Google Sheet URL", diff --git a/packages/server/src/integrations/snowflake.ts b/packages/server/src/integrations/snowflake.ts index dc5fb66c55..9b743131ae 100644 --- a/packages/server/src/integrations/snowflake.ts +++ b/packages/server/src/integrations/snowflake.ts @@ -22,7 +22,7 @@ const SCHEMA: Integration = { "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.", friendlyName: "Snowflake", type: "Relational", - features: [], + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { account: { type: "string", From e465422031aad5dbc5d30a934d04469a720ad854 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 May 2023 15:08:34 +0200 Subject: [PATCH 65/76] Implement connection --- packages/server/src/integrations/firebase.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/firebase.ts b/packages/server/src/integrations/firebase.ts index e8b3c8d92c..3907275f41 100644 --- a/packages/server/src/integrations/firebase.ts +++ b/packages/server/src/integrations/firebase.ts @@ -4,6 +4,7 @@ import { QueryType, IntegrationBase, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import { Firestore, WhereFilterOp } from "@google-cloud/firestore" @@ -19,7 +20,7 @@ const SCHEMA: Integration = { type: "Non-relational", description: "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.", - features: [], + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { email: { type: DatasourceFieldType.STRING, @@ -101,6 +102,18 @@ class FirebaseIntegration implements IntegrationBase { }) } + async testConnection(): Promise { + try { + await this.client.listCollections() + return { connected: true } + } catch (e: any) { + return { + connected: false, + error: e.message as string, + } + } + } + async create(query: { json: object; extra: { [key: string]: string } }) { try { const documentReference = this.client From 832f8b3aa852aff2903a3d1c44d72d501e3957eb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 16 May 2023 19:04:46 +0100 Subject: [PATCH 66/76] Adding Oracle connection checking. --- packages/server/src/integrations/oracle.ts | 28 ++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 82d8382a31..73e41e1d79 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -8,6 +8,7 @@ import { Table, DatasourcePlus, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import { buildExternalTableId, @@ -21,7 +22,7 @@ import { FieldTypes } from "../constants" import { BindParameters, Connection, - ConnectionAttributes, + ConnectionAttributes, DBError, ExecuteOptions, Result, } from "oracledb" @@ -49,7 +50,7 @@ const SCHEMA: Integration = { type: "Relational", description: "Oracle Database is an object-relational database management system developed by Oracle Corporation", - features: [], + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { host: { type: DatasourceFieldType.STRING, @@ -322,6 +323,29 @@ class OracleIntegration extends Sql implements DatasourcePlus { this.schemaErrors = final.errors } + async testConnection() { + const response: ConnectionInfo = { + connected: false, + } + let connection + try { + connection = await this.getConnection() + } catch (err: any) { + response.connected = false + response.error = err.message + } finally { + if (connection) { + try { + await connection.close() + } catch (err: any) { + response.connected = false + response.error = err.message + } + } + } + return response + } + private async internalQuery(query: SqlQuery): Promise> { let connection try { From 9f57d7c33a700621513f6ee9b3f9d70464db9077 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 16 May 2023 19:31:12 +0100 Subject: [PATCH 67/76] Fixes after testing against actual Oracle service. --- .../server/src/api/controllers/datasource.ts | 25 +++++++++++++------ packages/server/src/integrations/oracle.ts | 4 ++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 956bffa795..4676a84744 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -47,7 +47,10 @@ async function getConnector( datasource: Datasource ): Promise { const Connector = await getIntegration(datasource.source) - datasource = await sdk.datasources.enrich(datasource) + // can't enrich if it doesn't have an ID yet + if (datasource._id) { + datasource = await sdk.datasources.enrich(datasource) + } // Connect to the DB and build the schema return new Connector(datasource.config) } @@ -127,13 +130,19 @@ export async function verify( ctx: UserCtx ) { const { datasource } = ctx.request.body - const existingDatasource = await sdk.datasources.get(datasource._id!) - - const enrichedDatasource = sdk.datasources.mergeConfigs( - datasource, - existingDatasource - ) - + let existingDatasource: undefined | Datasource + try { + existingDatasource = await sdk.datasources.get(datasource._id!) + } catch (err) { + // doesn't exist - can ignore, first creation + } + let enrichedDatasource = datasource + if (existingDatasource) { + enrichedDatasource = sdk.datasources.mergeConfigs( + datasource, + existingDatasource + ) + } const connector = await getConnector(enrichedDatasource) if (!connector.testConnection) { ctx.throw(400, "Connection information verification not supported") diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 73e41e1d79..a2101ff303 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -22,7 +22,8 @@ import { FieldTypes } from "../constants" import { BindParameters, Connection, - ConnectionAttributes, DBError, + ConnectionAttributes, + DBError, ExecuteOptions, Result, } from "oracledb" @@ -330,6 +331,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { let connection try { connection = await this.getConnection() + response.connected = true } catch (err: any) { response.connected = false response.error = err.message From 896bc61934a79b46829cda6c6ce7efba69b1f5e2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 May 2023 09:38:37 +0200 Subject: [PATCH 68/76] Check elastic connection --- .../server/src/integrations/elasticsearch.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index 621e81f367..af52799c51 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -4,6 +4,7 @@ import { QueryType, IntegrationBase, DatasourceFeature, + ConnectionInfo, } from "@budibase/types" import { Client, ClientOptions } from "@elastic/elasticsearch" @@ -21,7 +22,7 @@ const SCHEMA: Integration = { "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.", friendlyName: "ElasticSearch", type: "Non-relational", - features: [], + features: [DatasourceFeature.CONNECTION_CHECKING], datasource: { url: { type: DatasourceFieldType.STRING, @@ -97,7 +98,7 @@ const SCHEMA: Integration = { class ElasticSearchIntegration implements IntegrationBase { private config: ElasticsearchConfig - private client: any + private client constructor(config: ElasticsearchConfig) { this.config = config @@ -116,6 +117,18 @@ class ElasticSearchIntegration implements IntegrationBase { this.client = new Client(clientConfig) } + async testConnection(): Promise { + try { + await this.client.info() + return { connected: true } + } catch (e: any) { + return { + connected: false, + error: e.message as string, + } + } + } + async create(query: { index: string; json: object }) { const { index, json } = query From 6b2e6f7bcbae67301476db23739f1f83530ca0b0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 May 2023 09:38:59 +0200 Subject: [PATCH 69/76] Test happy path --- .../integrations/validators/elastic.spec.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 qa-core/src/integrations/validators/elastic.spec.ts diff --git a/qa-core/src/integrations/validators/elastic.spec.ts b/qa-core/src/integrations/validators/elastic.spec.ts new file mode 100644 index 0000000000..61e4ed00f5 --- /dev/null +++ b/qa-core/src/integrations/validators/elastic.spec.ts @@ -0,0 +1,23 @@ +import { ElasticsearchContainer } from "testcontainers" +import elastic from "../../../../packages/server/src/integrations/elasticsearch" + +jest.unmock("@elastic/elasticsearch") + +describe("datasource validators", () => { + describe("elastic search", () => { + let url: string + + beforeAll(async () => { + const container = await new ElasticsearchContainer().start() + url = container.getHttpUrl() + }) + + it("test valid connection string", async () => { + const integration = new elastic.integration({ + url, + }) + const result = await integration.testConnection() + expect(result).toEqual({ connected: true }) + }) + }) +}) From 860a1ee27178ca26d4e52122e9f1a47688b0d6d2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 May 2023 09:44:55 +0200 Subject: [PATCH 70/76] Test invalid config --- qa-core/src/integrations/validators/elastic.spec.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/qa-core/src/integrations/validators/elastic.spec.ts b/qa-core/src/integrations/validators/elastic.spec.ts index 61e4ed00f5..39fd732744 100644 --- a/qa-core/src/integrations/validators/elastic.spec.ts +++ b/qa-core/src/integrations/validators/elastic.spec.ts @@ -19,5 +19,16 @@ describe("datasource validators", () => { const result = await integration.testConnection() expect(result).toEqual({ connected: true }) }) + + it("test wrong connection string", async () => { + const integration = new elastic.integration({ + url: `http://localhost:5656`, + }) + const result = await integration.testConnection() + expect(result).toEqual({ + connected: false, + error: "connect ECONNREFUSED 127.0.0.1:5656", + }) + }) }) }) From d2877fff6efd975d30648867f8a4ef1d676f24ba Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 May 2023 09:57:43 +0200 Subject: [PATCH 71/76] Update tests to check new formats --- qa-core/src/integrations/validators/arango.spec.ts | 4 +++- qa-core/src/integrations/validators/couch.spec.ts | 3 ++- qa-core/src/integrations/validators/dynamodb.spec.ts | 3 ++- qa-core/src/integrations/validators/mongo.spec.ts | 2 +- qa-core/src/integrations/validators/mssql.spec.ts | 3 ++- qa-core/src/integrations/validators/mysql.spec.ts | 4 +++- qa-core/src/integrations/validators/postgres.spec.ts | 3 ++- qa-core/src/integrations/validators/redis.spec.ts | 4 ++-- qa-core/src/integrations/validators/s3.spec.ts | 3 ++- 9 files changed, 19 insertions(+), 10 deletions(-) diff --git a/qa-core/src/integrations/validators/arango.spec.ts b/qa-core/src/integrations/validators/arango.spec.ts index 51282de13f..7c0faafd61 100644 --- a/qa-core/src/integrations/validators/arango.spec.ts +++ b/qa-core/src/integrations/validators/arango.spec.ts @@ -41,7 +41,7 @@ describe("datasource validators", () => { collection: "", }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) it("test wrong password", async () => { @@ -54,6 +54,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: "not authorized to execute this request", }) }) @@ -68,6 +69,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: "getaddrinfo ENOTFOUND not.here", }) }) diff --git a/qa-core/src/integrations/validators/couch.spec.ts b/qa-core/src/integrations/validators/couch.spec.ts index 18fbea8c43..9a736b4b95 100644 --- a/qa-core/src/integrations/validators/couch.spec.ts +++ b/qa-core/src/integrations/validators/couch.spec.ts @@ -37,7 +37,7 @@ describe("datasource validators", () => { database: "db", }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) it("test invalid database", async () => { @@ -56,6 +56,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: "request to http://invalid:123/any failed, reason: getaddrinfo ENOTFOUND invalid", }) diff --git a/qa-core/src/integrations/validators/dynamodb.spec.ts b/qa-core/src/integrations/validators/dynamodb.spec.ts index 53377ff06b..c885f64213 100644 --- a/qa-core/src/integrations/validators/dynamodb.spec.ts +++ b/qa-core/src/integrations/validators/dynamodb.spec.ts @@ -41,7 +41,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) it("test wrong endpoint", async () => { @@ -54,6 +54,7 @@ describe("datasource validators", () => { const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: "Inaccessible host: `wrong.url' at port `undefined'. This service may not be available in the `eu-west-1' region.", }) diff --git a/qa-core/src/integrations/validators/mongo.spec.ts b/qa-core/src/integrations/validators/mongo.spec.ts index dd8e188f27..ad09f1ffb7 100644 --- a/qa-core/src/integrations/validators/mongo.spec.ts +++ b/qa-core/src/integrations/validators/mongo.spec.ts @@ -49,7 +49,7 @@ describe("datasource validators", () => { tlsCAFile: "", }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) it("test invalid password", async () => { diff --git a/qa-core/src/integrations/validators/mssql.spec.ts b/qa-core/src/integrations/validators/mssql.spec.ts index 89af8e87c0..17f79d86ec 100644 --- a/qa-core/src/integrations/validators/mssql.spec.ts +++ b/qa-core/src/integrations/validators/mssql.spec.ts @@ -43,7 +43,7 @@ describe("datasource validators", () => { schema: "dbo", }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) it("test invalid password", async () => { @@ -57,6 +57,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: "ConnectionError: Login failed for user 'sa'.", }) }) diff --git a/qa-core/src/integrations/validators/mysql.spec.ts b/qa-core/src/integrations/validators/mysql.spec.ts index 166085fdd5..6ee39731fa 100644 --- a/qa-core/src/integrations/validators/mysql.spec.ts +++ b/qa-core/src/integrations/validators/mysql.spec.ts @@ -31,7 +31,7 @@ describe("datasource validators", () => { rejectUnauthorized: true, }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) it("test invalid database", async () => { @@ -45,6 +45,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: "Access denied for user 'user'@'%' to database 'test'", }) }) @@ -60,6 +61,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: "Access denied for user 'root'@'172.17.0.1' (using password: YES)", }) diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.spec.ts index 5aa3250e2a..5101cf1d2d 100644 --- a/qa-core/src/integrations/validators/postgres.spec.ts +++ b/qa-core/src/integrations/validators/postgres.spec.ts @@ -30,7 +30,7 @@ describe("datasource validators", () => { rejectUnauthorized: false, }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) it("test invalid connection string", async () => { @@ -46,6 +46,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: 'password authentication failed for user "wrong"', }) }) diff --git a/qa-core/src/integrations/validators/redis.spec.ts b/qa-core/src/integrations/validators/redis.spec.ts index fd35f5d23e..1c82e35b74 100644 --- a/qa-core/src/integrations/validators/redis.spec.ts +++ b/qa-core/src/integrations/validators/redis.spec.ts @@ -24,7 +24,7 @@ describe("datasource validators", () => { username: "", }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) it("test invalid connection even with wrong user/password", async () => { @@ -64,7 +64,7 @@ describe("datasource validators", () => { password: "P@ssW0rd!", }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) }) }) diff --git a/qa-core/src/integrations/validators/s3.spec.ts b/qa-core/src/integrations/validators/s3.spec.ts index 6ed5c44449..7bb415ee3d 100644 --- a/qa-core/src/integrations/validators/s3.spec.ts +++ b/qa-core/src/integrations/validators/s3.spec.ts @@ -30,7 +30,7 @@ describe("datasource validators", () => { endpoint: `http://${host}:${port}`, }) const result = await integration.testConnection() - expect(result).toBe(true) + expect(result).toEqual({ connected: true }) }) it("test wrong endpoint", async () => { @@ -43,6 +43,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: "Inaccessible host: `wrong' at port `undefined'. This service may not be available in the `eu-west-2' region.", }) From ed8550b271c06012bde9b3304b18a298952debde Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 May 2023 10:14:13 +0200 Subject: [PATCH 72/76] Fix tests asserts --- qa-core/src/integrations/validators/couch.spec.ts | 6 ++++-- qa-core/src/integrations/validators/mongo.spec.ts | 15 ++++++++++++--- qa-core/src/integrations/validators/redis.spec.ts | 1 + 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/qa-core/src/integrations/validators/couch.spec.ts b/qa-core/src/integrations/validators/couch.spec.ts index 9a736b4b95..b0f4254610 100644 --- a/qa-core/src/integrations/validators/couch.spec.ts +++ b/qa-core/src/integrations/validators/couch.spec.ts @@ -1,7 +1,7 @@ import { GenericContainer } from "testcontainers" -import { generator } from "@budibase/backend-core/tests" import couchdb from "../../../../packages/server/src/integrations/couchdb" +import { generator } from "../../shared" describe("datasource validators", () => { describe("couchdb", () => { @@ -46,7 +46,9 @@ describe("datasource validators", () => { database: "random_db", }) const result = await integration.testConnection() - expect(result).toBe(false) + expect(result).toEqual({ + connected: false, + }) }) it("test invalid url", async () => { diff --git a/qa-core/src/integrations/validators/mongo.spec.ts b/qa-core/src/integrations/validators/mongo.spec.ts index ad09f1ffb7..a20b7cd7fa 100644 --- a/qa-core/src/integrations/validators/mongo.spec.ts +++ b/qa-core/src/integrations/validators/mongo.spec.ts @@ -61,7 +61,10 @@ describe("datasource validators", () => { tlsCAFile: "", }) const result = await integration.testConnection() - expect(result).toEqual({ error: "Authentication failed." }) + expect(result).toEqual({ + connected: false, + error: "Authentication failed.", + }) }) it("test invalid username", async () => { @@ -73,7 +76,10 @@ describe("datasource validators", () => { tlsCAFile: "", }) const result = await integration.testConnection() - expect(result).toEqual({ error: "Authentication failed." }) + expect(result).toEqual({ + connected: false, + error: "Authentication failed.", + }) }) it("test invalid connection", async () => { @@ -85,7 +91,10 @@ describe("datasource validators", () => { tlsCAFile: "", }) const result = await integration.testConnection() - expect(result).toEqual({ error: "Error: getaddrinfo ENOTFOUND http" }) + expect(result).toEqual({ + connected: false, + error: "Error: getaddrinfo ENOTFOUND http", + }) }) }) }) diff --git a/qa-core/src/integrations/validators/redis.spec.ts b/qa-core/src/integrations/validators/redis.spec.ts index 1c82e35b74..89ada2fe2d 100644 --- a/qa-core/src/integrations/validators/redis.spec.ts +++ b/qa-core/src/integrations/validators/redis.spec.ts @@ -36,6 +36,7 @@ describe("datasource validators", () => { }) const result = await integration.testConnection() expect(result).toEqual({ + connected: false, error: "WRONGPASS invalid username-password pair or user is disabled.", }) From ee5f0ec64149c0c899844116ca0f6528da211197 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 May 2023 09:15:06 +0100 Subject: [PATCH 73/76] Remove unused type. --- packages/server/src/integrations/oracle.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index a2101ff303..f8ec6e8bae 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -23,7 +23,6 @@ import { BindParameters, Connection, ConnectionAttributes, - DBError, ExecuteOptions, Result, } from "oracledb" From e09550f7e31c3063c053a11e9d4e5cea25da9443 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 May 2023 09:45:25 +0100 Subject: [PATCH 74/76] PR comments. --- packages/server/src/api/controllers/datasource.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 4676a84744..8f13e0e618 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -131,10 +131,8 @@ export async function verify( ) { const { datasource } = ctx.request.body let existingDatasource: undefined | Datasource - try { - existingDatasource = await sdk.datasources.get(datasource._id!) - } catch (err) { - // doesn't exist - can ignore, first creation + if (datasource._id) { + existingDatasource = await sdk.datasources.get(datasource._id) } let enrichedDatasource = datasource if (existingDatasource) { From 79bc2afeca550368f5495e53183c56723080c5aa Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 May 2023 12:04:06 +0200 Subject: [PATCH 75/76] Run validators nightly --- .../validators/{arango.spec.ts => arango.nightly.spec.ts} | 0 .../validators/{couch.spec.ts => couch.nightly.spec.ts} | 0 .../validators/{dynamodb.spec.ts => dynamodb.nightly.spec.ts} | 0 .../validators/{elastic.spec.ts => elastic.nightly.spec.ts} | 0 .../validators/{mongo.spec.ts => mongo.nightly.spec.ts} | 0 .../validators/{mssql.spec.ts => mssql.nightly.spec.ts} | 0 .../validators/{mysql.spec.ts => mysql.nightly.spec.ts} | 0 .../validators/{postgres.spec.ts => postgres.nightly.spec.ts} | 0 .../validators/{redis.spec.ts => redis.nightly.spec.ts} | 0 .../integrations/validators/{s3.spec.ts => s3.nightly.spec.ts} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename qa-core/src/integrations/validators/{arango.spec.ts => arango.nightly.spec.ts} (100%) rename qa-core/src/integrations/validators/{couch.spec.ts => couch.nightly.spec.ts} (100%) rename qa-core/src/integrations/validators/{dynamodb.spec.ts => dynamodb.nightly.spec.ts} (100%) rename qa-core/src/integrations/validators/{elastic.spec.ts => elastic.nightly.spec.ts} (100%) rename qa-core/src/integrations/validators/{mongo.spec.ts => mongo.nightly.spec.ts} (100%) rename qa-core/src/integrations/validators/{mssql.spec.ts => mssql.nightly.spec.ts} (100%) rename qa-core/src/integrations/validators/{mysql.spec.ts => mysql.nightly.spec.ts} (100%) rename qa-core/src/integrations/validators/{postgres.spec.ts => postgres.nightly.spec.ts} (100%) rename qa-core/src/integrations/validators/{redis.spec.ts => redis.nightly.spec.ts} (100%) rename qa-core/src/integrations/validators/{s3.spec.ts => s3.nightly.spec.ts} (100%) diff --git a/qa-core/src/integrations/validators/arango.spec.ts b/qa-core/src/integrations/validators/arango.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/arango.spec.ts rename to qa-core/src/integrations/validators/arango.nightly.spec.ts diff --git a/qa-core/src/integrations/validators/couch.spec.ts b/qa-core/src/integrations/validators/couch.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/couch.spec.ts rename to qa-core/src/integrations/validators/couch.nightly.spec.ts diff --git a/qa-core/src/integrations/validators/dynamodb.spec.ts b/qa-core/src/integrations/validators/dynamodb.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/dynamodb.spec.ts rename to qa-core/src/integrations/validators/dynamodb.nightly.spec.ts diff --git a/qa-core/src/integrations/validators/elastic.spec.ts b/qa-core/src/integrations/validators/elastic.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/elastic.spec.ts rename to qa-core/src/integrations/validators/elastic.nightly.spec.ts diff --git a/qa-core/src/integrations/validators/mongo.spec.ts b/qa-core/src/integrations/validators/mongo.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/mongo.spec.ts rename to qa-core/src/integrations/validators/mongo.nightly.spec.ts diff --git a/qa-core/src/integrations/validators/mssql.spec.ts b/qa-core/src/integrations/validators/mssql.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/mssql.spec.ts rename to qa-core/src/integrations/validators/mssql.nightly.spec.ts diff --git a/qa-core/src/integrations/validators/mysql.spec.ts b/qa-core/src/integrations/validators/mysql.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/mysql.spec.ts rename to qa-core/src/integrations/validators/mysql.nightly.spec.ts diff --git a/qa-core/src/integrations/validators/postgres.spec.ts b/qa-core/src/integrations/validators/postgres.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/postgres.spec.ts rename to qa-core/src/integrations/validators/postgres.nightly.spec.ts diff --git a/qa-core/src/integrations/validators/redis.spec.ts b/qa-core/src/integrations/validators/redis.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/redis.spec.ts rename to qa-core/src/integrations/validators/redis.nightly.spec.ts diff --git a/qa-core/src/integrations/validators/s3.spec.ts b/qa-core/src/integrations/validators/s3.nightly.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/s3.spec.ts rename to qa-core/src/integrations/validators/s3.nightly.spec.ts From eb46037344e512fe1a9614ba01f65c3b669747ee Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 May 2023 12:59:20 +0200 Subject: [PATCH 76/76] Rename validator integration tests --- .../{arango.nightly.spec.ts => arango.integration.spec.ts} | 0 .../{couch.nightly.spec.ts => couch.integration.spec.ts} | 0 .../{dynamodb.nightly.spec.ts => dynamodb.integration.spec.ts} | 0 .../{elastic.nightly.spec.ts => elastic.integration.spec.ts} | 0 .../{mongo.nightly.spec.ts => mongo.integration.spec.ts} | 0 .../{mssql.nightly.spec.ts => mssql.integration.spec.ts} | 0 .../{mysql.nightly.spec.ts => mysql.integration.spec.ts} | 0 .../{postgres.nightly.spec.ts => postgres.integration.spec.ts} | 1 - .../{redis.nightly.spec.ts => redis.integration.spec.ts} | 0 .../validators/{s3.nightly.spec.ts => s3.integration.spec.ts} | 0 10 files changed, 1 deletion(-) rename qa-core/src/integrations/validators/{arango.nightly.spec.ts => arango.integration.spec.ts} (100%) rename qa-core/src/integrations/validators/{couch.nightly.spec.ts => couch.integration.spec.ts} (100%) rename qa-core/src/integrations/validators/{dynamodb.nightly.spec.ts => dynamodb.integration.spec.ts} (100%) rename qa-core/src/integrations/validators/{elastic.nightly.spec.ts => elastic.integration.spec.ts} (100%) rename qa-core/src/integrations/validators/{mongo.nightly.spec.ts => mongo.integration.spec.ts} (100%) rename qa-core/src/integrations/validators/{mssql.nightly.spec.ts => mssql.integration.spec.ts} (100%) rename qa-core/src/integrations/validators/{mysql.nightly.spec.ts => mysql.integration.spec.ts} (100%) rename qa-core/src/integrations/validators/{postgres.nightly.spec.ts => postgres.integration.spec.ts} (94%) rename qa-core/src/integrations/validators/{redis.nightly.spec.ts => redis.integration.spec.ts} (100%) rename qa-core/src/integrations/validators/{s3.nightly.spec.ts => s3.integration.spec.ts} (100%) diff --git a/qa-core/src/integrations/validators/arango.nightly.spec.ts b/qa-core/src/integrations/validators/arango.integration.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/arango.nightly.spec.ts rename to qa-core/src/integrations/validators/arango.integration.spec.ts diff --git a/qa-core/src/integrations/validators/couch.nightly.spec.ts b/qa-core/src/integrations/validators/couch.integration.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/couch.nightly.spec.ts rename to qa-core/src/integrations/validators/couch.integration.spec.ts diff --git a/qa-core/src/integrations/validators/dynamodb.nightly.spec.ts b/qa-core/src/integrations/validators/dynamodb.integration.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/dynamodb.nightly.spec.ts rename to qa-core/src/integrations/validators/dynamodb.integration.spec.ts diff --git a/qa-core/src/integrations/validators/elastic.nightly.spec.ts b/qa-core/src/integrations/validators/elastic.integration.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/elastic.nightly.spec.ts rename to qa-core/src/integrations/validators/elastic.integration.spec.ts diff --git a/qa-core/src/integrations/validators/mongo.nightly.spec.ts b/qa-core/src/integrations/validators/mongo.integration.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/mongo.nightly.spec.ts rename to qa-core/src/integrations/validators/mongo.integration.spec.ts diff --git a/qa-core/src/integrations/validators/mssql.nightly.spec.ts b/qa-core/src/integrations/validators/mssql.integration.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/mssql.nightly.spec.ts rename to qa-core/src/integrations/validators/mssql.integration.spec.ts diff --git a/qa-core/src/integrations/validators/mysql.nightly.spec.ts b/qa-core/src/integrations/validators/mysql.integration.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/mysql.nightly.spec.ts rename to qa-core/src/integrations/validators/mysql.integration.spec.ts diff --git a/qa-core/src/integrations/validators/postgres.nightly.spec.ts b/qa-core/src/integrations/validators/postgres.integration.spec.ts similarity index 94% rename from qa-core/src/integrations/validators/postgres.nightly.spec.ts rename to qa-core/src/integrations/validators/postgres.integration.spec.ts index 5101cf1d2d..029d929df0 100644 --- a/qa-core/src/integrations/validators/postgres.nightly.spec.ts +++ b/qa-core/src/integrations/validators/postgres.integration.spec.ts @@ -1,5 +1,4 @@ import { GenericContainer } from "testcontainers" -import postgres from "../../../../packages/server/src/integrations/postgres" jest.unmock("pg") diff --git a/qa-core/src/integrations/validators/redis.nightly.spec.ts b/qa-core/src/integrations/validators/redis.integration.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/redis.nightly.spec.ts rename to qa-core/src/integrations/validators/redis.integration.spec.ts diff --git a/qa-core/src/integrations/validators/s3.nightly.spec.ts b/qa-core/src/integrations/validators/s3.integration.spec.ts similarity index 100% rename from qa-core/src/integrations/validators/s3.nightly.spec.ts rename to qa-core/src/integrations/validators/s3.integration.spec.ts