From 3c50b241066de3c7f3fc4664c642b3149a98732f Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 30 Aug 2022 11:17:11 +0100 Subject: [PATCH 01/10] Add additional params to listObjects --- packages/server/src/integrations/s3.ts | 34 ++++++++++++++++--- .../server/src/integrations/tests/s3.spec.js | 12 +++++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index e8da696424..fabb6daa3d 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -1,4 +1,4 @@ -import { Integration, QueryType, IntegrationBase } from "@budibase/types" +import { Integration, QueryType, IntegrationBase, DatasourceFieldType } from "@budibase/types" module S3Module { const AWS = require("aws-sdk") @@ -46,12 +46,25 @@ module S3Module { type: QueryType.FIELDS, fields: { bucket: { - type: "string", + type: DatasourceFieldType.STRING, required: true, }, + delimiter: { + type: DatasourceFieldType.STRING, + }, + marker: { + type: DatasourceFieldType.STRING, + }, + maxKeys: { + type: DatasourceFieldType.NUMBER, + display: "Max Keys", + }, + prefix: { + type: DatasourceFieldType.STRING, + }, }, - }, - }, + } + } } class S3Integration implements IntegrationBase { @@ -69,10 +82,21 @@ module S3Module { this.client = new AWS.S3(this.config) } - async read(query: { bucket: string }) { + async read(query: { + bucket: string, + delimiter: string, + expectedBucketOwner: string, + marker: string, + maxKeys: number, + prefix: string, + }) { const response = await this.client .listObjects({ Bucket: query.bucket, + Delimiter: query.delimiter, + Marker: query.marker, + MaxKeys: query.maxKeys, + Prefix: query.prefix, }) .promise() return response.Contents diff --git a/packages/server/src/integrations/tests/s3.spec.js b/packages/server/src/integrations/tests/s3.spec.js index 7ac403dbd4..a655112973 100644 --- a/packages/server/src/integrations/tests/s3.spec.js +++ b/packages/server/src/integrations/tests/s3.spec.js @@ -17,10 +17,18 @@ describe("S3 Integration", () => { it("calls the read method with the correct params", async () => { const response = await config.integration.read({ - bucket: "test" + bucket: "test", + delimiter: "/", + marker: "file.txt", + maxKeys: 999, + prefix: "directory/", }) expect(config.integration.client.listObjects).toHaveBeenCalledWith({ - Bucket: "test" + Bucket: "test", + Delimiter: "/", + Marker: "file.txt", + MaxKeys: 999, + Prefix: "directory/" }) }) }) \ No newline at end of file From 36ebb30825c151571cfbae026eb4d08f389ff871 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 30 Aug 2022 13:42:49 +0100 Subject: [PATCH 02/10] WIP --- packages/server/package.json | 2 +- packages/server/src/integrations/s3.ts | 66 ++++++++++++++++++++++---- packages/server/yarn.lock | 2 +- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 4ef6a5bd18..4d4c580709 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -95,7 +95,7 @@ "bcryptjs": "2.4.3", "bull": "4.8.5", "chmodr": "1.2.0", - "csvtojson": "2.0.10", + "csvtojson": "^2.0.10", "curlconverter": "3.21.0", "dotenv": "8.2.0", "download": "8.0.0", diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index fabb6daa3d..ddae25eee8 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -1,7 +1,13 @@ -import { Integration, QueryType, IntegrationBase, DatasourceFieldType } from "@budibase/types" +import { + Integration, + QueryType, + IntegrationBase, + DatasourceFieldType, +} from "@budibase/types" module S3Module { const AWS = require("aws-sdk") + const csv = require("csvtojson") interface S3Config { region: string @@ -63,8 +69,22 @@ module S3Module { type: DatasourceFieldType.STRING, }, }, - } - } + }, + readCsv: { + displayName: "Read CSV", + type: QueryType.FIELDS, + fields: { + bucket: { + type: DatasourceFieldType.STRING, + required: true, + }, + key: { + type: DatasourceFieldType.STRING, + required: true, + }, + }, + }, + }, } class S3Integration implements IntegrationBase { @@ -82,13 +102,13 @@ module S3Module { this.client = new AWS.S3(this.config) } - async read(query: { - bucket: string, - delimiter: string, - expectedBucketOwner: string, - marker: string, - maxKeys: number, - prefix: string, + async read(query: { + bucket: string + delimiter: string + expectedBucketOwner: string + marker: string + maxKeys: number + prefix: string }) { const response = await this.client .listObjects({ @@ -101,6 +121,32 @@ module S3Module { .promise() return response.Contents } + + async readCsv(query: { bucket: string; key: string }) { + let streamErr: string | undefined = undefined + const stream = this.client + .getObject({ + Bucket: query.bucket, + Key: query.key, + }) + .createReadStream() + .on("error", (err: string) => { + //stream.destroy() + // + console.log("err ", err) + streamErr = "ERROR" + }) + + if (streamErr) { + throw new Error("ERROR") + } + + try { + return await csv().fromStream(stream) + } catch (err) { + throw new Error("Failed to read CSV") + } + } } module.exports = { diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 0e6b40cc30..64239ce820 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -4924,7 +4924,7 @@ cssstyle@^2.3.0: dependencies: cssom "~0.3.6" -csvtojson@2.0.10: +csvtojson@^2.0.10: version "2.0.10" resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz#11e7242cc630da54efce7958a45f443210357574" integrity sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ== From 33fa0b50b7acb18759297707b2e0d434b7d6ed33 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 30 Aug 2022 13:55:55 +0100 Subject: [PATCH 03/10] Promisify stream --- packages/server/src/integrations/s3.ts | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index ddae25eee8..43aa04de6a 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -123,29 +123,22 @@ module S3Module { } async readCsv(query: { bucket: string; key: string }) { - let streamErr: string | undefined = undefined const stream = this.client .getObject({ Bucket: query.bucket, Key: query.key, }) .createReadStream() - .on("error", (err: string) => { - //stream.destroy() - // - console.log("err ", err) - streamErr = "ERROR" - }) - if (streamErr) { - throw new Error("ERROR") - } - - try { - return await csv().fromStream(stream) - } catch (err) { - throw new Error("Failed to read CSV") - } + return new Promise((resolve, reject) => { + stream + .on("error", (err: Error) => { + reject(err) + }) + .on("finish", async () => { + resolve(csv().fromStream(stream)) + }) + }) } } From 15d0d178d6d69aaa9d7b49ff26bc221482e6313a Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 30 Aug 2022 15:18:44 +0100 Subject: [PATCH 04/10] Handle non-csv file error --- packages/server/src/integrations/s3.ts | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 43aa04de6a..eb3605af8f 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -130,14 +130,25 @@ module S3Module { }) .createReadStream() - return new Promise((resolve, reject) => { - stream - .on("error", (err: Error) => { - reject(err) - }) - .on("finish", async () => { - resolve(csv().fromStream(stream)) + let csvError = false + return new Promise(async (resolve, reject) => { + stream.on("error", (err: Error) => { + reject(err) + }) + const response = csv() + .fromStream(stream) + .on("error", () => { + csvError = true }) + stream.on("finish", () => { + resolve(response) + }) + }).catch(err => { + if (csvError) { + throw new Error("Could not read CSV") + } else { + throw err + } }) } } From f6d4a97cdd1a274cac950f634c2d74fba0aa7dbe Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 30 Aug 2022 15:19:18 +0100 Subject: [PATCH 05/10] remove async --- packages/server/src/integrations/s3.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index eb3605af8f..f24fe92f6b 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -131,7 +131,7 @@ module S3Module { .createReadStream() let csvError = false - return new Promise(async (resolve, reject) => { + return new Promise((resolve, reject) => { stream.on("error", (err: Error) => { reject(err) }) From 6a1dd4b0aa3fca508512302c7006821d547be4fc Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 30 Aug 2022 16:56:56 +0100 Subject: [PATCH 06/10] Create WIP --- packages/server/src/integrations/s3.ts | 92 +++++++++++++++++++ .../server/src/integrations/tests/s3.spec.js | 32 ++++++- 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index f24fe92f6b..67568436c3 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -48,6 +48,41 @@ module S3Module { }, }, query: { + create: { + type: QueryType.FIELDS, + fields: { + bucket: { + display: "New Bucket", + type: DatasourceFieldType.STRING, + required: true, + }, + location: { + required: true, + default: "us-east-1", + type: DatasourceFieldType.STRING, + }, + grantFullControl: { + display: "Grant full control", + type: DatasourceFieldType.STRING, + }, + grantRead: { + display: "Grant read", + type: DatasourceFieldType.STRING, + }, + grantReadAcp: { + display: "Grant read ACP", + type: DatasourceFieldType.STRING, + }, + grantWrite: { + display: "Grant write", + type: DatasourceFieldType.STRING, + }, + grantWriteAcp: { + display: "Grant write ACP", + type: DatasourceFieldType.STRING, + }, + }, + }, read: { type: QueryType.FIELDS, fields: { @@ -85,6 +120,33 @@ module S3Module { }, }, }, + extra: { + acl: { + required: false, + displayName: "ACL", + type: DatasourceFieldType.LIST, + data: { + create: [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + ] + } + }, + objectOwnership: { + required: false, + displayName: "Object ownership", + type: DatasourceFieldType.LIST, + data: { + create: [ + "BucketOwnerPreferred", + "ObjectWriter", + "BucketOwnerEnforced", + ], + }, + }, + } } class S3Integration implements IntegrationBase { @@ -102,6 +164,36 @@ module S3Module { this.client = new AWS.S3(this.config) } + async create(query: { + bucket: string, + location: string, + grantFullControl: string, + grantRead: string, + grantReadAcp: string, + grantWrite: string, + grantWriteAcp: string, + extra: { + acl: string, + objectOwnership: string, + }}) { + const response = await this.client.createBucket({ + Bucket: query.bucket, + // ACL: query.extra?.acl, + CreateBucketConfiguration: { + LocationConstraint: query.location + }, + GrantFullControl: query.grantFullControl, + GrantRead: query.grantRead, + GrantReadACP: query.grantReadAcp, + GrantWrite: query.grantWrite, + GrantWriteACP: query.grantWriteAcp, + }, (err: any) => { + console.log("ERR ", err) + }) + .promise() + return response.Contents + } + async read(query: { bucket: string delimiter: string diff --git a/packages/server/src/integrations/tests/s3.spec.js b/packages/server/src/integrations/tests/s3.spec.js index a655112973..48e7221ef8 100644 --- a/packages/server/src/integrations/tests/s3.spec.js +++ b/packages/server/src/integrations/tests/s3.spec.js @@ -16,7 +16,7 @@ describe("S3 Integration", () => { }) it("calls the read method with the correct params", async () => { - const response = await config.integration.read({ + await config.integration.read({ bucket: "test", delimiter: "/", marker: "file.txt", @@ -31,4 +31,34 @@ describe("S3 Integration", () => { Prefix: "directory/" }) }) + + it("calls the create method with the correct params", async () => { + await config.integration.create({ + bucket: "test", + location: "af-south-1", + grantFullControl: "me", + grantRead: "him", + grantReadAcp: "her", + grantWrite: "she", + grantWriteAcp: "he", + objectLockEnabledForBucket: true + }, { + acl: "private", + objectOwnership: "BucketOwnerPreferred" + }) + expect(config.integration.client.createBucket).toHaveBeenCalledWith({ + Bucket: "test", + CreateBucketConfiguration: { + LocationConstraint: "af-south-1" + }, + GrantFullControl: "me", + GrantRead: "him", + GrantReadAcp: "her", + GrantWrite: "she", + GrantWriteAcp: "he", + ObjectLockEnabledForBucket: true, + ACL: "private", + ObjectOwnership: "BucketOwnerPreferred" + }) + }) }) \ No newline at end of file From 18e9f0151cb1967186666033144c36a97127af7d Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 30 Aug 2022 19:17:10 +0100 Subject: [PATCH 07/10] Create Bucket --- packages/server/__mocks__/aws-sdk.ts | 7 +++ packages/server/src/integrations/s3.ts | 54 ++++++++----------- .../server/src/integrations/tests/s3.spec.js | 29 +++++++--- 3 files changed, 49 insertions(+), 41 deletions(-) diff --git a/packages/server/__mocks__/aws-sdk.ts b/packages/server/__mocks__/aws-sdk.ts index 75353db7e6..24873ac174 100644 --- a/packages/server/__mocks__/aws-sdk.ts +++ b/packages/server/__mocks__/aws-sdk.ts @@ -37,6 +37,13 @@ module AwsMock { Contents: {}, }) ) + + // @ts-ignore + this.createBucket = jest.fn( + response({ + Contents: {}, + }) + ) } aws.DynamoDB = { DocumentClient } diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 67568436c3..b717ece222 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -131,22 +131,10 @@ module S3Module { "public-read", "public-read-write", "authenticated-read", - ] - } - }, - objectOwnership: { - required: false, - displayName: "Object ownership", - type: DatasourceFieldType.LIST, - data: { - create: [ - "BucketOwnerPreferred", - "ObjectWriter", - "BucketOwnerEnforced", ], }, }, - } + }, } class S3Integration implements IntegrationBase { @@ -165,33 +153,33 @@ module S3Module { } async create(query: { - bucket: string, - location: string, - grantFullControl: string, - grantRead: string, - grantReadAcp: string, - grantWrite: string, - grantWriteAcp: string, + bucket: string + location: string + grantFullControl: string + grantRead: string + grantReadAcp: string + grantWrite: string + grantWriteAcp: string extra: { - acl: string, - objectOwnership: string, - }}) { - const response = await this.client.createBucket({ + acl: string + } + }) { + let params: any = { Bucket: query.bucket, - // ACL: query.extra?.acl, - CreateBucketConfiguration: { - LocationConstraint: query.location - }, + ACL: query.extra?.acl, GrantFullControl: query.grantFullControl, GrantRead: query.grantRead, GrantReadACP: query.grantReadAcp, GrantWrite: query.grantWrite, GrantWriteACP: query.grantWriteAcp, - }, (err: any) => { - console.log("ERR ", err) - }) - .promise() - return response.Contents + } + if (query.location) { + params["CreateBucketConfiguration"] = { + LocationConstraint: query.location, + } + } + const response = await this.client.createBucket(params).promise() + return response } async read(query: { diff --git a/packages/server/src/integrations/tests/s3.spec.js b/packages/server/src/integrations/tests/s3.spec.js index 48e7221ef8..26a87fa99f 100644 --- a/packages/server/src/integrations/tests/s3.spec.js +++ b/packages/server/src/integrations/tests/s3.spec.js @@ -41,10 +41,10 @@ describe("S3 Integration", () => { grantReadAcp: "her", grantWrite: "she", grantWriteAcp: "he", - objectLockEnabledForBucket: true - }, { - acl: "private", - objectOwnership: "BucketOwnerPreferred" + objectLockEnabledForBucket: true, + extra: { + acl: "private" + } }) expect(config.integration.client.createBucket).toHaveBeenCalledWith({ Bucket: "test", @@ -53,12 +53,25 @@ describe("S3 Integration", () => { }, GrantFullControl: "me", GrantRead: "him", - GrantReadAcp: "her", + GrantReadACP: "her", GrantWrite: "she", - GrantWriteAcp: "he", - ObjectLockEnabledForBucket: true, + GrantWriteACP: "he", ACL: "private", - ObjectOwnership: "BucketOwnerPreferred" + }) + }) + + it("does not add undefined location constraint when calling the create method", async () => { + await config.integration.create({ + bucket: "test" + }) + expect(config.integration.client.createBucket).toHaveBeenCalledWith({ + Bucket: "test", + GrantFullControl: undefined, + GrantRead: undefined, + GrantReadACP: undefined, + GrantWrite: undefined, + GrantWriteACP: undefined, + ACL: undefined, }) }) }) \ No newline at end of file From ff5113f916c17cd31498a9b23fb95c5f1ae2cd74 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 30 Aug 2022 19:56:08 +0100 Subject: [PATCH 08/10] Delete method S3 Bucket --- packages/server/__mocks__/aws-sdk.ts | 7 ++++ packages/server/src/integrations/s3.ts | 25 ++++++++++++-- .../server/src/integrations/tests/s3.spec.js | 33 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/packages/server/__mocks__/aws-sdk.ts b/packages/server/__mocks__/aws-sdk.ts index 24873ac174..b0da0e97b6 100644 --- a/packages/server/__mocks__/aws-sdk.ts +++ b/packages/server/__mocks__/aws-sdk.ts @@ -44,6 +44,13 @@ module AwsMock { Contents: {}, }) ) + + // @ts-ignore + this.deleteObjects = jest.fn( + response({ + Contents: {}, + }) + ) } aws.DynamoDB = { DocumentClient } diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index b717ece222..72e664fe16 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -119,6 +119,19 @@ module S3Module { }, }, }, + delete: { + type: QueryType.FIELDS, + fields: { + bucket: { + type: DatasourceFieldType.STRING, + required: true, + }, + delete: { + type: DatasourceFieldType.JSON, + required: true, + }, + }, + } }, extra: { acl: { @@ -178,8 +191,7 @@ module S3Module { LocationConstraint: query.location, } } - const response = await this.client.createBucket(params).promise() - return response + return await this.client.createBucket(params).promise() } async read(query: { @@ -231,6 +243,15 @@ module S3Module { } }) } + + async delete(query: { bucket: string, delete: string }) { + return await this.client + .deleteObjects({ + Bucket: query.bucket, + Delete: JSON.parse(query.delete), + }) + .promise() + } } module.exports = { diff --git a/packages/server/src/integrations/tests/s3.spec.js b/packages/server/src/integrations/tests/s3.spec.js index 26a87fa99f..fc2477a225 100644 --- a/packages/server/src/integrations/tests/s3.spec.js +++ b/packages/server/src/integrations/tests/s3.spec.js @@ -74,4 +74,37 @@ describe("S3 Integration", () => { ACL: undefined, }) }) + + it("calls the delete method with the correct params ", async () => { + await config.integration.delete({ + bucket: "test", + delete: `{ + "Objects": [ + { + "Key": "HappyFace.jpg", + "VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b" + }, + { + "Key": "HappyFace.jpg", + "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd" + } + ] + }` + }) + expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({ + Bucket: "test", + Delete: { + Objects: [ + { + Key: "HappyFace.jpg", + VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b" + }, + { + Key: "HappyFace.jpg", + VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd" + } + ] + } + }) + }) }) \ No newline at end of file From 00c76d7960e63262889ae2c2876a131c6970a8ec Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 30 Aug 2022 19:56:38 +0100 Subject: [PATCH 09/10] lint --- packages/server/src/integrations/s3.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 72e664fe16..2ff594495d 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -131,7 +131,7 @@ module S3Module { required: true, }, }, - } + }, }, extra: { acl: { @@ -244,13 +244,13 @@ module S3Module { }) } - async delete(query: { bucket: string, delete: string }) { + async delete(query: { bucket: string; delete: string }) { return await this.client - .deleteObjects({ - Bucket: query.bucket, - Delete: JSON.parse(query.delete), - }) - .promise() + .deleteObjects({ + Bucket: query.bucket, + Delete: JSON.parse(query.delete), + }) + .promise() } } From f240f5ffbfe3d8b45ef7ff50c49212f083d3dcba Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Wed, 26 Oct 2022 16:44:25 +0100 Subject: [PATCH 10/10] lint --- packages/server/src/integrations/s3.ts | 419 +++++++++--------- .../server/src/integrations/tests/s3.spec.ts | 36 +- 2 files changed, 227 insertions(+), 228 deletions(-) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 5a3cc32b4e..b965b177d0 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -5,9 +5,8 @@ import { DatasourceFieldType, } from "@budibase/types" -module S3Module { - const AWS = require("aws-sdk") - const csv = require("csvtojson") +const AWS = require("aws-sdk") +const csv = require("csvtojson") interface S3Config { region: string @@ -17,138 +16,138 @@ interface S3Config { endpoint?: string } - const SCHEMA: Integration = { - docs: "https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", - description: - "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.", - friendlyName: "Amazon S3", - type: "Object store", - datasource: { - region: { - type: "string", - required: false, - default: "us-east-1", - }, - accessKeyId: { - type: "password", - required: true, - }, - secretAccessKey: { - type: "password", - required: true, - }, - endpoint: { - type: "string", - required: false, - }, - signatureVersion: { - type: "string", - required: false, - default: "v4", - }, +const SCHEMA: Integration = { + docs: "https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", + description: + "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.", + friendlyName: "Amazon S3", + type: "Object store", + datasource: { + region: { + type: "string", + required: false, + default: "us-east-1", }, - query: { - create: { - type: QueryType.FIELDS, - fields: { - bucket: { - display: "New Bucket", - type: DatasourceFieldType.STRING, - required: true, - }, - location: { - required: true, - default: "us-east-1", - type: DatasourceFieldType.STRING, - }, - grantFullControl: { - display: "Grant full control", - type: DatasourceFieldType.STRING, - }, - grantRead: { - display: "Grant read", - type: DatasourceFieldType.STRING, - }, - grantReadAcp: { - display: "Grant read ACP", - type: DatasourceFieldType.STRING, - }, - grantWrite: { - display: "Grant write", - type: DatasourceFieldType.STRING, - }, - grantWriteAcp: { - display: "Grant write ACP", - type: DatasourceFieldType.STRING, - }, + accessKeyId: { + type: "password", + required: true, + }, + secretAccessKey: { + type: "password", + required: true, + }, + endpoint: { + type: "string", + required: false, + }, + signatureVersion: { + type: "string", + required: false, + default: "v4", + }, + }, + query: { + create: { + type: QueryType.FIELDS, + fields: { + bucket: { + display: "New Bucket", + type: DatasourceFieldType.STRING, + required: true, }, - }, - read: { - type: QueryType.FIELDS, - fields: { - bucket: { - type: DatasourceFieldType.STRING, - required: true, - }, - delimiter: { - type: DatasourceFieldType.STRING, - }, - marker: { - type: DatasourceFieldType.STRING, - }, - maxKeys: { - type: DatasourceFieldType.NUMBER, - display: "Max Keys", - }, - prefix: { - type: DatasourceFieldType.STRING, - }, + location: { + required: true, + default: "us-east-1", + type: DatasourceFieldType.STRING, }, - }, - readCsv: { - displayName: "Read CSV", - type: QueryType.FIELDS, - fields: { - bucket: { - type: DatasourceFieldType.STRING, - required: true, - }, - key: { - type: DatasourceFieldType.STRING, - required: true, - }, + grantFullControl: { + display: "Grant full control", + type: DatasourceFieldType.STRING, }, - }, - delete: { - type: QueryType.FIELDS, - fields: { - bucket: { - type: DatasourceFieldType.STRING, - required: true, - }, - delete: { - type: DatasourceFieldType.JSON, - required: true, - }, + grantRead: { + display: "Grant read", + type: DatasourceFieldType.STRING, + }, + grantReadAcp: { + display: "Grant read ACP", + type: DatasourceFieldType.STRING, + }, + grantWrite: { + display: "Grant write", + type: DatasourceFieldType.STRING, + }, + grantWriteAcp: { + display: "Grant write ACP", + type: DatasourceFieldType.STRING, }, }, }, - extra: { - acl: { - required: false, - displayName: "ACL", - type: DatasourceFieldType.LIST, - data: { - create: [ - "private", - "public-read", - "public-read-write", - "authenticated-read", - ], + read: { + type: QueryType.FIELDS, + fields: { + bucket: { + type: DatasourceFieldType.STRING, + required: true, + }, + delimiter: { + type: DatasourceFieldType.STRING, + }, + marker: { + type: DatasourceFieldType.STRING, + }, + maxKeys: { + type: DatasourceFieldType.NUMBER, + display: "Max Keys", + }, + prefix: { + type: DatasourceFieldType.STRING, }, }, }, - } + readCsv: { + displayName: "Read CSV", + type: QueryType.FIELDS, + fields: { + bucket: { + type: DatasourceFieldType.STRING, + required: true, + }, + key: { + type: DatasourceFieldType.STRING, + required: true, + }, + }, + }, + delete: { + type: QueryType.FIELDS, + fields: { + bucket: { + type: DatasourceFieldType.STRING, + required: true, + }, + delete: { + type: DatasourceFieldType.JSON, + required: true, + }, + }, + }, + }, + extra: { + acl: { + required: false, + displayName: "ACL", + type: DatasourceFieldType.LIST, + data: { + create: [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + ], + }, + }, + }, +} class S3Integration implements IntegrationBase { private readonly config: S3Config @@ -165,95 +164,95 @@ class S3Integration implements IntegrationBase { this.client = new AWS.S3(this.config) } - async create(query: { - bucket: string - location: string - grantFullControl: string - grantRead: string - grantReadAcp: string - grantWrite: string - grantWriteAcp: string - extra: { - acl: string + async create(query: { + bucket: string + location: string + grantFullControl: string + grantRead: string + grantReadAcp: string + grantWrite: string + grantWriteAcp: string + extra: { + acl: string + } + }) { + let params: any = { + Bucket: query.bucket, + ACL: query.extra?.acl, + GrantFullControl: query.grantFullControl, + GrantRead: query.grantRead, + GrantReadACP: query.grantReadAcp, + GrantWrite: query.grantWrite, + GrantWriteACP: query.grantWriteAcp, + } + if (query.location) { + params["CreateBucketConfiguration"] = { + LocationConstraint: query.location, } - }) { - let params: any = { - Bucket: query.bucket, - ACL: query.extra?.acl, - GrantFullControl: query.grantFullControl, - GrantRead: query.grantRead, - GrantReadACP: query.grantReadAcp, - GrantWrite: query.grantWrite, - GrantWriteACP: query.grantWriteAcp, - } - if (query.location) { - params["CreateBucketConfiguration"] = { - LocationConstraint: query.location, - } - } - return await this.client.createBucket(params).promise() - } - - async read(query: { - bucket: string - delimiter: string - expectedBucketOwner: string - marker: string - maxKeys: number - prefix: string - }) { - const response = await this.client - .listObjects({ - Bucket: query.bucket, - Delimiter: query.delimiter, - Marker: query.marker, - MaxKeys: query.maxKeys, - Prefix: query.prefix, - }) - .promise() - return response.Contents - } - - async readCsv(query: { bucket: string; key: string }) { - const stream = this.client - .getObject({ - Bucket: query.bucket, - Key: query.key, - }) - .createReadStream() - - let csvError = false - return new Promise((resolve, reject) => { - stream.on("error", (err: Error) => { - reject(err) - }) - const response = csv() - .fromStream(stream) - .on("error", () => { - csvError = true - }) - stream.on("finish", () => { - resolve(response) - }) - }).catch(err => { - if (csvError) { - throw new Error("Could not read CSV") - } else { - throw err - } - }) - } - - async delete(query: { bucket: string; delete: string }) { - return await this.client - .deleteObjects({ - Bucket: query.bucket, - Delete: JSON.parse(query.delete), - }) - .promise() } + return await this.client.createBucket(params).promise() } + async read(query: { + bucket: string + delimiter: string + expectedBucketOwner: string + marker: string + maxKeys: number + prefix: string + }) { + const response = await this.client + .listObjects({ + Bucket: query.bucket, + Delimiter: query.delimiter, + Marker: query.marker, + MaxKeys: query.maxKeys, + Prefix: query.prefix, + }) + .promise() + return response.Contents + } + + async readCsv(query: { bucket: string; key: string }) { + const stream = this.client + .getObject({ + Bucket: query.bucket, + Key: query.key, + }) + .createReadStream() + + let csvError = false + return new Promise((resolve, reject) => { + stream.on("error", (err: Error) => { + reject(err) + }) + const response = csv() + .fromStream(stream) + .on("error", () => { + csvError = true + }) + stream.on("finish", () => { + resolve(response) + }) + }).catch(err => { + if (csvError) { + throw new Error("Could not read CSV") + } else { + throw err + } + }) + } + + async delete(query: { bucket: string; delete: string }) { + return await this.client + .deleteObjects({ + Bucket: query.bucket, + Delete: JSON.parse(query.delete), + }) + .promise() + } +} + export default { schema: SCHEMA, integration: S3Integration, diff --git a/packages/server/src/integrations/tests/s3.spec.ts b/packages/server/src/integrations/tests/s3.spec.ts index 592ae632bf..35107adefe 100644 --- a/packages/server/src/integrations/tests/s3.spec.ts +++ b/packages/server/src/integrations/tests/s3.spec.ts @@ -18,7 +18,7 @@ describe("S3 Integration", () => { }) it("calls the read method with the correct params", async () => { - await config.integration.read({ + await config.integration.read({ bucket: "test", delimiter: "/", marker: "file.txt", @@ -30,12 +30,12 @@ describe("S3 Integration", () => { Delimiter: "/", Marker: "file.txt", MaxKeys: 999, - Prefix: "directory/" + Prefix: "directory/", }) }) it("calls the create method with the correct params", async () => { - await config.integration.create({ + await config.integration.create({ bucket: "test", location: "af-south-1", grantFullControl: "me", @@ -45,13 +45,13 @@ describe("S3 Integration", () => { grantWriteAcp: "he", objectLockEnabledForBucket: true, extra: { - acl: "private" - } + acl: "private", + }, }) expect(config.integration.client.createBucket).toHaveBeenCalledWith({ Bucket: "test", CreateBucketConfiguration: { - LocationConstraint: "af-south-1" + LocationConstraint: "af-south-1", }, GrantFullControl: "me", GrantRead: "him", @@ -63,8 +63,8 @@ describe("S3 Integration", () => { }) it("does not add undefined location constraint when calling the create method", async () => { - await config.integration.create({ - bucket: "test" + await config.integration.create({ + bucket: "test", }) expect(config.integration.client.createBucket).toHaveBeenCalledWith({ Bucket: "test", @@ -78,7 +78,7 @@ describe("S3 Integration", () => { }) it("calls the delete method with the correct params ", async () => { - await config.integration.delete({ + await config.integration.delete({ bucket: "test", delete: `{ "Objects": [ @@ -91,22 +91,22 @@ describe("S3 Integration", () => { "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd" } ] - }` + }`, }) expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({ Bucket: "test", Delete: { Objects: [ { - Key: "HappyFace.jpg", - VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b" - }, + Key: "HappyFace.jpg", + VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b", + }, { - Key: "HappyFace.jpg", - VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd" - } - ] - } + Key: "HappyFace.jpg", + VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd", + }, + ], + }, }) }) })