diff --git a/packages/server/__mocks__/aws-sdk.ts b/packages/server/__mocks__/aws-sdk.ts index 75353db7e6..b0da0e97b6 100644 --- a/packages/server/__mocks__/aws-sdk.ts +++ b/packages/server/__mocks__/aws-sdk.ts @@ -37,6 +37,20 @@ module AwsMock { Contents: {}, }) ) + + // @ts-ignore + this.createBucket = jest.fn( + response({ + Contents: {}, + }) + ) + + // @ts-ignore + this.deleteObjects = jest.fn( + response({ + Contents: {}, + }) + ) } aws.DynamoDB = { DocumentClient } diff --git a/packages/server/package.json b/packages/server/package.json index cd9eb977e3..306a86273b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -200,4 +200,4 @@ "oracledb": "5.3.0" }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" -} +} \ No newline at end of file diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index f92124082e..b965b177d0 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -1,5 +1,12 @@ -import { Integration, QueryType, IntegrationBase } from "@budibase/types" +import { + Integration, + QueryType, + IntegrationBase, + DatasourceFieldType, +} from "@budibase/types" + const AWS = require("aws-sdk") +const csv = require("csvtojson") interface S3Config { region: string @@ -40,13 +47,103 @@ const SCHEMA: Integration = { }, }, query: { + create: { + type: QueryType.FIELDS, + fields: { + bucket: { + display: "New Bucket", + type: DatasourceFieldType.STRING, + required: true, + }, + location: { + required: true, + default: "us-east-1", + type: DatasourceFieldType.STRING, + }, + grantFullControl: { + display: "Grant full control", + type: DatasourceFieldType.STRING, + }, + grantRead: { + display: "Grant read", + type: DatasourceFieldType.STRING, + }, + grantReadAcp: { + display: "Grant read ACP", + type: DatasourceFieldType.STRING, + }, + grantWrite: { + display: "Grant write", + type: DatasourceFieldType.STRING, + }, + grantWriteAcp: { + display: "Grant write ACP", + type: DatasourceFieldType.STRING, + }, + }, + }, read: { type: QueryType.FIELDS, fields: { bucket: { - type: "string", + type: DatasourceFieldType.STRING, required: true, }, + delimiter: { + type: DatasourceFieldType.STRING, + }, + marker: { + type: DatasourceFieldType.STRING, + }, + maxKeys: { + type: DatasourceFieldType.NUMBER, + display: "Max Keys", + }, + prefix: { + type: DatasourceFieldType.STRING, + }, + }, + }, + readCsv: { + displayName: "Read CSV", + type: QueryType.FIELDS, + fields: { + bucket: { + type: DatasourceFieldType.STRING, + required: true, + }, + key: { + type: DatasourceFieldType.STRING, + required: true, + }, + }, + }, + delete: { + type: QueryType.FIELDS, + fields: { + bucket: { + type: DatasourceFieldType.STRING, + required: true, + }, + delete: { + type: DatasourceFieldType.JSON, + required: true, + }, + }, + }, + }, + extra: { + acl: { + required: false, + displayName: "ACL", + type: DatasourceFieldType.LIST, + data: { + create: [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + ], }, }, }, @@ -67,14 +164,93 @@ class S3Integration implements IntegrationBase { this.client = new AWS.S3(this.config) } - async read(query: { bucket: string }) { + async create(query: { + bucket: string + location: string + grantFullControl: string + grantRead: string + grantReadAcp: string + grantWrite: string + grantWriteAcp: string + extra: { + acl: string + } + }) { + let params: any = { + Bucket: query.bucket, + ACL: query.extra?.acl, + GrantFullControl: query.grantFullControl, + GrantRead: query.grantRead, + GrantReadACP: query.grantReadAcp, + GrantWrite: query.grantWrite, + GrantWriteACP: query.grantWriteAcp, + } + if (query.location) { + params["CreateBucketConfiguration"] = { + LocationConstraint: query.location, + } + } + return await this.client.createBucket(params).promise() + } + + async read(query: { + bucket: string + delimiter: string + expectedBucketOwner: string + marker: string + maxKeys: number + prefix: string + }) { const response = await this.client .listObjects({ Bucket: query.bucket, + Delimiter: query.delimiter, + Marker: query.marker, + MaxKeys: query.maxKeys, + Prefix: query.prefix, }) .promise() return response.Contents } + + async readCsv(query: { bucket: string; key: string }) { + const stream = this.client + .getObject({ + Bucket: query.bucket, + Key: query.key, + }) + .createReadStream() + + let csvError = false + return new Promise((resolve, reject) => { + stream.on("error", (err: Error) => { + reject(err) + }) + const response = csv() + .fromStream(stream) + .on("error", () => { + csvError = true + }) + stream.on("finish", () => { + resolve(response) + }) + }).catch(err => { + if (csvError) { + throw new Error("Could not read CSV") + } else { + throw err + } + }) + } + + async delete(query: { bucket: string; delete: string }) { + return await this.client + .deleteObjects({ + Bucket: query.bucket, + Delete: JSON.parse(query.delete), + }) + .promise() + } } export default { diff --git a/packages/server/src/integrations/tests/s3.spec.ts b/packages/server/src/integrations/tests/s3.spec.ts index 503702c042..35107adefe 100644 --- a/packages/server/src/integrations/tests/s3.spec.ts +++ b/packages/server/src/integrations/tests/s3.spec.ts @@ -18,11 +18,95 @@ describe("S3 Integration", () => { }) it("calls the read method with the correct params", async () => { - const response = await config.integration.read({ + await config.integration.read({ bucket: "test", + delimiter: "/", + marker: "file.txt", + maxKeys: 999, + prefix: "directory/", }) expect(config.integration.client.listObjects).toHaveBeenCalledWith({ Bucket: "test", + Delimiter: "/", + Marker: "file.txt", + MaxKeys: 999, + Prefix: "directory/", + }) + }) + + it("calls the create method with the correct params", async () => { + await config.integration.create({ + bucket: "test", + location: "af-south-1", + grantFullControl: "me", + grantRead: "him", + grantReadAcp: "her", + grantWrite: "she", + grantWriteAcp: "he", + objectLockEnabledForBucket: true, + extra: { + acl: "private", + }, + }) + expect(config.integration.client.createBucket).toHaveBeenCalledWith({ + Bucket: "test", + CreateBucketConfiguration: { + LocationConstraint: "af-south-1", + }, + GrantFullControl: "me", + GrantRead: "him", + GrantReadACP: "her", + GrantWrite: "she", + GrantWriteACP: "he", + ACL: "private", + }) + }) + + it("does not add undefined location constraint when calling the create method", async () => { + await config.integration.create({ + bucket: "test", + }) + expect(config.integration.client.createBucket).toHaveBeenCalledWith({ + Bucket: "test", + GrantFullControl: undefined, + GrantRead: undefined, + GrantReadACP: undefined, + GrantWrite: undefined, + GrantWriteACP: undefined, + ACL: undefined, + }) + }) + + it("calls the delete method with the correct params ", async () => { + await config.integration.delete({ + bucket: "test", + delete: `{ + "Objects": [ + { + "Key": "HappyFace.jpg", + "VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b" + }, + { + "Key": "HappyFace.jpg", + "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd" + } + ] + }`, + }) + expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({ + Bucket: "test", + Delete: { + Objects: [ + { + Key: "HappyFace.jpg", + VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b", + }, + { + Key: "HappyFace.jpg", + VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd", + }, + ], + }, }) }) }) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index beb431bfaf..36ccb0bd65 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -5172,7 +5172,7 @@ cssstyle@^2.3.0: dependencies: cssom "~0.3.6" -csvtojson@2.0.10: +csvtojson@^2.0.10: version "2.0.10" resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz#11e7242cc630da54efce7958a45f443210357574" integrity sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==