1
0
Fork 0
mirror of synced 2024-06-14 00:14:39 +12:00
This commit is contained in:
Mel O'Hagan 2022-10-26 16:44:25 +01:00
parent 4661c083e2
commit f240f5ffbf
2 changed files with 227 additions and 228 deletions

View file

@ -5,9 +5,8 @@ import {
DatasourceFieldType,
} from "@budibase/types"
module S3Module {
const AWS = require("aws-sdk")
const csv = require("csvtojson")
const AWS = require("aws-sdk")
const csv = require("csvtojson")
interface S3Config {
region: string
@ -17,138 +16,138 @@ interface S3Config {
endpoint?: string
}
const SCHEMA: Integration = {
docs: "https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html",
description:
"Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
friendlyName: "Amazon S3",
type: "Object store",
datasource: {
region: {
type: "string",
required: false,
default: "us-east-1",
},
accessKeyId: {
type: "password",
required: true,
},
secretAccessKey: {
type: "password",
required: true,
},
endpoint: {
type: "string",
required: false,
},
signatureVersion: {
type: "string",
required: false,
default: "v4",
},
const SCHEMA: Integration = {
docs: "https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html",
description:
"Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
friendlyName: "Amazon S3",
type: "Object store",
datasource: {
region: {
type: "string",
required: false,
default: "us-east-1",
},
query: {
create: {
type: QueryType.FIELDS,
fields: {
bucket: {
display: "New Bucket",
type: DatasourceFieldType.STRING,
required: true,
},
location: {
required: true,
default: "us-east-1",
type: DatasourceFieldType.STRING,
},
grantFullControl: {
display: "Grant full control",
type: DatasourceFieldType.STRING,
},
grantRead: {
display: "Grant read",
type: DatasourceFieldType.STRING,
},
grantReadAcp: {
display: "Grant read ACP",
type: DatasourceFieldType.STRING,
},
grantWrite: {
display: "Grant write",
type: DatasourceFieldType.STRING,
},
grantWriteAcp: {
display: "Grant write ACP",
type: DatasourceFieldType.STRING,
},
accessKeyId: {
type: "password",
required: true,
},
secretAccessKey: {
type: "password",
required: true,
},
endpoint: {
type: "string",
required: false,
},
signatureVersion: {
type: "string",
required: false,
default: "v4",
},
},
query: {
create: {
type: QueryType.FIELDS,
fields: {
bucket: {
display: "New Bucket",
type: DatasourceFieldType.STRING,
required: true,
},
},
read: {
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
delimiter: {
type: DatasourceFieldType.STRING,
},
marker: {
type: DatasourceFieldType.STRING,
},
maxKeys: {
type: DatasourceFieldType.NUMBER,
display: "Max Keys",
},
prefix: {
type: DatasourceFieldType.STRING,
},
location: {
required: true,
default: "us-east-1",
type: DatasourceFieldType.STRING,
},
},
readCsv: {
displayName: "Read CSV",
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
key: {
type: DatasourceFieldType.STRING,
required: true,
},
grantFullControl: {
display: "Grant full control",
type: DatasourceFieldType.STRING,
},
},
delete: {
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
delete: {
type: DatasourceFieldType.JSON,
required: true,
},
grantRead: {
display: "Grant read",
type: DatasourceFieldType.STRING,
},
grantReadAcp: {
display: "Grant read ACP",
type: DatasourceFieldType.STRING,
},
grantWrite: {
display: "Grant write",
type: DatasourceFieldType.STRING,
},
grantWriteAcp: {
display: "Grant write ACP",
type: DatasourceFieldType.STRING,
},
},
},
extra: {
acl: {
required: false,
displayName: "ACL",
type: DatasourceFieldType.LIST,
data: {
create: [
"private",
"public-read",
"public-read-write",
"authenticated-read",
],
read: {
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
delimiter: {
type: DatasourceFieldType.STRING,
},
marker: {
type: DatasourceFieldType.STRING,
},
maxKeys: {
type: DatasourceFieldType.NUMBER,
display: "Max Keys",
},
prefix: {
type: DatasourceFieldType.STRING,
},
},
},
}
readCsv: {
displayName: "Read CSV",
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
key: {
type: DatasourceFieldType.STRING,
required: true,
},
},
},
delete: {
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
delete: {
type: DatasourceFieldType.JSON,
required: true,
},
},
},
},
extra: {
acl: {
required: false,
displayName: "ACL",
type: DatasourceFieldType.LIST,
data: {
create: [
"private",
"public-read",
"public-read-write",
"authenticated-read",
],
},
},
},
}
class S3Integration implements IntegrationBase {
private readonly config: S3Config
@ -165,95 +164,95 @@ class S3Integration implements IntegrationBase {
this.client = new AWS.S3(this.config)
}
async create(query: {
bucket: string
location: string
grantFullControl: string
grantRead: string
grantReadAcp: string
grantWrite: string
grantWriteAcp: string
extra: {
acl: string
async create(query: {
bucket: string
location: string
grantFullControl: string
grantRead: string
grantReadAcp: string
grantWrite: string
grantWriteAcp: string
extra: {
acl: string
}
}) {
let params: any = {
Bucket: query.bucket,
ACL: query.extra?.acl,
GrantFullControl: query.grantFullControl,
GrantRead: query.grantRead,
GrantReadACP: query.grantReadAcp,
GrantWrite: query.grantWrite,
GrantWriteACP: query.grantWriteAcp,
}
if (query.location) {
params["CreateBucketConfiguration"] = {
LocationConstraint: query.location,
}
}) {
let params: any = {
Bucket: query.bucket,
ACL: query.extra?.acl,
GrantFullControl: query.grantFullControl,
GrantRead: query.grantRead,
GrantReadACP: query.grantReadAcp,
GrantWrite: query.grantWrite,
GrantWriteACP: query.grantWriteAcp,
}
if (query.location) {
params["CreateBucketConfiguration"] = {
LocationConstraint: query.location,
}
}
return await this.client.createBucket(params).promise()
}
async read(query: {
bucket: string
delimiter: string
expectedBucketOwner: string
marker: string
maxKeys: number
prefix: string
}) {
const response = await this.client
.listObjects({
Bucket: query.bucket,
Delimiter: query.delimiter,
Marker: query.marker,
MaxKeys: query.maxKeys,
Prefix: query.prefix,
})
.promise()
return response.Contents
}
async readCsv(query: { bucket: string; key: string }) {
const stream = this.client
.getObject({
Bucket: query.bucket,
Key: query.key,
})
.createReadStream()
let csvError = false
return new Promise((resolve, reject) => {
stream.on("error", (err: Error) => {
reject(err)
})
const response = csv()
.fromStream(stream)
.on("error", () => {
csvError = true
})
stream.on("finish", () => {
resolve(response)
})
}).catch(err => {
if (csvError) {
throw new Error("Could not read CSV")
} else {
throw err
}
})
}
async delete(query: { bucket: string; delete: string }) {
return await this.client
.deleteObjects({
Bucket: query.bucket,
Delete: JSON.parse(query.delete),
})
.promise()
}
return await this.client.createBucket(params).promise()
}
async read(query: {
bucket: string
delimiter: string
expectedBucketOwner: string
marker: string
maxKeys: number
prefix: string
}) {
const response = await this.client
.listObjects({
Bucket: query.bucket,
Delimiter: query.delimiter,
Marker: query.marker,
MaxKeys: query.maxKeys,
Prefix: query.prefix,
})
.promise()
return response.Contents
}
async readCsv(query: { bucket: string; key: string }) {
const stream = this.client
.getObject({
Bucket: query.bucket,
Key: query.key,
})
.createReadStream()
let csvError = false
return new Promise((resolve, reject) => {
stream.on("error", (err: Error) => {
reject(err)
})
const response = csv()
.fromStream(stream)
.on("error", () => {
csvError = true
})
stream.on("finish", () => {
resolve(response)
})
}).catch(err => {
if (csvError) {
throw new Error("Could not read CSV")
} else {
throw err
}
})
}
async delete(query: { bucket: string; delete: string }) {
return await this.client
.deleteObjects({
Bucket: query.bucket,
Delete: JSON.parse(query.delete),
})
.promise()
}
}
export default {
schema: SCHEMA,
integration: S3Integration,

View file

@ -18,7 +18,7 @@ describe("S3 Integration", () => {
})
it("calls the read method with the correct params", async () => {
await config.integration.read({
await config.integration.read({
bucket: "test",
delimiter: "/",
marker: "file.txt",
@ -30,12 +30,12 @@ describe("S3 Integration", () => {
Delimiter: "/",
Marker: "file.txt",
MaxKeys: 999,
Prefix: "directory/"
Prefix: "directory/",
})
})
it("calls the create method with the correct params", async () => {
await config.integration.create({
await config.integration.create({
bucket: "test",
location: "af-south-1",
grantFullControl: "me",
@ -45,13 +45,13 @@ describe("S3 Integration", () => {
grantWriteAcp: "he",
objectLockEnabledForBucket: true,
extra: {
acl: "private"
}
acl: "private",
},
})
expect(config.integration.client.createBucket).toHaveBeenCalledWith({
Bucket: "test",
CreateBucketConfiguration: {
LocationConstraint: "af-south-1"
LocationConstraint: "af-south-1",
},
GrantFullControl: "me",
GrantRead: "him",
@ -63,8 +63,8 @@ describe("S3 Integration", () => {
})
it("does not add undefined location constraint when calling the create method", async () => {
await config.integration.create({
bucket: "test"
await config.integration.create({
bucket: "test",
})
expect(config.integration.client.createBucket).toHaveBeenCalledWith({
Bucket: "test",
@ -78,7 +78,7 @@ describe("S3 Integration", () => {
})
it("calls the delete method with the correct params ", async () => {
await config.integration.delete({
await config.integration.delete({
bucket: "test",
delete: `{
"Objects": [
@ -91,22 +91,22 @@ describe("S3 Integration", () => {
"VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
}
]
}`
}`,
})
expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
Bucket: "test",
Delete: {
Objects: [
{
Key: "HappyFace.jpg",
VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
},
Key: "HappyFace.jpg",
VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
},
{
Key: "HappyFace.jpg",
VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
}
]
}
Key: "HappyFace.jpg",
VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
},
],
},
})
})
})