lint
This commit is contained in:
parent
d69abc4569
commit
86fef02567
2 changed files with 227 additions and 228 deletions
|
@ -5,9 +5,8 @@ import {
|
||||||
DatasourceFieldType,
|
DatasourceFieldType,
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
|
|
||||||
module S3Module {
|
const AWS = require("aws-sdk")
|
||||||
const AWS = require("aws-sdk")
|
const csv = require("csvtojson")
|
||||||
const csv = require("csvtojson")
|
|
||||||
|
|
||||||
interface S3Config {
|
interface S3Config {
|
||||||
region: string
|
region: string
|
||||||
|
@ -17,138 +16,138 @@ interface S3Config {
|
||||||
endpoint?: string
|
endpoint?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
const SCHEMA: Integration = {
|
const SCHEMA: Integration = {
|
||||||
docs: "https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html",
|
docs: "https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html",
|
||||||
description:
|
description:
|
||||||
"Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
|
"Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
|
||||||
friendlyName: "Amazon S3",
|
friendlyName: "Amazon S3",
|
||||||
type: "Object store",
|
type: "Object store",
|
||||||
datasource: {
|
datasource: {
|
||||||
region: {
|
region: {
|
||||||
type: "string",
|
type: "string",
|
||||||
required: false,
|
required: false,
|
||||||
default: "us-east-1",
|
default: "us-east-1",
|
||||||
},
|
|
||||||
accessKeyId: {
|
|
||||||
type: "password",
|
|
||||||
required: true,
|
|
||||||
},
|
|
||||||
secretAccessKey: {
|
|
||||||
type: "password",
|
|
||||||
required: true,
|
|
||||||
},
|
|
||||||
endpoint: {
|
|
||||||
type: "string",
|
|
||||||
required: false,
|
|
||||||
},
|
|
||||||
signatureVersion: {
|
|
||||||
type: "string",
|
|
||||||
required: false,
|
|
||||||
default: "v4",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
query: {
|
accessKeyId: {
|
||||||
create: {
|
type: "password",
|
||||||
type: QueryType.FIELDS,
|
required: true,
|
||||||
fields: {
|
},
|
||||||
bucket: {
|
secretAccessKey: {
|
||||||
display: "New Bucket",
|
type: "password",
|
||||||
type: DatasourceFieldType.STRING,
|
required: true,
|
||||||
required: true,
|
},
|
||||||
},
|
endpoint: {
|
||||||
location: {
|
type: "string",
|
||||||
required: true,
|
required: false,
|
||||||
default: "us-east-1",
|
},
|
||||||
type: DatasourceFieldType.STRING,
|
signatureVersion: {
|
||||||
},
|
type: "string",
|
||||||
grantFullControl: {
|
required: false,
|
||||||
display: "Grant full control",
|
default: "v4",
|
||||||
type: DatasourceFieldType.STRING,
|
},
|
||||||
},
|
},
|
||||||
grantRead: {
|
query: {
|
||||||
display: "Grant read",
|
create: {
|
||||||
type: DatasourceFieldType.STRING,
|
type: QueryType.FIELDS,
|
||||||
},
|
fields: {
|
||||||
grantReadAcp: {
|
bucket: {
|
||||||
display: "Grant read ACP",
|
display: "New Bucket",
|
||||||
type: DatasourceFieldType.STRING,
|
type: DatasourceFieldType.STRING,
|
||||||
},
|
required: true,
|
||||||
grantWrite: {
|
|
||||||
display: "Grant write",
|
|
||||||
type: DatasourceFieldType.STRING,
|
|
||||||
},
|
|
||||||
grantWriteAcp: {
|
|
||||||
display: "Grant write ACP",
|
|
||||||
type: DatasourceFieldType.STRING,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
},
|
location: {
|
||||||
read: {
|
required: true,
|
||||||
type: QueryType.FIELDS,
|
default: "us-east-1",
|
||||||
fields: {
|
type: DatasourceFieldType.STRING,
|
||||||
bucket: {
|
|
||||||
type: DatasourceFieldType.STRING,
|
|
||||||
required: true,
|
|
||||||
},
|
|
||||||
delimiter: {
|
|
||||||
type: DatasourceFieldType.STRING,
|
|
||||||
},
|
|
||||||
marker: {
|
|
||||||
type: DatasourceFieldType.STRING,
|
|
||||||
},
|
|
||||||
maxKeys: {
|
|
||||||
type: DatasourceFieldType.NUMBER,
|
|
||||||
display: "Max Keys",
|
|
||||||
},
|
|
||||||
prefix: {
|
|
||||||
type: DatasourceFieldType.STRING,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
},
|
grantFullControl: {
|
||||||
readCsv: {
|
display: "Grant full control",
|
||||||
displayName: "Read CSV",
|
type: DatasourceFieldType.STRING,
|
||||||
type: QueryType.FIELDS,
|
|
||||||
fields: {
|
|
||||||
bucket: {
|
|
||||||
type: DatasourceFieldType.STRING,
|
|
||||||
required: true,
|
|
||||||
},
|
|
||||||
key: {
|
|
||||||
type: DatasourceFieldType.STRING,
|
|
||||||
required: true,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
},
|
grantRead: {
|
||||||
delete: {
|
display: "Grant read",
|
||||||
type: QueryType.FIELDS,
|
type: DatasourceFieldType.STRING,
|
||||||
fields: {
|
},
|
||||||
bucket: {
|
grantReadAcp: {
|
||||||
type: DatasourceFieldType.STRING,
|
display: "Grant read ACP",
|
||||||
required: true,
|
type: DatasourceFieldType.STRING,
|
||||||
},
|
},
|
||||||
delete: {
|
grantWrite: {
|
||||||
type: DatasourceFieldType.JSON,
|
display: "Grant write",
|
||||||
required: true,
|
type: DatasourceFieldType.STRING,
|
||||||
},
|
},
|
||||||
|
grantWriteAcp: {
|
||||||
|
display: "Grant write ACP",
|
||||||
|
type: DatasourceFieldType.STRING,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
extra: {
|
read: {
|
||||||
acl: {
|
type: QueryType.FIELDS,
|
||||||
required: false,
|
fields: {
|
||||||
displayName: "ACL",
|
bucket: {
|
||||||
type: DatasourceFieldType.LIST,
|
type: DatasourceFieldType.STRING,
|
||||||
data: {
|
required: true,
|
||||||
create: [
|
},
|
||||||
"private",
|
delimiter: {
|
||||||
"public-read",
|
type: DatasourceFieldType.STRING,
|
||||||
"public-read-write",
|
},
|
||||||
"authenticated-read",
|
marker: {
|
||||||
],
|
type: DatasourceFieldType.STRING,
|
||||||
|
},
|
||||||
|
maxKeys: {
|
||||||
|
type: DatasourceFieldType.NUMBER,
|
||||||
|
display: "Max Keys",
|
||||||
|
},
|
||||||
|
prefix: {
|
||||||
|
type: DatasourceFieldType.STRING,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
readCsv: {
|
||||||
|
displayName: "Read CSV",
|
||||||
|
type: QueryType.FIELDS,
|
||||||
|
fields: {
|
||||||
|
bucket: {
|
||||||
|
type: DatasourceFieldType.STRING,
|
||||||
|
required: true,
|
||||||
|
},
|
||||||
|
key: {
|
||||||
|
type: DatasourceFieldType.STRING,
|
||||||
|
required: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
delete: {
|
||||||
|
type: QueryType.FIELDS,
|
||||||
|
fields: {
|
||||||
|
bucket: {
|
||||||
|
type: DatasourceFieldType.STRING,
|
||||||
|
required: true,
|
||||||
|
},
|
||||||
|
delete: {
|
||||||
|
type: DatasourceFieldType.JSON,
|
||||||
|
required: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
extra: {
|
||||||
|
acl: {
|
||||||
|
required: false,
|
||||||
|
displayName: "ACL",
|
||||||
|
type: DatasourceFieldType.LIST,
|
||||||
|
data: {
|
||||||
|
create: [
|
||||||
|
"private",
|
||||||
|
"public-read",
|
||||||
|
"public-read-write",
|
||||||
|
"authenticated-read",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
class S3Integration implements IntegrationBase {
|
class S3Integration implements IntegrationBase {
|
||||||
private readonly config: S3Config
|
private readonly config: S3Config
|
||||||
|
@ -165,95 +164,95 @@ class S3Integration implements IntegrationBase {
|
||||||
this.client = new AWS.S3(this.config)
|
this.client = new AWS.S3(this.config)
|
||||||
}
|
}
|
||||||
|
|
||||||
async create(query: {
|
async create(query: {
|
||||||
bucket: string
|
bucket: string
|
||||||
location: string
|
location: string
|
||||||
grantFullControl: string
|
grantFullControl: string
|
||||||
grantRead: string
|
grantRead: string
|
||||||
grantReadAcp: string
|
grantReadAcp: string
|
||||||
grantWrite: string
|
grantWrite: string
|
||||||
grantWriteAcp: string
|
grantWriteAcp: string
|
||||||
extra: {
|
extra: {
|
||||||
acl: string
|
acl: string
|
||||||
|
}
|
||||||
|
}) {
|
||||||
|
let params: any = {
|
||||||
|
Bucket: query.bucket,
|
||||||
|
ACL: query.extra?.acl,
|
||||||
|
GrantFullControl: query.grantFullControl,
|
||||||
|
GrantRead: query.grantRead,
|
||||||
|
GrantReadACP: query.grantReadAcp,
|
||||||
|
GrantWrite: query.grantWrite,
|
||||||
|
GrantWriteACP: query.grantWriteAcp,
|
||||||
|
}
|
||||||
|
if (query.location) {
|
||||||
|
params["CreateBucketConfiguration"] = {
|
||||||
|
LocationConstraint: query.location,
|
||||||
}
|
}
|
||||||
}) {
|
|
||||||
let params: any = {
|
|
||||||
Bucket: query.bucket,
|
|
||||||
ACL: query.extra?.acl,
|
|
||||||
GrantFullControl: query.grantFullControl,
|
|
||||||
GrantRead: query.grantRead,
|
|
||||||
GrantReadACP: query.grantReadAcp,
|
|
||||||
GrantWrite: query.grantWrite,
|
|
||||||
GrantWriteACP: query.grantWriteAcp,
|
|
||||||
}
|
|
||||||
if (query.location) {
|
|
||||||
params["CreateBucketConfiguration"] = {
|
|
||||||
LocationConstraint: query.location,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return await this.client.createBucket(params).promise()
|
|
||||||
}
|
|
||||||
|
|
||||||
async read(query: {
|
|
||||||
bucket: string
|
|
||||||
delimiter: string
|
|
||||||
expectedBucketOwner: string
|
|
||||||
marker: string
|
|
||||||
maxKeys: number
|
|
||||||
prefix: string
|
|
||||||
}) {
|
|
||||||
const response = await this.client
|
|
||||||
.listObjects({
|
|
||||||
Bucket: query.bucket,
|
|
||||||
Delimiter: query.delimiter,
|
|
||||||
Marker: query.marker,
|
|
||||||
MaxKeys: query.maxKeys,
|
|
||||||
Prefix: query.prefix,
|
|
||||||
})
|
|
||||||
.promise()
|
|
||||||
return response.Contents
|
|
||||||
}
|
|
||||||
|
|
||||||
async readCsv(query: { bucket: string; key: string }) {
|
|
||||||
const stream = this.client
|
|
||||||
.getObject({
|
|
||||||
Bucket: query.bucket,
|
|
||||||
Key: query.key,
|
|
||||||
})
|
|
||||||
.createReadStream()
|
|
||||||
|
|
||||||
let csvError = false
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
stream.on("error", (err: Error) => {
|
|
||||||
reject(err)
|
|
||||||
})
|
|
||||||
const response = csv()
|
|
||||||
.fromStream(stream)
|
|
||||||
.on("error", () => {
|
|
||||||
csvError = true
|
|
||||||
})
|
|
||||||
stream.on("finish", () => {
|
|
||||||
resolve(response)
|
|
||||||
})
|
|
||||||
}).catch(err => {
|
|
||||||
if (csvError) {
|
|
||||||
throw new Error("Could not read CSV")
|
|
||||||
} else {
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async delete(query: { bucket: string; delete: string }) {
|
|
||||||
return await this.client
|
|
||||||
.deleteObjects({
|
|
||||||
Bucket: query.bucket,
|
|
||||||
Delete: JSON.parse(query.delete),
|
|
||||||
})
|
|
||||||
.promise()
|
|
||||||
}
|
}
|
||||||
|
return await this.client.createBucket(params).promise()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async read(query: {
|
||||||
|
bucket: string
|
||||||
|
delimiter: string
|
||||||
|
expectedBucketOwner: string
|
||||||
|
marker: string
|
||||||
|
maxKeys: number
|
||||||
|
prefix: string
|
||||||
|
}) {
|
||||||
|
const response = await this.client
|
||||||
|
.listObjects({
|
||||||
|
Bucket: query.bucket,
|
||||||
|
Delimiter: query.delimiter,
|
||||||
|
Marker: query.marker,
|
||||||
|
MaxKeys: query.maxKeys,
|
||||||
|
Prefix: query.prefix,
|
||||||
|
})
|
||||||
|
.promise()
|
||||||
|
return response.Contents
|
||||||
|
}
|
||||||
|
|
||||||
|
async readCsv(query: { bucket: string; key: string }) {
|
||||||
|
const stream = this.client
|
||||||
|
.getObject({
|
||||||
|
Bucket: query.bucket,
|
||||||
|
Key: query.key,
|
||||||
|
})
|
||||||
|
.createReadStream()
|
||||||
|
|
||||||
|
let csvError = false
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
stream.on("error", (err: Error) => {
|
||||||
|
reject(err)
|
||||||
|
})
|
||||||
|
const response = csv()
|
||||||
|
.fromStream(stream)
|
||||||
|
.on("error", () => {
|
||||||
|
csvError = true
|
||||||
|
})
|
||||||
|
stream.on("finish", () => {
|
||||||
|
resolve(response)
|
||||||
|
})
|
||||||
|
}).catch(err => {
|
||||||
|
if (csvError) {
|
||||||
|
throw new Error("Could not read CSV")
|
||||||
|
} else {
|
||||||
|
throw err
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete(query: { bucket: string; delete: string }) {
|
||||||
|
return await this.client
|
||||||
|
.deleteObjects({
|
||||||
|
Bucket: query.bucket,
|
||||||
|
Delete: JSON.parse(query.delete),
|
||||||
|
})
|
||||||
|
.promise()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
schema: SCHEMA,
|
schema: SCHEMA,
|
||||||
integration: S3Integration,
|
integration: S3Integration,
|
||||||
|
|
|
@ -18,7 +18,7 @@ describe("S3 Integration", () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
it("calls the read method with the correct params", async () => {
|
it("calls the read method with the correct params", async () => {
|
||||||
await config.integration.read({
|
await config.integration.read({
|
||||||
bucket: "test",
|
bucket: "test",
|
||||||
delimiter: "/",
|
delimiter: "/",
|
||||||
marker: "file.txt",
|
marker: "file.txt",
|
||||||
|
@ -30,12 +30,12 @@ describe("S3 Integration", () => {
|
||||||
Delimiter: "/",
|
Delimiter: "/",
|
||||||
Marker: "file.txt",
|
Marker: "file.txt",
|
||||||
MaxKeys: 999,
|
MaxKeys: 999,
|
||||||
Prefix: "directory/"
|
Prefix: "directory/",
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it("calls the create method with the correct params", async () => {
|
it("calls the create method with the correct params", async () => {
|
||||||
await config.integration.create({
|
await config.integration.create({
|
||||||
bucket: "test",
|
bucket: "test",
|
||||||
location: "af-south-1",
|
location: "af-south-1",
|
||||||
grantFullControl: "me",
|
grantFullControl: "me",
|
||||||
|
@ -45,13 +45,13 @@ describe("S3 Integration", () => {
|
||||||
grantWriteAcp: "he",
|
grantWriteAcp: "he",
|
||||||
objectLockEnabledForBucket: true,
|
objectLockEnabledForBucket: true,
|
||||||
extra: {
|
extra: {
|
||||||
acl: "private"
|
acl: "private",
|
||||||
}
|
},
|
||||||
})
|
})
|
||||||
expect(config.integration.client.createBucket).toHaveBeenCalledWith({
|
expect(config.integration.client.createBucket).toHaveBeenCalledWith({
|
||||||
Bucket: "test",
|
Bucket: "test",
|
||||||
CreateBucketConfiguration: {
|
CreateBucketConfiguration: {
|
||||||
LocationConstraint: "af-south-1"
|
LocationConstraint: "af-south-1",
|
||||||
},
|
},
|
||||||
GrantFullControl: "me",
|
GrantFullControl: "me",
|
||||||
GrantRead: "him",
|
GrantRead: "him",
|
||||||
|
@ -63,8 +63,8 @@ describe("S3 Integration", () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
it("does not add undefined location constraint when calling the create method", async () => {
|
it("does not add undefined location constraint when calling the create method", async () => {
|
||||||
await config.integration.create({
|
await config.integration.create({
|
||||||
bucket: "test"
|
bucket: "test",
|
||||||
})
|
})
|
||||||
expect(config.integration.client.createBucket).toHaveBeenCalledWith({
|
expect(config.integration.client.createBucket).toHaveBeenCalledWith({
|
||||||
Bucket: "test",
|
Bucket: "test",
|
||||||
|
@ -78,7 +78,7 @@ describe("S3 Integration", () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
it("calls the delete method with the correct params ", async () => {
|
it("calls the delete method with the correct params ", async () => {
|
||||||
await config.integration.delete({
|
await config.integration.delete({
|
||||||
bucket: "test",
|
bucket: "test",
|
||||||
delete: `{
|
delete: `{
|
||||||
"Objects": [
|
"Objects": [
|
||||||
|
@ -91,22 +91,22 @@ describe("S3 Integration", () => {
|
||||||
"VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
|
"VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}`
|
}`,
|
||||||
})
|
})
|
||||||
expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
|
expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
|
||||||
Bucket: "test",
|
Bucket: "test",
|
||||||
Delete: {
|
Delete: {
|
||||||
Objects: [
|
Objects: [
|
||||||
{
|
{
|
||||||
Key: "HappyFace.jpg",
|
Key: "HappyFace.jpg",
|
||||||
VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
|
VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Key: "HappyFace.jpg",
|
Key: "HappyFace.jpg",
|
||||||
VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
|
VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
|
||||||
}
|
},
|
||||||
]
|
],
|
||||||
}
|
},
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
Loading…
Reference in a new issue