CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-06-28 02:50:50 +12:00
Code Issues Projects Releases Wiki Activity

lint

Browse source
This commit is contained in:
Mel O'Hagan 2022-10-26 16:44:25 +01:00
parent 4661c083e2
commit f240f5ffbf
2 changed files with 227 additions and 228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

419
packages/server/src/integrations/s3.ts
View file

@ -5,9 +5,8 @@ import {
DatasourceFieldType, DatasourceFieldType,
} from "@budibase/types" } from "@budibase/types"
module S3Module { const AWS = require("aws-sdk")
const AWS = require("aws-sdk") const csv = require("csvtojson")
const csv = require("csvtojson")
interface S3Config { interface S3Config {
region: string region: string
@ -17,138 +16,138 @@ interface S3Config {
endpoint?: string endpoint?: string
} }
const SCHEMA: Integration = { const SCHEMA: Integration = {
docs: "https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", docs: "https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html",
description: description:
"Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.", "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
friendlyName: "Amazon S3", friendlyName: "Amazon S3",
type: "Object store", type: "Object store",
datasource: { datasource: {
region: { region: {
type: "string", type: "string",
required: false, required: false,
default: "us-east-1", default: "us-east-1",
},
accessKeyId: {
type: "password",
required: true,
},
secretAccessKey: {
type: "password",
required: true,
},
endpoint: {
type: "string",
required: false,
},
signatureVersion: {
type: "string",
required: false,
default: "v4",
},
}, },
query: { accessKeyId: {
create: { type: "password",
type: QueryType.FIELDS, required: true,
fields: { },
bucket: { secretAccessKey: {
display: "New Bucket", type: "password",
type: DatasourceFieldType.STRING, required: true,
required: true, },
}, endpoint: {
location: { type: "string",
required: true, required: false,
default: "us-east-1", },
type: DatasourceFieldType.STRING, signatureVersion: {
}, type: "string",
grantFullControl: { required: false,
display: "Grant full control", default: "v4",
type: DatasourceFieldType.STRING, },
}, },
grantRead: { query: {
display: "Grant read", create: {
type: DatasourceFieldType.STRING, type: QueryType.FIELDS,
}, fields: {
grantReadAcp: { bucket: {
display: "Grant read ACP", display: "New Bucket",
type: DatasourceFieldType.STRING, type: DatasourceFieldType.STRING,
}, required: true,
grantWrite: {
display: "Grant write",
type: DatasourceFieldType.STRING,
},
grantWriteAcp: {
display: "Grant write ACP",
type: DatasourceFieldType.STRING,
},
}, },
}, location: {
read: { required: true,
type: QueryType.FIELDS, default: "us-east-1",
fields: { type: DatasourceFieldType.STRING,
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
delimiter: {
type: DatasourceFieldType.STRING,
},
marker: {
type: DatasourceFieldType.STRING,
},
maxKeys: {
type: DatasourceFieldType.NUMBER,
display: "Max Keys",
},
prefix: {
type: DatasourceFieldType.STRING,
},
}, },
}, grantFullControl: {
readCsv: { display: "Grant full control",
displayName: "Read CSV", type: DatasourceFieldType.STRING,
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
key: {
type: DatasourceFieldType.STRING,
required: true,
},
}, },
}, grantRead: {
delete: { display: "Grant read",
type: QueryType.FIELDS, type: DatasourceFieldType.STRING,
fields: { },
bucket: { grantReadAcp: {
type: DatasourceFieldType.STRING, display: "Grant read ACP",
required: true, type: DatasourceFieldType.STRING,
}, },
delete: { grantWrite: {
type: DatasourceFieldType.JSON, display: "Grant write",
required: true, type: DatasourceFieldType.STRING,
}, },
grantWriteAcp: {
display: "Grant write ACP",
type: DatasourceFieldType.STRING,
}, },
}, },
}, },
extra: { read: {
acl: { type: QueryType.FIELDS,
required: false, fields: {
displayName: "ACL", bucket: {
type: DatasourceFieldType.LIST, type: DatasourceFieldType.STRING,
data: { required: true,
create: [ },
"private", delimiter: {
"public-read", type: DatasourceFieldType.STRING,
"public-read-write", },
"authenticated-read", marker: {
], type: DatasourceFieldType.STRING,
},
maxKeys: {
type: DatasourceFieldType.NUMBER,
display: "Max Keys",
},
prefix: {
type: DatasourceFieldType.STRING,
}, },
}, },
}, },
} readCsv: {
displayName: "Read CSV",
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
key: {
type: DatasourceFieldType.STRING,
required: true,
},
},
},
delete: {
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
delete: {
type: DatasourceFieldType.JSON,
required: true,
},
},
},
},
extra: {
acl: {
required: false,
displayName: "ACL",
type: DatasourceFieldType.LIST,
data: {
create: [
"private",
"public-read",
"public-read-write",
"authenticated-read",
],
},
},
},
}
class S3Integration implements IntegrationBase { class S3Integration implements IntegrationBase {
private readonly config: S3Config private readonly config: S3Config
@ -165,95 +164,95 @@ class S3Integration implements IntegrationBase {
this.client = new AWS.S3(this.config) this.client = new AWS.S3(this.config)
} }
async create(query: { async create(query: {
bucket: string bucket: string
location: string location: string
grantFullControl: string grantFullControl: string
grantRead: string grantRead: string
grantReadAcp: string grantReadAcp: string
grantWrite: string grantWrite: string
grantWriteAcp: string grantWriteAcp: string
extra: { extra: {
acl: string acl: string
}
}) {
let params: any = {
Bucket: query.bucket,
ACL: query.extra?.acl,
GrantFullControl: query.grantFullControl,
GrantRead: query.grantRead,
GrantReadACP: query.grantReadAcp,
GrantWrite: query.grantWrite,
GrantWriteACP: query.grantWriteAcp,
}
if (query.location) {
params["CreateBucketConfiguration"] = {
LocationConstraint: query.location,
} }
}) {
let params: any = {
Bucket: query.bucket,
ACL: query.extra?.acl,
GrantFullControl: query.grantFullControl,
GrantRead: query.grantRead,
GrantReadACP: query.grantReadAcp,
GrantWrite: query.grantWrite,
GrantWriteACP: query.grantWriteAcp,
}
if (query.location) {
params["CreateBucketConfiguration"] = {
LocationConstraint: query.location,
}
}
return await this.client.createBucket(params).promise()
}
async read(query: {
bucket: string
delimiter: string
expectedBucketOwner: string
marker: string
maxKeys: number
prefix: string
}) {
const response = await this.client
.listObjects({
Bucket: query.bucket,
Delimiter: query.delimiter,
Marker: query.marker,
MaxKeys: query.maxKeys,
Prefix: query.prefix,
})
.promise()
return response.Contents
}
async readCsv(query: { bucket: string; key: string }) {
const stream = this.client
.getObject({
Bucket: query.bucket,
Key: query.key,
})
.createReadStream()
let csvError = false
return new Promise((resolve, reject) => {
stream.on("error", (err: Error) => {
reject(err)
})
const response = csv()
.fromStream(stream)
.on("error", () => {
csvError = true
})
stream.on("finish", () => {
resolve(response)
})
}).catch(err => {
if (csvError) {
throw new Error("Could not read CSV")
} else {
throw err
}
})
}
async delete(query: { bucket: string; delete: string }) {
return await this.client
.deleteObjects({
Bucket: query.bucket,
Delete: JSON.parse(query.delete),
})
.promise()
} }
return await this.client.createBucket(params).promise()
} }
async read(query: {
bucket: string
delimiter: string
expectedBucketOwner: string
marker: string
maxKeys: number
prefix: string
}) {
const response = await this.client
.listObjects({
Bucket: query.bucket,
Delimiter: query.delimiter,
Marker: query.marker,
MaxKeys: query.maxKeys,
Prefix: query.prefix,
})
.promise()
return response.Contents
}
async readCsv(query: { bucket: string; key: string }) {
const stream = this.client
.getObject({
Bucket: query.bucket,
Key: query.key,
})
.createReadStream()
let csvError = false
return new Promise((resolve, reject) => {
stream.on("error", (err: Error) => {
reject(err)
})
const response = csv()
.fromStream(stream)
.on("error", () => {
csvError = true
})
stream.on("finish", () => {
resolve(response)
})
}).catch(err => {
if (csvError) {
throw new Error("Could not read CSV")
} else {
throw err
}
})
}
async delete(query: { bucket: string; delete: string }) {
return await this.client
.deleteObjects({
Bucket: query.bucket,
Delete: JSON.parse(query.delete),
})
.promise()
}
}
export default { export default {
schema: SCHEMA, schema: SCHEMA,
integration: S3Integration, integration: S3Integration,

36
packages/server/src/integrations/tests/s3.spec.ts
View file

@ -18,7 +18,7 @@ describe("S3 Integration", () => {
}) })
it("calls the read method with the correct params", async () => { it("calls the read method with the correct params", async () => {
await config.integration.read({ await config.integration.read({
bucket: "test", bucket: "test",
delimiter: "/", delimiter: "/",
marker: "file.txt", marker: "file.txt",
@ -30,12 +30,12 @@ describe("S3 Integration", () => {
Delimiter: "/", Delimiter: "/",
Marker: "file.txt", Marker: "file.txt",
MaxKeys: 999, MaxKeys: 999,
Prefix: "directory/" Prefix: "directory/",
}) })
}) })
it("calls the create method with the correct params", async () => { it("calls the create method with the correct params", async () => {
await config.integration.create({ await config.integration.create({
bucket: "test", bucket: "test",
location: "af-south-1", location: "af-south-1",
grantFullControl: "me", grantFullControl: "me",
@ -45,13 +45,13 @@ describe("S3 Integration", () => {
grantWriteAcp: "he", grantWriteAcp: "he",
objectLockEnabledForBucket: true, objectLockEnabledForBucket: true,
extra: { extra: {
acl: "private" acl: "private",
} },
}) })
expect(config.integration.client.createBucket).toHaveBeenCalledWith({ expect(config.integration.client.createBucket).toHaveBeenCalledWith({
Bucket: "test", Bucket: "test",
CreateBucketConfiguration: { CreateBucketConfiguration: {
LocationConstraint: "af-south-1" LocationConstraint: "af-south-1",
}, },
GrantFullControl: "me", GrantFullControl: "me",
GrantRead: "him", GrantRead: "him",
@ -63,8 +63,8 @@ describe("S3 Integration", () => {
}) })
it("does not add undefined location constraint when calling the create method", async () => { it("does not add undefined location constraint when calling the create method", async () => {
await config.integration.create({ await config.integration.create({
bucket: "test" bucket: "test",
}) })
expect(config.integration.client.createBucket).toHaveBeenCalledWith({ expect(config.integration.client.createBucket).toHaveBeenCalledWith({
Bucket: "test", Bucket: "test",
@ -78,7 +78,7 @@ describe("S3 Integration", () => {
}) })
it("calls the delete method with the correct params ", async () => { it("calls the delete method with the correct params ", async () => {
await config.integration.delete({ await config.integration.delete({
bucket: "test", bucket: "test",
delete: `{ delete: `{
"Objects": [ "Objects": [
@ -91,22 +91,22 @@ describe("S3 Integration", () => {
"VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd" "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
} }
] ]
}` }`,
}) })
expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({ expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
Bucket: "test", Bucket: "test",
Delete: { Delete: {
Objects: [ Objects: [
{ {
Key: "HappyFace.jpg", Key: "HappyFace.jpg",
VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b" VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
}, },
{ {
Key: "HappyFace.jpg", Key: "HappyFace.jpg",
VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd" VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
} },
] ],
} },
}) })
}) })
}) })