WIP

packages/server/package.json

@@ -95,7 +95,7 @@
     "bcryptjs": "2.4.3",
     "bull": "4.8.5",
     "chmodr": "1.2.0",
-    "csvtojson": "2.0.10",
+    "csvtojson": "^2.0.10",
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",
     "download": "8.0.0",

packages/server/src/integrations/s3.ts

@@ -1,7 +1,13 @@
-import { Integration, QueryType, IntegrationBase, DatasourceFieldType } from "@budibase/types"
+import {
+  Integration,
+  QueryType,
+  IntegrationBase,
+  DatasourceFieldType,
+} from "@budibase/types"
 
 module S3Module {
   const AWS = require("aws-sdk")
+  const csv = require("csvtojson")
 
   interface S3Config {
     region: string
@@ -63,8 +69,22 @@ module S3Module {
             type: DatasourceFieldType.STRING,
           },
         },
-      }
-    }
+      },
+      readCsv: {
+        displayName: "Read CSV",
+        type: QueryType.FIELDS,
+        fields: {
+          bucket: {
+            type: DatasourceFieldType.STRING,
+            required: true,
+          },
+          key: {
+            type: DatasourceFieldType.STRING,
+            required: true,
+          },
+        },
+      },
+    },
   }
 
   class S3Integration implements IntegrationBase {
@@ -82,13 +102,13 @@ module S3Module {
       this.client = new AWS.S3(this.config)
     }
 
-    async read(query: { 
-      bucket: string, 
-      delimiter: string, 
-      expectedBucketOwner: string, 
-      marker: string, 
-      maxKeys: number, 
-      prefix: string, 
+    async read(query: {
+      bucket: string
+      delimiter: string
+      expectedBucketOwner: string
+      marker: string
+      maxKeys: number
+      prefix: string
     }) {
       const response = await this.client
         .listObjects({
@@ -101,6 +121,32 @@ module S3Module {
         .promise()
       return response.Contents
     }
+
+    async readCsv(query: { bucket: string; key: string }) {
+      let streamErr: string | undefined = undefined
+      const stream = this.client
+        .getObject({
+          Bucket: query.bucket,
+          Key: query.key,
+        })
+        .createReadStream()
+        .on("error", (err: string) => {
+          //stream.destroy()
+          //
+          console.log("err ", err)
+          streamErr = "ERROR"
+        })
+
+      if (streamErr) {
+        throw new Error("ERROR")
+      }
+
+      try {
+        return await csv().fromStream(stream)
+      } catch (err) {
+        throw new Error("Failed to read CSV")
+      }
+    }
   }
 
   module.exports = {

packages/server/yarn.lock

@@ -4924,7 +4924,7 @@ cssstyle@^2.3.0:
   dependencies:
     cssom "~0.3.6"
 
-csvtojson@2.0.10:
+csvtojson@^2.0.10:
   version "2.0.10"
   resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz#11e7242cc630da54efce7958a45f443210357574"
   integrity sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==