REST file handling and SMTP automation block attachments (#13403)

* handle files in rest connector

* fetch presigned url and return

* further updates to handle files in rest connector

* remove unused important and fix extension bug

* wrong expiry param

* tests

* add const for temp bucket

* handle ttl on bucket

* more bucket ttl work

* split out fileresponse and xmlresponse into utils

* lint

* remove log

* fix tests

* some pr comments

* update function naming and lint

* adding back needed response for frontend

* use fsp

* handle different content-disposition and potential path traversal

* add test container for s3 / minio

* add test case for filename* and ascii filenames

* move tests into separate describe

* remove log

* up timeout

* switch to minio image instead of localstack

* use minio image instead of s3 for testing

* stream file upload instead

* use streamUpload and update signatures

* update bucketcreate return

* throw real error

* tidy up

* pro

* pro ref fix?

* pro fix

* pro fix?

* move minio test provider to backend-core

* update email builder to allow attachments

* testing for sending files via smtp

* use backend-core minio test container in server

* handle different types of url

* fix minio test provider

* test with container host

* lint

* try different hostname?

* Revert "try different hostname?"

This reverts commit cfefdb8ded.

* fix issue with fetching of signed url with test minio

* update autoamtion attachments to take filename and url

* fix tests

* pro ref

* fix parsing of url object

* pr comments and linting

* pro ref

* fix pro again

* fix pro

* account-portal

* fix null issue

* fix ref

* ref

* When sending a file attachment in email fetch it directly from our object store

* add more checks to ensure we're working with a signed url

* update test to account for direct object store read

* formatting

* fix time issues within test

* update bucket and path extraction to regex

* use const in regex

* pro

* Updating TTL handling in upload functions (#13539)

* Updating TTL handling in upload functions

* describe ttl type

* account for ttl creation in existing buckets and update types

* fix tests

* pro

* pro

packages/backend-core/src/environment.ts

@@ -29,6 +29,7 @@ const DefaultBucketName = {
   TEMPLATES: "templates",
   GLOBAL: "global",
   PLUGINS: "plugins",
+  TEMP: "tmp-file-attachments",
 }
 
 const selfHosted = !!parseInt(process.env.SELF_HOSTED || "")
@@ -146,6 +147,7 @@ const environment = {
     process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL,
   PLUGIN_BUCKET_NAME:
     process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
+  TEMP_BUCKET_NAME: process.env.TEMP_BUCKET_NAME || DefaultBucketName.TEMP,
   USE_COUCH: process.env.USE_COUCH || true,
   MOCK_REDIS: process.env.MOCK_REDIS,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,

packages/backend-core/src/objectStore/objectStore.ts

@@ -7,31 +7,41 @@ import tar from "tar-fs"
 import zlib from "zlib"
 import { promisify } from "util"
 import { join } from "path"
-import fs, { ReadStream } from "fs"
+import fs, { PathLike, ReadStream } from "fs"
 import env from "../environment"
-import { budibaseTempDir } from "./utils"
+import { bucketTTLConfig, budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
+import fsp from "fs/promises"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
 const STATE = {
   bucketCreationPromises: {},
 }
+const signedFilePrefix = "/files/signed"
 
 type ListParams = {
   ContinuationToken?: string
 }
 
-type UploadParams = {
+type BaseUploadParams = {
   bucket: string
   filename: string
-  path: string
   type?: string | null
-  // can be undefined, we will remove it
+  metadata?: { [key: string]: string | undefined }
-  metadata?: {
+  body?: ReadableStream | Buffer
-    [key: string]: string | undefined
+  ttl?: number
+  addTTL?: boolean
+  extra?: any
 }
+
+type UploadParams = BaseUploadParams & {
+  path?: string | PathLike
+}
+
+type StreamUploadParams = BaseUploadParams & {
+  stream: ReadStream
 }
 
 const CONTENT_TYPE_MAP: any = {
@@ -41,6 +51,8 @@ const CONTENT_TYPE_MAP: any = {
   js: "application/javascript",
   json: "application/json",
   gz: "application/gzip",
+  svg: "image/svg+xml",
+  form: "multipart/form-data",
 }
 
 const STRING_CONTENT_TYPES = [
@@ -105,7 +117,10 @@ export function ObjectStore(
  * Given an object store and a bucket name this will make sure the bucket exists,
  * if it does not exist then it will create it.
  */
-export async function makeSureBucketExists(client: any, bucketName: string) {
+export async function createBucketIfNotExists(
+  client: any,
+  bucketName: string
+): Promise<{ created: boolean; exists: boolean }> {
   bucketName = sanitizeBucket(bucketName)
   try {
     await client
@@ -113,15 +128,16 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
         Bucket: bucketName,
       })
       .promise()
+    return { created: false, exists: true }
   } catch (err: any) {
     const promises: any = STATE.bucketCreationPromises
     const doesntExist = err.statusCode === 404,
       noAccess = err.statusCode === 403
     if (promises[bucketName]) {
       await promises[bucketName]
+      return { created: false, exists: true }
     } else if (doesntExist || noAccess) {
       if (doesntExist) {
-        // bucket doesn't exist create it
         promises[bucketName] = client
           .createBucket({
             Bucket: bucketName,
@@ -129,13 +145,15 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
           .promise()
         await promises[bucketName]
         delete promises[bucketName]
+        return { created: true, exists: false }
+      } else {
+        throw new Error("Access denied to object store bucket." + err)
       }
     } else {
       throw new Error("Unable to write to object store bucket.")
     }
   }
 }
-
 /**
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
@@ -146,12 +164,22 @@ export async function upload({
   path,
   type,
   metadata,
+  body,
+  ttl,
 }: UploadParams) {
   const extension = filename.split(".").pop()
-  const fileBytes = fs.readFileSync(path)
+
+  const fileBytes = path ? (await fsp.open(path)).createReadStream() : body
 
   const objectStore = ObjectStore(bucketName)
-  await makeSureBucketExists(objectStore, bucketName)
+  const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
+
+  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+    let ttlConfig = bucketTTLConfig(bucketName, ttl)
+    if (objectStore.putBucketLifecycleConfiguration) {
+      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
+    }
+  }
 
   let contentType = type
   if (!contentType) {
@@ -174,6 +202,7 @@ export async function upload({
     }
     config.Metadata = metadata
   }
+
   return objectStore.upload(config).promise()
 }
 
@@ -181,14 +210,24 @@ export async function upload({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-export async function streamUpload(
+export async function streamUpload({
-  bucketName: string,
+  bucket: bucketName,
-  filename: string,
+  stream,
-  stream: ReadStream | ReadableStream,
+  filename,
-  extra = {}
+  type,
-) {
+  extra,
+  ttl,
+}: StreamUploadParams) {
+  const extension = filename.split(".").pop()
   const objectStore = ObjectStore(bucketName)
-  await makeSureBucketExists(objectStore, bucketName)
+  const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
+
+  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+    let ttlConfig = bucketTTLConfig(bucketName, ttl)
+    if (objectStore.putBucketLifecycleConfiguration) {
+      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
+    }
+  }
 
   // Set content type for certain known extensions
   if (filename?.endsWith(".js")) {
@@ -203,10 +242,18 @@ export async function streamUpload(
     }
   }
 
+  let contentType = type
+  if (!contentType) {
+    contentType = extension
+      ? CONTENT_TYPE_MAP[extension.toLowerCase()]
+      : CONTENT_TYPE_MAP.txt
+  }
+
   const params = {
     Bucket: sanitizeBucket(bucketName),
     Key: sanitizeKey(filename),
     Body: stream,
+    ContentType: contentType,
     ...extra,
   }
   return objectStore.upload(params).promise()
@@ -286,7 +333,7 @@ export function getPresignedUrl(
     const signedUrl = new URL(url)
     const path = signedUrl.pathname
     const query = signedUrl.search
-    return `/files/signed${path}${query}`
+    return `${signedFilePrefix}${path}${query}`
   }
 }
 
@@ -341,7 +388,7 @@ export async function retrieveDirectory(bucketName: string, path: string) {
  */
 export async function deleteFile(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
-  await makeSureBucketExists(objectStore, bucketName)
+  await createBucketIfNotExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
     Key: sanitizeKey(filepath),
@@ -351,7 +398,7 @@ export async function deleteFile(bucketName: string, filepath: string) {
 
 export async function deleteFiles(bucketName: string, filepaths: string[]) {
   const objectStore = ObjectStore(bucketName)
-  await makeSureBucketExists(objectStore, bucketName)
+  await createBucketIfNotExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
     Delete: {
@@ -412,7 +459,13 @@ export async function uploadDirectory(
     if (file.isDirectory()) {
       uploads.push(uploadDirectory(bucketName, local, path))
     } else {
-      uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
+      uploads.push(
+        streamUpload({
+          bucket: bucketName,
+          filename: path,
+          stream: fs.createReadStream(local),
+        })
+      )
     }
   }
   await Promise.all(uploads)
@@ -467,3 +520,23 @@ export async function getReadStream(
   }
   return client.getObject(params).createReadStream()
 }
+
+/*
+Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
+the bucket and the path from it
+*/
+export function extractBucketAndPath(
+  url: string
+): { bucket: string; path: string } | null {
+  const baseUrl = url.split("?")[0]
+
+  const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
+  const match = baseUrl.match(regex)
+
+  if (match && match.groups) {
+    const { bucket, path } = match.groups
+    return { bucket, path }
+  }
+
+  return null
+}

packages/backend-core/src/objectStore/utils.ts

@@ -2,6 +2,7 @@ import { join } from "path"
 import { tmpdir } from "os"
 import fs from "fs"
 import env from "../environment"
+import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
 
 /****************************************************
  *      NOTE: When adding a new bucket - name       *
@@ -15,6 +16,7 @@ export const ObjectStoreBuckets = {
   TEMPLATES: env.TEMPLATES_BUCKET_NAME,
   GLOBAL: env.GLOBAL_BUCKET_NAME,
   PLUGINS: env.PLUGIN_BUCKET_NAME,
+  TEMP: env.TEMP_BUCKET_NAME,
 }
 
 const bbTmp = join(tmpdir(), ".budibase")
@@ -29,3 +31,27 @@ try {
 export function budibaseTempDir() {
   return bbTmp
 }
+
+export const bucketTTLConfig = (
+  bucketName: string,
+  days: number
+): PutBucketLifecycleConfigurationRequest => {
+  const lifecycleRule = {
+    ID: `${bucketName}-ExpireAfter${days}days`,
+    Prefix: "",
+    Status: "Enabled",
+    Expiration: {
+      Days: days,
+    },
+  }
+  const lifecycleConfiguration = {
+    Rules: [lifecycleRule],
+  }
+
+  const params = {
+    Bucket: bucketName,
+    LifecycleConfiguration: lifecycleConfiguration,
+  }
+
+  return params
+}

packages/backend-core/tests/core/utilities/index.ts

@@ -4,3 +4,6 @@ export { generator } from "./structures"
 export * as testContainerUtils from "./testContainerUtils"
 export * as utils from "./utils"
 export * from "./jestUtils"
+import * as minio from "./minio"
+
+export const objectStoreTestProviders = { minio }

packages/backend-core/tests/core/utilities/minio.ts

@@ -0,0 +1,34 @@
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
+import env from "../../../src/environment"
+
+let container: StartedTestContainer | undefined
+
+class ObjectStoreWaitStrategy extends AbstractWaitStrategy {
+  async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
+    const logs = Wait.forListeningPorts()
+    await logs.waitUntilReady(container, boundPorts, startTime)
+  }
+}
+
+export async function start(): Promise<void> {
+  container = await new GenericContainer("minio/minio")
+    .withExposedPorts(9000)
+    .withCommand(["server", "/data"])
+    .withEnvironment({
+      MINIO_ACCESS_KEY: "budibase",
+      MINIO_SECRET_KEY: "budibase",
+    })
+    .withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000))
+    .start()
+
+  const port = container.getMappedPort(9000)
+  env._set("MINIO_URL", `http://0.0.0.0:${port}`)
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -32,6 +32,7 @@
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte"
   import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte"
+  import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { BindingHelpers, BindingType } from "components/common/bindings/utils"
   import {
     bindingsToCompletions,
@@ -356,7 +357,8 @@
       value.customType !== "queryParams" &&
       value.customType !== "cron" &&
       value.customType !== "triggerSchema" &&
-      value.customType !== "automationFields"
+      value.customType !== "automationFields" &&
+      value.type !== "attachment"
     )
   }
 
@@ -372,6 +374,15 @@
       console.error(error)
     }
   })
+  const handleAttachmentParams = keyValuObj => {
+    let params = {}
+    if (keyValuObj?.length) {
+      for (let param of keyValuObj) {
+        params[param.url] = param.filename
+      }
+    }
+    return params
+  }
 </script>
 
 <div class="fields">
@@ -437,6 +448,33 @@
               value={inputData[key]}
               options={Object.keys(table?.schema || {})}
             />
+          {:else if value.type === "attachment"}
+            <div class="attachment-field-wrapper">
+              <div class="label-wrapper">
+                <Label>{label}</Label>
+              </div>
+              <div class="attachment-field-width">
+                <KeyValueBuilder
+                  on:change={e =>
+                    onChange(
+                      {
+                        detail: e.detail.map(({ name, value }) => ({
+                          url: name,
+                          filename: value,
+                        })),
+                      },
+                      key
+                    )}
+                  object={handleAttachmentParams(inputData[key])}
+                  allowJS
+                  {bindings}
+                  keyBindings
+                  customButtonText={"Add attachment"}
+                  keyPlaceholder={"URL"}
+                  valuePlaceholder={"Filename"}
+                />
+              </div>
+            </div>
           {:else if value.customType === "filters"}
             <ActionButton on:click={drawer.show}>Define filters</ActionButton>
             <Drawer bind:this={drawer} title="Filtering">
@@ -651,14 +689,22 @@
   }
 
   .block-field {
-    display: flex; /* Use Flexbox */
+    display: flex;
     justify-content: space-between;
-    flex-direction: row; /* Arrange label and field side by side */
+    flex-direction: row;
-    align-items: center; /* Align vertically in the center */
+    align-items: center;
-    gap: 10px; /* Add some space between label and field */
+    gap: 10px;
     flex: 1;
   }
 
+  .attachment-field-width {
+    margin-top: var(--spacing-xs);
+  }
+
+  .label-wrapper {
+    margin-top: var(--spacing-s);
+  }
+
   .test :global(.drawer) {
     width: 10000px !important;
   }

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -35,6 +35,8 @@
   export let bindingDrawerLeft
   export let allowHelpers = true
   export let customButtonText = null
+  export let keyBindings = false
+  export let allowJS = false
   export let compare = (option, value) => option === value
 
   let fields = Object.entries(object || {}).map(([name, value]) => ({
@@ -116,12 +118,23 @@
     class:readOnly-menu={readOnly && showMenu}
   >
     {#each fields as field, idx}
-      <Input
+      {#if keyBindings}
+        <DrawerBindableInput
+          {bindings}
           placeholder={keyPlaceholder}
-        readonly={readOnly}
+          on:blur={e => {
-        bind:value={field.name}
+            field.name = e.detail
-        on:blur={changed}
+            changed()
+          }}
+          disabled={readOnly}
+          value={field.name}
+          {allowJS}
+          {allowHelpers}
+          drawerLeft={bindingDrawerLeft}
         />
+      {:else}
+        <Input readonly={readOnly} bind:value={field.name} on:blur={changed} />
+      {/if}
       {#if isJsonArray(field.value)}
         <Select readonly={true} value="Array" options={["Array"]} />
       {:else if options}
@@ -134,14 +147,14 @@
       {:else if bindings && bindings.length}
         <DrawerBindableInput
           {bindings}
-          placeholder="Value"
+          placeholder={valuePlaceholder}
           on:blur={e => {
             field.value = e.detail
             changed()
           }}
           disabled={readOnly}
           value={field.value}
-          allowJS={false}
+          {allowJS}
           {allowHelpers}
           drawerLeft={bindingDrawerLeft}
         />

packages/cli/src/backups/objectStore.ts

@@ -9,7 +9,7 @@ const {
   ObjectStore,
   retrieve,
   uploadDirectory,
-  makeSureBucketExists,
+  createBucketIfNotExists,
 } = objectStore
 
 const bucketList = Object.values(ObjectStoreBuckets)
@@ -61,7 +61,7 @@ export async function importObjects() {
   let count = 0
   for (let bucket of buckets) {
     const client = ObjectStore(bucket)
-    await makeSureBucketExists(client, bucket)
+    await createBucketIfNotExists(client, bucket)
     const files = await uploadDirectory(bucket, join(path, bucket), "/")
     count += files.length
     bar.update(count)

packages/pro

@@ -1 +1 @@
-Subproject commit b55d5b32003e3e999a1cbf2e5f3e6ce8d71eace7
+Subproject commit dff7b5a9dd1fd770f8a48fb8e6df1740be605f18

packages/server/package.json

@@ -61,14 +61,17 @@
     "@google-cloud/firestore": "6.8.0",
     "@koa/router": "8.0.8",
     "@socket.io/redis-adapter": "^8.2.1",
+    "@types/xml2js": "^0.4.14",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "archiver": "7.0.1",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
+    "bl": "^6.0.12",
     "bull": "4.10.1",
     "chokidar": "3.5.3",
+    "content-disposition": "^0.5.4",
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",

packages/server/src/automations/steps/sendSmtpEmail.ts

@@ -7,6 +7,7 @@ import {
   AutomationStepType,
   AutomationIOType,
   AutomationFeature,
+  AutomationCustomIOType,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -72,10 +73,10 @@ export const definition: AutomationStepSchema = {
           title: "Location",
           dependsOn: "addInvite",
         },
-        url: {
+        attachments: {
-          type: AutomationIOType.STRING,
+          type: AutomationIOType.ATTACHMENT,
-          title: "URL",
+          customType: AutomationCustomIOType.MULTI_ATTACHMENTS,
-          dependsOn: "addInvite",
+          title: "Attachments",
         },
       },
       required: ["to", "from", "subject", "contents"],
@@ -110,11 +111,13 @@ export async function run({ inputs }: AutomationStepInput) {
     summary,
     location,
     url,
+    attachments,
   } = inputs
   if (!contents) {
     contents = "<h1>No content</h1>"
   }
   to = to || undefined
+
   try {
     let response = await sendSmtpEmail({
       to,
@@ -124,6 +127,7 @@ export async function run({ inputs }: AutomationStepInput) {
       cc,
       bcc,
       automation: true,
+      attachments,
       invite: addInvite
         ? {
             startTime,

packages/server/src/automations/tests/sendSmtpEmail.spec.ts

@@ -50,6 +50,10 @@ describe("test the outgoing webhook action", () => {
       cc: "cc",
       bcc: "bcc",
       addInvite: true,
+      attachments: [
+        { url: "attachment1", filename: "attachment1.txt" },
+        { url: "attachment2", filename: "attachment2.txt" },
+      ],
       ...invite,
     }
     let resp = generateResponse(inputs.to, inputs.from)
@@ -69,6 +73,10 @@ describe("test the outgoing webhook action", () => {
       bcc: "bcc",
       invite,
       automation: true,
+      attachments: [
+        { url: "attachment1", filename: "attachment1.txt" },
+        { url: "attachment2", filename: "attachment2.txt" },
+      ],
     })
   })
 })

packages/server/src/integrations/rest.ts

@@ -21,6 +21,10 @@ import { performance } from "perf_hooks"
 import FormData from "form-data"
 import { URLSearchParams } from "url"
 import { blacklist } from "@budibase/backend-core"
+import { handleFileResponse, handleXml } from "./utils"
+import { parse } from "content-disposition"
+import path from "path"
+import { Builder as XmlBuilder } from "xml2js"
 
 const BodyTypes = {
   NONE: "none",
@@ -57,8 +61,6 @@ const coreFields = {
   },
 }
 
-const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
-
 const SCHEMA: Integration = {
   docs: "https://github.com/node-fetch/node-fetch",
   description:
@@ -129,9 +131,22 @@ class RestIntegration implements IntegrationBase {
   }
 
   async parseResponse(response: any, pagination: PaginationConfig | null) {
-    let data, raw, headers
+    let data, raw, headers, filename
+
     const contentType = response.headers.get("content-type") || ""
+    const contentDisposition = response.headers.get("content-disposition") || ""
+    if (
+      contentDisposition.includes("attachment") ||
+      contentDisposition.includes("form-data")
+    ) {
+      filename =
+        path.basename(parse(contentDisposition).parameters?.filename) || ""
+    }
+
     try {
+      if (filename) {
+        return handleFileResponse(response, filename, this.startTimeMs)
+      } else {
         if (response.status === 204) {
           data = []
           raw = []
@@ -142,29 +157,18 @@ class RestIntegration implements IntegrationBase {
           contentType.includes("text/xml") ||
           contentType.includes("application/xml")
         ) {
-        const rawXml = await response.text()
+          let xmlResponse = await handleXml(response)
-        data =
+          data = xmlResponse.data
-          (await xmlParser(rawXml, {
+          raw = xmlResponse.rawXml
-            explicitArray: false,
-            trim: true,
-            explicitRoot: false,
-          })) || {}
-        // there is only one structure, its an array, return the array so it appears as rows
-        const keys = Object.keys(data)
-        if (keys.length === 1 && Array.isArray(data[keys[0]])) {
-          data = data[keys[0]]
-        }
-        raw = rawXml
-      } else if (contentType.includes("application/pdf")) {
-        data = await response.arrayBuffer() // Save PDF as ArrayBuffer
-        raw = Buffer.from(data)
         } else {
           data = await response.text()
           raw = data
         }
-    } catch (err) {
-      throw "Failed to parse response body."
       }
+    } catch (err) {
+      throw `Failed to parse response body: ${err}`
+    }
+
     const size = formatBytes(
       response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
     )

packages/server/src/integrations/tests/rest.spec.ts

@@ -13,9 +13,23 @@ jest.mock("node-fetch", () => {
   }))
 })
 
-import fetch from "node-fetch"
+jest.mock("@budibase/backend-core", () => {
+  const core = jest.requireActual("@budibase/backend-core")
+  return {
+    ...core,
+    context: {
+      ...core.context,
+      getProdAppId: jest.fn(() => "app-id"),
+    },
+  }
+})
+jest.mock("uuid", () => ({ v4: () => "00000000-0000-0000-0000-000000000000" }))
+
 import { default as RestIntegration } from "../rest"
 import { RestAuthType } from "@budibase/types"
+import fetch from "node-fetch"
+import { objectStoreTestProviders } from "@budibase/backend-core/tests"
+import { Readable } from "stream"
 
 const FormData = require("form-data")
 const { URLSearchParams } = require("url")
@@ -611,4 +625,104 @@ describe("REST Integration", () => {
     expect(calledConfig.headers).toEqual({})
     expect(calledConfig.agent.options.rejectUnauthorized).toBe(false)
   })
+
+  describe("File Handling", () => {
+    beforeAll(async () => {
+      jest.unmock("aws-sdk")
+      await objectStoreTestProviders.minio.start()
+    })
+
+    afterAll(async () => {
+      await objectStoreTestProviders.minio.stop()
+    })
+
+    it("uploads file to object store and returns signed URL", async () => {
+      const responseData = Buffer.from("teest file contnt")
+      const filename = "test.tar.gz"
+      const contentType = "application/gzip"
+      const mockReadable = new Readable()
+      mockReadable.push(responseData)
+      mockReadable.push(null)
+      ;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
+        Promise.resolve({
+          headers: {
+            raw: () => ({
+              "content-type": [contentType],
+              "content-disposition": [`attachment; filename="${filename}"`],
+            }),
+            get: (header: any) => {
+              if (header === "content-type") return contentType
+              if (header === "content-disposition")
+                return `attachment; filename="${filename}"`
+            },
+          },
+          body: mockReadable,
+        })
+      )
+
+      const query = {
+        path: "api",
+      }
+
+      const response = await config.integration.read(query)
+
+      expect(response.data).toEqual({
+        size: responseData.byteLength,
+        name: "00000000-0000-0000-0000-000000000000.tar.gz",
+        url: expect.stringContaining(
+          "/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.tar.gz"
+        ),
+        extension: "tar.gz",
+        key: expect.stringContaining(
+          "app-id/00000000-0000-0000-0000-000000000000.tar.gz"
+        ),
+      })
+    })
+
+    it("uploads file with non ascii filename to object store and returns signed URL", async () => {
+      const responseData = Buffer.from("teest file contnt")
+      const contentType = "text/plain"
+      const mockReadable = new Readable()
+      mockReadable.push(responseData)
+      mockReadable.push(null)
+      ;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
+        Promise.resolve({
+          headers: {
+            raw: () => ({
+              "content-type": [contentType],
+              "content-disposition": [
+                // eslint-disable-next-line no-useless-escape
+                `attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`,
+              ],
+            }),
+            get: (header: any) => {
+              if (header === "content-type") return contentType
+              if (header === "content-disposition")
+                // eslint-disable-next-line no-useless-escape
+                return `attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`
+            },
+          },
+          body: mockReadable,
+        })
+      )
+
+      const query = {
+        path: "api",
+      }
+
+      const response = await config.integration.read(query)
+
+      expect(response.data).toEqual({
+        size: responseData.byteLength,
+        name: "00000000-0000-0000-0000-000000000000.pdf",
+        url: expect.stringContaining(
+          "/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.pdf"
+        ),
+        extension: "pdf",
+        key: expect.stringContaining(
+          "app-id/00000000-0000-0000-0000-000000000000.pdf"
+        ),
+      })
+    })
+  })
 })

packages/server/src/integrations/utils/utils.ts

@@ -6,10 +6,15 @@ import {
   TableSourceType,
   FieldSchema,
 } from "@budibase/types"
+import { context, objectStore } from "@budibase/backend-core"
+import { v4 } from "uuid"
+import { parseStringPromise as xmlParser } from "xml2js"
+import { formatBytes } from "../../utilities"
+import bl from "bl"
+import env from "../../environment"
 import { DocumentType, SEPARATOR } from "../../db/utils"
 import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
 import { helpers, utils } from "@budibase/shared-core"
-import env from "../../environment"
 import { Knex } from "knex"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
@@ -467,3 +472,74 @@ export function getPrimaryDisplay(testValue: unknown): string | undefined {
 export function isValidFilter(value: any) {
   return value != null && value !== ""
 }
+
+export async function handleXml(response: any) {
+  let data,
+    rawXml = await response.text()
+  data =
+    (await xmlParser(rawXml, {
+      explicitArray: false,
+      trim: true,
+      explicitRoot: false,
+    })) || {}
+  // there is only one structure, its an array, return the array so it appears as rows
+  const keys = Object.keys(data)
+  if (keys.length === 1 && Array.isArray(data[keys[0]])) {
+    data = data[keys[0]]
+  }
+  return { data, rawXml }
+}
+
+export async function handleFileResponse(
+  response: any,
+  filename: string,
+  startTime: number
+) {
+  let presignedUrl,
+    size = 0
+  const fileExtension = filename.includes(".")
+    ? filename.split(".").slice(1).join(".")
+    : ""
+
+  const processedFileName = `${v4()}.${fileExtension}`
+  const key = `${context.getProdAppId()}/${processedFileName}`
+  const bucket = objectStore.ObjectStoreBuckets.TEMP
+
+  const stream = response.body.pipe(bl((error, data) => data))
+
+  if (response.body) {
+    const contentLength = response.headers.get("content-length")
+    if (contentLength) {
+      size = parseInt(contentLength, 10)
+    } else {
+      const chunks: Buffer[] = []
+      for await (const chunk of response.body) {
+        chunks.push(chunk)
+        size += chunk.length
+      }
+    }
+
+    await objectStore.streamUpload({
+      bucket,
+      filename: key,
+      stream,
+      ttl: 1,
+      type: response.headers["content-type"],
+    })
+  }
+  presignedUrl = await objectStore.getPresignedUrl(bucket, key)
+  return {
+    data: {
+      size,
+      name: processedFileName,
+      url: presignedUrl,
+      extension: fileExtension,
+      key: key,
+    },
+    info: {
+      code: response.status,
+      size: formatBytes(size.toString()),
+      time: `${Math.round(performance.now() - startTime)}ms`,
+    },
+  }
+}

packages/server/src/utilities/fileSystem/clientLibrary.ts

@@ -106,22 +106,22 @@ export async function updateClientLibrary(appId: string) {
   }
 
   // Upload latest manifest and client library
-  const manifestUpload = objectStore.streamUpload(
+  const manifestUpload = objectStore.streamUpload({
-    ObjectStoreBuckets.APPS,
+    bucket: ObjectStoreBuckets.APPS,
-    join(appId, "manifest.json"),
+    filename: join(appId, "manifest.json"),
-    fs.createReadStream(manifest),
+    stream: fs.createReadStream(manifest),
-    {
+    extra: {
       ContentType: "application/json",
-    }
+    },
-  )
+  })
-  const clientUpload = objectStore.streamUpload(
+  const clientUpload = objectStore.streamUpload({
-    ObjectStoreBuckets.APPS,
+    bucket: ObjectStoreBuckets.APPS,
-    join(appId, "budibase-client.js"),
+    filename: join(appId, "budibase-client.js"),
-    fs.createReadStream(client),
+    stream: fs.createReadStream(client),
-    {
+    extra: {
       ContentType: "application/javascript",
-    }
+    },
-  )
+  })
 
   const manifestSrc = fs.promises.readFile(manifest, "utf8")
 

packages/server/src/utilities/workerRequests.ts

@@ -8,7 +8,7 @@ import {
   logging,
   env as coreEnv,
 } from "@budibase/backend-core"
-import { Ctx, User, EmailInvite } from "@budibase/types"
+import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types"
 
 interface Request {
   ctx?: Ctx
@@ -97,6 +97,7 @@ export async function sendSmtpEmail({
   bcc,
   automation,
   invite,
+  attachments,
 }: {
   to: string
   from: string
@@ -105,6 +106,7 @@ export async function sendSmtpEmail({
   cc: string
   bcc: string
   automation: boolean
+  attachments?: EmailAttachment[]
   invite?: EmailInvite
 }) {
   // tenant ID will be set in header
@@ -122,6 +124,7 @@ export async function sendSmtpEmail({
         purpose: "custom",
         automation,
         invite,
+        attachments,
       },
     })
   )

packages/types/src/documents/app/automation.ts

@@ -10,6 +10,7 @@ export enum AutomationIOType {
   ARRAY = "array",
   JSON = "json",
   DATE = "date",
+  ATTACHMENT = "attachment",
 }
 
 export enum AutomationCustomIOType {
@@ -30,6 +31,7 @@ export enum AutomationCustomIOType {
   WEBHOOK_URL = "webhookUrl",
   AUTOMATION = "automation",
   AUTOMATION_FIELDS = "automationFields",
+  MULTI_ATTACHMENTS = "multi_attachments",
 }
 
 export enum AutomationTriggerStepId {
@@ -80,6 +82,11 @@ export interface EmailInvite {
   url?: string
 }
 
+export interface EmailAttachment {
+  url: string
+  filename: string
+}
+
 export interface SendEmailOpts {
   // workspaceId If finer grain controls being used then this will lookup config for workspace.
   workspaceId?: string
@@ -97,6 +104,7 @@ export interface SendEmailOpts {
   bcc?: boolean
   automation?: boolean
   invite?: EmailInvite
+  attachments?: EmailAttachment[]
 }
 
 export const AutomationStepIdArray = [

packages/worker/src/api/controllers/global/email.ts

@@ -15,6 +15,7 @@ export async function sendEmail(ctx: BBContext) {
     bcc,
     automation,
     invite,
+    attachments,
   } = ctx.request.body
   let user: any
   if (userId) {
@@ -31,6 +32,7 @@ export async function sendEmail(ctx: BBContext) {
     bcc,
     automation,
     invite,
+    attachments,
   })
   ctx.body = {
     ...response,

packages/worker/src/api/routes/global/tests/realEmail.spec.ts

@@ -1,9 +1,15 @@
 jest.unmock("node-fetch")
+jest.unmock("aws-sdk")
 import { TestConfiguration } from "../../../../tests"
 import { EmailTemplatePurpose } from "../../../../constants"
+import { objectStoreTestProviders } from "@budibase/backend-core/tests"
+import { objectStore } from "@budibase/backend-core"
+import tk from "timekeeper"
+import { EmailAttachment } from "@budibase/types"
+
+const fetch = require("node-fetch")
 
 const nodemailer = require("nodemailer")
-const fetch = require("node-fetch")
 
 // for the real email tests give them a long time to try complete/fail
 jest.setTimeout(30000)
@@ -12,14 +18,20 @@ describe("/api/global/email", () => {
   const config = new TestConfiguration()
 
   beforeAll(async () => {
+    tk.reset()
+    await objectStoreTestProviders.minio.start()
     await config.beforeAll()
   })
 
   afterAll(async () => {
+    await objectStoreTestProviders.minio.stop()
     await config.afterAll()
   })
 
-  async function sendRealEmail(purpose: string) {
+  async function sendRealEmail(
+    purpose: string,
+    attachments?: EmailAttachment[]
+  ) {
     let response, text
     try {
       const timeout = () =>
@@ -35,8 +47,14 @@ describe("/api/global/email", () => {
         )
       await Promise.race([config.saveEtherealSmtpConfig(), timeout()])
       await Promise.race([config.saveSettingsConfig(), timeout()])
-
+      let res
-      const res = await config.api.emails.sendEmail(purpose).timeout(20000)
+      if (attachments) {
+        res = await config.api.emails
+          .sendEmail(purpose, attachments)
+          .timeout(20000)
+      } else {
+        res = await config.api.emails.sendEmail(purpose).timeout(20000)
+      }
       // ethereal hiccup, can't test right now
       if (res.status >= 300) {
         return
@@ -80,4 +98,25 @@ describe("/api/global/email", () => {
   it("should be able to send a password recovery email", async () => {
     await sendRealEmail(EmailTemplatePurpose.PASSWORD_RECOVERY)
   })
+
+  it("should be able to send an email with attachments", async () => {
+    let bucket = "testbucket"
+    let filename = "test.txt"
+    await objectStore.upload({
+      bucket,
+      filename,
+      body: Buffer.from("test data"),
+    })
+    let presignedUrl = await objectStore.getPresignedUrl(
+      bucket,
+      filename,
+      60000
+    )
+
+    let attachmentObject = {
+      url: presignedUrl,
+      filename,
+    }
+    await sendRealEmail(EmailTemplatePurpose.WELCOME, [attachmentObject])
+  })
 })

packages/worker/src/tests/api/email.ts

@@ -1,3 +1,4 @@
+import { EmailAttachment } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -6,11 +7,12 @@ export class EmailAPI extends TestAPI {
     super(config)
   }
 
-  sendEmail = (purpose: string) => {
+  sendEmail = (purpose: string, attachments?: EmailAttachment[]) => {
     return this.request
       .post(`/api/global/email/send`)
       .send({
         email: "test@example.com",
+        attachments,
         purpose,
         tenantId: this.config.getTenantId(),
         userId: this.config.user?._id!,

packages/worker/src/tests/jestEnv.ts

@@ -4,8 +4,8 @@ process.env.JWT_SECRET = "test-jwtsecret"
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.MULTI_TENANCY = "1"
 process.env.MINIO_URL = "http://localhost"
-process.env.MINIO_ACCESS_KEY = "test"
+process.env.MINIO_ACCESS_KEY = "budibase"
-process.env.MINIO_SECRET_KEY = "test"
+process.env.MINIO_SECRET_KEY = "budibase"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.INTERNAL_API_KEY = "tet"
 process.env.DISABLE_ACCOUNT_PORTAL = "0"

packages/worker/src/tests/structures/configs.ts

@@ -62,8 +62,8 @@ export function smtpEthereal(): SMTPConfig {
       from: "testfrom@example.com",
       secure: false,
       auth: {
-        user: "wyatt.zulauf29@ethereal.email",
+        user: "mortimer.leuschke@ethereal.email",
-        pass: "tEwDtHBWWxusVWAPfa",
+        pass: "5hSjsPbzRv7gEUsfzx",
       },
       connectionTimeout: 1000, // must be less than the jest default of 5000
     },

packages/worker/src/utilities/email.ts

@@ -4,8 +4,10 @@ import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
 import { getSettingsTemplateContext } from "./templates"
 import { processString } from "@budibase/string-templates"
 import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
-import { configs, cache } from "@budibase/backend-core"
+import { configs, cache, objectStore } from "@budibase/backend-core"
 import ical from "ical-generator"
+import fetch from "node-fetch"
+import path from "path"
 
 const nodemailer = require("nodemailer")
 
@@ -162,6 +164,42 @@ export async function sendEmail(
       contents: opts?.contents,
     }),
   }
+  if (opts?.attachments) {
+    const attachments = await Promise.all(
+      opts.attachments?.map(async attachment => {
+        const isFullyFormedUrl =
+          attachment.url.startsWith("http://") ||
+          attachment.url.startsWith("https://")
+        if (isFullyFormedUrl) {
+          const response = await fetch(attachment.url)
+          if (!response.ok) {
+            throw new Error(`unexpected response ${response.statusText}`)
+          }
+          const fallbackFilename = path.basename(
+            new URL(attachment.url).pathname
+          )
+          return {
+            filename: attachment.filename || fallbackFilename,
+            content: response?.body,
+          }
+        } else {
+          const url = attachment.url
+          const result = objectStore.extractBucketAndPath(url)
+          if (result === null) {
+            throw new Error("Invalid signed URL")
+          }
+          const { bucket, path } = result
+          const readStream = await objectStore.getReadStream(bucket, path)
+          const fallbackFilename = path.split("/").pop() || ""
+          return {
+            filename: attachment.filename || fallbackFilename,
+            content: readStream,
+          }
+        }
+      })
+    )
+    message = { ...message, attachments }
+  }
 
   message = {
     ...message,

yarn.lock

@@ -6147,6 +6147,13 @@
   dependencies:
     "@types/webidl-conversions" "*"
 
+"@types/xml2js@^0.4.14":
+  version "0.4.14"
+  resolved "https://registry.yarnpkg.com/@types/xml2js/-/xml2js-0.4.14.tgz#5d462a2a7330345e2309c6b549a183a376de8f9a"
+  integrity sha512-4YnrRemBShWRO2QjvUin8ESA41rH+9nQGLUGZV/1IDhi3SL9OhdpNC/MrulTWuptXKwhx/aDxE7toV0f/ypIXQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/yargs-parser@*":
   version "21.0.0"
   resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b"
@@ -7607,6 +7614,16 @@ bl@^4.0.3, bl@^4.1.0:
     inherits "^2.0.4"
     readable-stream "^3.4.0"
 
+bl@^6.0.12:
+  version "6.0.12"
+  resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8"
+  integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w==
+  dependencies:
+    "@types/readable-stream" "^4.0.0"
+    buffer "^6.0.3"
+    inherits "^2.0.4"
+    readable-stream "^4.2.0"
+
 bl@^6.0.3:
   version "6.0.9"
   resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.9.tgz#df8fcb2ef7be2e5ee8f65afa493502914e0d816f"
@@ -8781,7 +8798,7 @@ consolidate@^0.16.0:
   dependencies:
     bluebird "^3.7.2"
 
-content-disposition@^0.5.2, content-disposition@^0.5.3, content-disposition@~0.5.2:
+content-disposition@^0.5.2, content-disposition@^0.5.3, content-disposition@^0.5.4, content-disposition@~0.5.2:
   version "0.5.4"
   resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe"
   integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==