Merge pull request #4030 from Budibase/s3-upload

S3 file upload component

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -147,7 +147,9 @@
           <img alt="preview" src={selectedUrl} />
         {:else}
           <div class="placeholder">
-            <div class="extension">{selectedImage.extension}</div>
+            <div class="extension">
+              {selectedImage.name || "Unknown file"}
+            </div>
             <div>Preview not supported</div>
           </div>
         {/if}
@@ -359,18 +361,21 @@
     white-space: nowrap;
     width: 0;
     margin-right: 10px;
+    user-select: all;
   }
   .placeholder {
     display: flex;
     flex-direction: column;
     justify-content: center;
     align-items: center;
+    text-align: center;
   }
   .extension {
     color: var(--spectrum-global-color-gray-600);
     text-transform: uppercase;
     font-weight: 600;
     margin-bottom: 5px;
+    user-select: all;
   }
 
   .nav {

packages/builder/src/components/design/AppPreview/AppThemeSelect.svelte

@@ -1,6 +1,7 @@
 <script>
   import { Select } from "@budibase/bbui"
   import { store } from "builderStore"
+  import { get } from "svelte/store"
 
   const themeOptions = [
     {
@@ -20,6 +21,17 @@
       value: "spectrum--darkest",
     },
   ]
+
+  const onChangeTheme = async theme => {
+    await store.actions.theme.save(theme)
+    await store.actions.customTheme.save({
+      ...get(store).customTheme,
+      navBackground:
+        theme === "spectrum--light"
+          ? "var(--spectrum-global-color-gray-50)"
+          : "var(--spectrum-global-color-gray-100)",
+    })
+  }
 </script>
 
 <div>
@@ -27,7 +39,7 @@
     value={$store.theme}
     options={themeOptions}
     placeholder={null}
-    on:change={e => store.actions.theme.save(e.detail)}
+    on:change={e => onChangeTheme(e.detail)}
   />
 </div>
 

packages/builder/src/components/design/AppPreview/ThemeEditor.svelte

@@ -19,7 +19,7 @@
     primaryColor: "var(--spectrum-global-color-blue-600)",
     primaryColorHover: "var(--spectrum-global-color-blue-500)",
     buttonBorderRadius: "16px",
-    navBackground: "var(--spectrum-global-color-gray-100)",
+    navBackground: "var(--spectrum-global-color-gray-50)",
     navTextColor: "var(--spectrum-global-color-gray-800)",
   }
 
@@ -52,7 +52,14 @@
   }
 
   const resetTheme = () => {
-    store.actions.customTheme.save(null)
+    const theme = get(store).theme
+    store.actions.customTheme.save({
+      ...defaultTheme,
+      navBackground:
+        theme === "spectrum--light"
+          ? "var(--spectrum-global-color-gray-50)"
+          : "var(--spectrum-global-color-gray-100)",
+    })
   }
 </script>
 

packages/builder/src/components/design/AppPreview/componentStructure.json

@@ -44,7 +44,8 @@
       "relationshipfield",
       "daterangepicker",
       "multifieldselect",
-      "jsonfield"
+      "jsonfield",
+      "s3upload"
     ]
   },
   {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/S3Upload.svelte

@@ -0,0 +1,33 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { currentAsset } from "builderStore"
+  import { findAllMatchingComponents } from "builderStore/componentUtils"
+
+  export let parameters
+
+  $: components = findAllMatchingComponents($currentAsset.props, component =>
+    component._component.endsWith("s3upload")
+  )
+</script>
+
+<div class="root">
+  <Label small>S3 Upload Component</Label>
+  <Select
+    bind:value={parameters.componentId}
+    options={components}
+    getOptionLabel={x => x._instanceName}
+    getOptionValue={x => x._id}
+  />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 120px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/index.js

@@ -11,3 +11,4 @@ export { default as ChangeFormStep } from "./ChangeFormStep.svelte"
 export { default as UpdateState } from "./UpdateState.svelte"
 export { default as RefreshDataProvider } from "./RefreshDataProvider.svelte"
 export { default as DuplicateRow } from "./DuplicateRow.svelte"
+export { default as S3Upload } from "./S3Upload.svelte"

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/manifest.json

@@ -70,6 +70,16 @@
       "name": "Update State",
       "component": "UpdateState",
       "dependsOnFeature": "state"
+    },
+    {
+      "name": "Upload File to S3",
+      "component": "S3Upload",
+      "context": [
+        {
+          "label": "File URL",
+          "value": "publicUrl"
+        }
+      ]
     }
   ]
 }

packages/builder/src/components/design/PropertiesPanel/PropertyControls/S3DataSourceSelect.svelte

@@ -0,0 +1,15 @@
+<script>
+  import { Select } from "@budibase/bbui"
+  import { datasources } from "stores/backend"
+
+  export let value = null
+
+  $: dataSources = $datasources.list
+    .filter(ds => ds.source === "S3" && !ds.config?.endpoint)
+    .map(ds => ({
+      label: ds.name,
+      value: ds._id,
+    }))
+</script>
+
+<Select options={dataSources} {value} on:change />

packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js

@@ -1,5 +1,6 @@
 import { Checkbox, Select, Stepper } from "@budibase/bbui"
 import DataSourceSelect from "./DataSourceSelect.svelte"
+import S3DataSourceSelect from "./S3DataSourceSelect.svelte"
 import DataProviderSelect from "./DataProviderSelect.svelte"
 import ButtonActionEditor from "./ButtonActionEditor/ButtonActionEditor.svelte"
 import TableSelect from "./TableSelect.svelte"
@@ -22,6 +23,7 @@ const componentMap = {
   text: DrawerBindableCombobox,
   select: Select,
   dataSource: DataSourceSelect,
+  "dataSource/s3": S3DataSourceSelect,
   dataProvider: DataProviderSelect,
   boolean: Checkbox,
   number: Stepper,

packages/client/manifest.json

@@ -3340,5 +3340,50 @@
         "suffix": "repeater"
       }
     ]
+  },
+  "s3upload": {
+    "name": "S3 File Upload",
+    "info": "This component can't be used with S3 datasources that use custom endpoints.",
+    "icon": "UploadToCloud",
+    "styles": ["size"],
+    "editable": true,
+    "settings": [
+      {
+        "type": "field/attachment",
+        "label": "Field",
+        "key": "field"
+      },
+      {
+        "type": "text",
+        "label": "Label",
+        "key": "label"
+      },
+      {
+        "type": "dataSource/s3",
+        "label": "S3 Datasource",
+        "key": "datasourceId"
+      },
+      {
+        "type": "text",
+        "label": "Bucket",
+        "key": "bucket"
+      },
+      {
+        "type": "text",
+        "label": "File Name",
+        "key": "key"
+      },
+      {
+        "type": "boolean",
+        "label": "Disabled",
+        "key": "disabled",
+        "defaultValue": false
+      },
+      {
+        "type": "validation/attachment",
+        "label": "Validation",
+        "key": "validation"
+      }
+    ]
   }
 }

packages/client/package.json

@@ -23,6 +23,7 @@
     "@budibase/standard-components": "^0.9.139",
     "@budibase/string-templates": "^1.0.46-alpha.3",
     "regexparam": "^1.3.0",
+    "rollup-plugin-polyfill-node": "^0.8.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"
   },
@@ -45,8 +46,6 @@
     "postcss": "^8.2.10",
     "rollup": "^2.44.0",
     "rollup-plugin-json": "^4.0.0",
-    "rollup-plugin-node-builtins": "^2.1.2",
-    "rollup-plugin-node-globals": "^1.4.0",
     "rollup-plugin-postcss": "^4.0.0",
     "rollup-plugin-svelte": "^7.1.0",
     "rollup-plugin-svg": "^2.0.0",

packages/client/rollup.config.js

@@ -6,8 +6,7 @@ import { terser } from "rollup-plugin-terser"
 import postcss from "rollup-plugin-postcss"
 import svg from "rollup-plugin-svg"
 import json from "rollup-plugin-json"
-import builtins from "rollup-plugin-node-builtins"
-import globals from "rollup-plugin-node-globals"
+import nodePolyfills from "rollup-plugin-polyfill-node"
 import path from "path"
 
 const production = !process.env.ROLLUP_WATCH
@@ -75,8 +74,7 @@ export default {
     }),
     postcss(),
     commonjs(),
-    globals(),
-    builtins(),
+    nodePolyfills(),
     resolve({
       preferBuiltins: true,
       browser: true,

packages/client/src/api/api.js

@@ -36,7 +36,11 @@ const makeApiCall = async ({ method, url, body, json = true }) => {
     })
     switch (response.status) {
       case 200:
-        return response.json()
+        try {
+          return await response.json()
+        } catch (error) {
+          return null
+        }
       case 401:
         notificationStore.actions.error("Invalid credentials")
         return handleError(`Invalid credentials`)
@@ -82,14 +86,15 @@ const makeCachedApiCall = async params => {
  * Constructs an API call function for a particular HTTP method.
  */
 const requestApiCall = method => async params => {
-  const { url, cache = false } = params
-  const fixedUrl = `/${url}`.replace("//", "/")
+  const { external = false, url, cache = false } = params
+  const fixedUrl = external ? url : `/${url}`.replace("//", "/")
   const enrichedParams = { ...params, method, url: fixedUrl }
   return await (cache ? makeCachedApiCall : makeApiCall)(enrichedParams)
 }
 
 export default {
   post: requestApiCall("POST"),
+  put: requestApiCall("PUT"),
   get: requestApiCall("GET"),
   patch: requestApiCall("PATCH"),
   del: requestApiCall("DELETE"),

packages/client/src/api/attachments.js

@@ -10,3 +10,41 @@ export const uploadAttachment = async (data, tableId = "") => {
     json: false,
   })
 }
+
+/**
+ * Generates a signed URL to upload a file to an external datasource.
+ */
+export const getSignedDatasourceURL = async (datasourceId, bucket, key) => {
+  if (!datasourceId) {
+    return null
+  }
+  const res = await API.post({
+    url: `/api/attachments/${datasourceId}/url`,
+    body: { bucket, key },
+  })
+  if (res.error) {
+    throw "Could not generate signed upload URL"
+  }
+  return res
+}
+
+/**
+ * Uploads a file to an external datasource.
+ */
+export const externalUpload = async (datasourceId, bucket, key, data) => {
+  const { signedUrl, publicUrl } = await getSignedDatasourceURL(
+    datasourceId,
+    bucket,
+    key
+  )
+  const res = await API.put({
+    url: signedUrl,
+    body: data,
+    json: false,
+    external: true,
+  })
+  if (res?.error) {
+    throw "Could not upload file to signed URL"
+  }
+  return { publicUrl }
+}

packages/client/src/components/app/forms/S3Upload.svelte

@@ -0,0 +1,143 @@
+<script>
+  import Field from "./Field.svelte"
+  import { CoreDropzone, ProgressCircle } from "@budibase/bbui"
+  import { getContext, onMount, onDestroy } from "svelte"
+
+  export let datasourceId
+  export let bucket
+  export let key
+  export let field
+  export let label
+  export let disabled = false
+  export let validation
+
+  let fieldState
+  let fieldApi
+
+  const { API, notificationStore, uploadStore } = getContext("sdk")
+  const component = getContext("component")
+
+  // 5GB cap per item sent via S3 REST API
+  const MaxFileSize = 1000000000 * 5
+
+  // Actual file data to upload
+  let data
+  let loading = false
+
+  const handleFileTooLarge = () => {
+    notificationStore.actions.warning(
+      "Files cannot exceed 5GB. Please try again with a smaller file."
+    )
+  }
+
+  // Process the file input and return a serializable structure expected by
+  // the dropzone component to display the file
+  const processFiles = async fileList => {
+    return await new Promise(resolve => {
+      if (!fileList?.length) {
+        return []
+      }
+
+      // Don't read in non-image files
+      data = fileList[0]
+      if (!data.type?.startsWith("image")) {
+        resolve([
+          {
+            name: data.name,
+            type: data.type,
+          },
+        ])
+      }
+
+      // Read image files and display as preview
+      const reader = new FileReader()
+      reader.addEventListener(
+        "load",
+        () => {
+          resolve([
+            {
+              url: reader.result,
+              name: data.name,
+              type: data.type,
+            },
+          ])
+        },
+        false
+      )
+      reader.readAsDataURL(fileList[0])
+    })
+  }
+
+  const upload = async () => {
+    loading = true
+    try {
+      const res = await API.externalUpload(datasourceId, bucket, key, data)
+      notificationStore.actions.success("File uploaded successfully")
+      loading = false
+      return res
+    } catch (error) {
+      notificationStore.actions.error(`Error uploading file: ${error}`)
+    }
+  }
+
+  onMount(() => {
+    uploadStore.actions.registerFileUpload($component.id, upload)
+  })
+
+  onDestroy(() => {
+    uploadStore.actions.unregisterFileUpload($component.id)
+  })
+</script>
+
+<Field
+  {label}
+  {field}
+  {disabled}
+  {validation}
+  type="s3upload"
+  bind:fieldState
+  bind:fieldApi
+  defaultValue={[]}
+>
+  <div class="content">
+    {#if fieldState}
+      <CoreDropzone
+        value={fieldState.value}
+        disabled={loading || fieldState.disabled}
+        error={fieldState.error}
+        on:change={e => {
+          fieldApi.setValue(e.detail)
+        }}
+        {processFiles}
+        {handleFileTooLarge}
+        maximum={1}
+        fileSizeLimit={MaxFileSize}
+      />
+    {/if}
+    {#if loading}
+      <div class="overlay" />
+      <div class="loading">
+        <ProgressCircle />
+      </div>
+    {/if}
+  </div>
+</Field>
+
+<style>
+  .content {
+    position: relative;
+  }
+  .overlay,
+  .loading {
+    position: absolute;
+    top: 0;
+    height: 100%;
+    width: 100%;
+    display: grid;
+    place-items: center;
+  }
+  .overlay {
+    background-color: var(--spectrum-global-color-gray-50);
+    opacity: 0.5;
+  }
+</style>

packages/client/src/components/app/forms/index.js

@@ -12,3 +12,4 @@ export { default as relationshipfield } from "./RelationshipField.svelte"
 export { default as passwordfield } from "./PasswordField.svelte"
 export { default as formstep } from "./FormStep.svelte"
 export { default as jsonfield } from "./JSONField.svelte"
+export { default as s3upload } from "./S3Upload.svelte"

packages/client/src/sdk.js

@@ -5,6 +5,7 @@ import {
   routeStore,
   screenStore,
   builderStore,
+  uploadStore,
 } from "stores"
 import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
@@ -20,6 +21,7 @@ export default {
   routeStore,
   screenStore,
   builderStore,
+  uploadStore,
   styleable,
   linkable,
   getAction,

packages/client/src/stores/index.js

@@ -9,6 +9,7 @@ export { confirmationStore } from "./confirmation"
 export { peekStore } from "./peek"
 export { stateStore } from "./state"
 export { themeStore } from "./theme"
+export { uploadStore } from "./uploads.js"
 
 // Context stores are layered and duplicated, so it is not a singleton
 export { createContextStore } from "./context"

packages/client/src/stores/uploads.js

@@ -0,0 +1,42 @@
+import { writable, get } from "svelte/store"
+
+export const createUploadStore = () => {
+  const store = writable([])
+
+  // Registers a new file upload component
+  const registerFileUpload = (componentId, callback) => {
+    if (!componentId || !callback) {
+      return
+    }
+
+    store.update(state => {
+      state.push({
+        componentId,
+        callback,
+      })
+      return state
+    })
+  }
+
+  // Unregisters a file upload component
+  const unregisterFileUpload = componentId => {
+    store.update(state => state.filter(c => c.componentId !== componentId))
+  }
+
+  // Processes a file upload for a given component ID
+  const processFileUpload = async componentId => {
+    if (!componentId) {
+      return
+    }
+
+    const component = get(store).find(c => c.componentId === componentId)
+    return await component?.callback()
+  }
+
+  return {
+    subscribe: store.subscribe,
+    actions: { registerFileUpload, unregisterFileUpload, processFileUpload },
+  }
+}
+
+export const uploadStore = createUploadStore()

packages/client/src/utils/buttonActions.js

@@ -5,6 +5,7 @@ import {
   confirmationStore,
   authStore,
   stateStore,
+  uploadStore,
 } from "stores"
 import { saveRow, deleteRow, executeQuery, triggerAutomation } from "api"
 import { ActionTypes } from "constants"
@@ -169,6 +170,17 @@ const updateStateHandler = action => {
   }
 }
 
+const s3UploadHandler = async action => {
+  const { componentId } = action.parameters
+  if (!componentId) {
+    return
+  }
+  const res = await uploadStore.actions.processFileUpload(componentId)
+  return {
+    publicUrl: res?.publicUrl,
+  }
+}
+
 const handlerMap = {
   ["Save Row"]: saveRowHandler,
   ["Duplicate Row"]: duplicateRowHandler,
@@ -183,6 +195,7 @@ const handlerMap = {
   ["Close Screen Modal"]: closeScreenModalHandler,
   ["Change Form Step"]: changeFormStepHandler,
   ["Update State"]: updateStateHandler,
+  ["Upload File to S3"]: s3UploadHandler,
 }
 
 const confirmTextMap = {

packages/server/src/api/controllers/static/index.js

@@ -17,6 +17,8 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
+const AWS = require("aws-sdk")
+const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -104,3 +106,51 @@ exports.serveClientLibrary = async function (ctx) {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
+
+exports.getSignedUploadURL = async function (ctx) {
+  const database = new CouchDB(ctx.appId)
+
+  // Ensure datasource is valid
+  let datasource
+  try {
+    const { datasourceId } = ctx.params
+    datasource = await database.get(datasourceId)
+    if (!datasource) {
+      ctx.throw(400, "The specified datasource could not be found")
+    }
+  } catch (error) {
+    ctx.throw(400, "The specified datasource could not be found")
+  }
+
+  // Ensure we aren't using a custom endpoint
+  if (datasource?.config?.endpoint) {
+    ctx.throw(400, "S3 datasources with custom endpoints are not supported")
+  }
+
+  // Determine type of datasource and generate signed URL
+  let signedUrl
+  let publicUrl
+  if (datasource.source === "S3") {
+    const { bucket, key } = ctx.request.body || {}
+    if (!bucket || !key) {
+      ctx.throw(400, "bucket and key values are required")
+      return
+    }
+    try {
+      const s3 = new AWS.S3({
+        region: AWS_REGION,
+        accessKeyId: datasource?.config?.accessKeyId,
+        secretAccessKey: datasource?.config?.secretAccessKey,
+        apiVersion: "2006-03-01",
+        signatureVersion: "v4",
+      })
+      const params = { Bucket: bucket, Key: key }
+      signedUrl = s3.getSignedUrl("putObject", params)
+      publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
+    } catch (error) {
+      ctx.throw(400, error)
+    }
+  }
+
+  ctx.body = { signedUrl, publicUrl }
+}

packages/server/src/api/routes/static.js

@@ -46,5 +46,10 @@ router
   )
   // TODO: this likely needs to be secured in some way
   .get("/:appId/:path*", controller.serveApp)
+  .post(
+    "/api/attachments/:datasourceId/url",
+    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
+    controller.getSignedUploadURL
+  )
 
 module.exports = router

packages/server/src/api/routes/tests/static.spec.js

@@ -0,0 +1,98 @@
+jest.mock("node-fetch")
+jest.mock("aws-sdk", () => ({
+  config: {
+    update: jest.fn(),
+  },
+  DynamoDB: {
+    DocumentClient: jest.fn(),
+  },
+  S3: jest.fn(() => ({
+    getSignedUrl: jest.fn(() => {
+      return "my-url"
+    }),
+  })),
+}))
+
+const setup = require("./utilities")
+
+describe("/attachments", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let app
+
+  afterAll(setup.afterAll)
+
+  beforeEach(async () => {
+    app = await config.init()
+  })
+
+  describe("generateSignedUrls", () => {
+    let datasource
+
+    beforeEach(async () => {
+      datasource = await config.createDatasource({
+        datasource: {
+          type: "datasource",
+          name: "Test",
+          source: "S3",
+          config: {},
+        },
+      })
+    })
+
+    it("should be able to generate a signed upload URL", async () => {
+      const bucket = "foo"
+      const key = "bar"
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({ bucket, key })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body.signedUrl).toEqual("my-url")
+      expect(res.body.publicUrl).toEqual(
+        `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
+      )
+    })
+
+    it("should handle an invalid datasource ID", async () => {
+      const res = await request
+        .post(`/api/attachments/foo/url`)
+        .send({
+          bucket: "foo",
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual(
+        "The specified datasource could not be found"
+      )
+    })
+
+    it("should require a bucket parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: undefined,
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+
+    it("should require a key parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: "foo",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+  })
+})