Merge pull request #12202 from Budibase/bug/budi-7689-cdn-issues-attachment-filtering

Limit cloud uploads to a specific set of filetypes.

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -159,8 +159,10 @@
           {#if selectedImage.size}
             <div class="filesize">
               {#if selectedImage.size <= BYTES_IN_MB}
-                {`${selectedImage.size / BYTES_IN_KB} KB`}
-              {:else}{`${selectedImage.size / BYTES_IN_MB} MB`}{/if}
+                {`${(selectedImage.size / BYTES_IN_KB).toFixed(1)} KB`}
+              {:else}{`${(selectedImage.size / BYTES_IN_MB).toFixed(
+                  1
+                )} MB`}{/if}
             </div>
           {/if}
           {#if !disabled}
@@ -203,8 +205,8 @@
             {#if file.size}
               <div class="filesize">
                 {#if file.size <= BYTES_IN_MB}
-                  {`${file.size / BYTES_IN_KB} KB`}
-                {:else}{`${file.size / BYTES_IN_MB} MB`}{/if}
+                  {`${(file.size / BYTES_IN_KB).toFixed(1)} KB`}
+                {:else}{`${(file.size / BYTES_IN_MB).toFixed(1)} MB`}{/if}
               </div>
             {/if}
             {#if !disabled}

packages/builder/src/components/common/Dropzone.svelte

@@ -23,7 +23,7 @@
     try {
       return await API.uploadBuilderAttachment(data)
     } catch (error) {
-      notifications.error("Failed to upload attachment")
+      notifications.error(error.message || "Failed to upload attachment")
       return []
     }
   }

packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte

@@ -55,7 +55,7 @@
     try {
       return await API.uploadBuilderAttachment(data)
     } catch (error) {
-      $notifications.error("Failed to upload attachment")
+      $notifications.error(error.message || "Failed to upload attachment")
       return []
     }
   }

packages/server/src/api/controllers/static/index.ts

@@ -1,3 +1,5 @@
+import { ValidFileExtensions } from "@budibase/shared-core"
+
 require("svelte/register")
 
 import { join } from "../../../utilities/centralPath"
@@ -11,34 +13,21 @@ import {
 } from "../../../utilities/fileSystem"
 import env from "../../../environment"
 import { DocumentType } from "../../../db/utils"
-import { context, objectStore, utils, configs } from "@budibase/backend-core"
+import {
+  context,
+  objectStore,
+  utils,
+  configs,
+  BadRequestError,
+} from "@budibase/backend-core"
 import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App, Ctx } from "@budibase/types"
+import { App, Ctx, ProcessAttachmentResponse, Upload } from "@budibase/types"
 
 const send = require("koa-send")
 
-async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
-  const response = await objectStore.upload({
-    bucket,
-    metadata,
-    filename: s3Key,
-    path: file.path,
-    type: file.type,
-  })
-
-  // don't store a URL, work this out on the way out as the URL could change
-  return {
-    size: file.size,
-    name: file.name,
-    url: objectStore.getAppFileUrl(s3Key),
-    extension: [...file.name.split(".")].pop(),
-    key: response.Key,
-  }
-}
-
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
 
@@ -72,23 +61,58 @@ export const serveBuilder = async function (ctx: Ctx) {
   await send(ctx, ctx.file, { root: builderPath })
 }
 
-export const uploadFile = async function (ctx: Ctx) {
+export const uploadFile = async function (
+  ctx: Ctx<{}, ProcessAttachmentResponse>
+) {
   const file = ctx.request?.files?.file
+  if (!file) {
+    throw new BadRequestError("No file provided")
+  }
+
   let files = file && Array.isArray(file) ? Array.from(file) : [file]
 
-  const uploads = files.map(async (file: any) => {
-    const fileExtension = [...file.name.split(".")].pop()
-    // filenames converted to UUIDs so they are unique
-    const processedFileName = `${uuid.v4()}.${fileExtension}`
+  ctx.body = await Promise.all(
+    files.map(async file => {
+      if (!file.name) {
+        throw new BadRequestError(
+          "Attempted to upload a file without a filename"
+        )
+      }
 
-    return prepareUpload({
-      file,
-      s3Key: `${context.getProdAppId()}/attachments/${processedFileName}`,
-      bucket: ObjectStoreBuckets.APPS,
+      const extension = [...file.name.split(".")].pop()
+      if (!extension) {
+        throw new BadRequestError(
+          `File "${file.name}" has no extension, an extension is required to upload a file`
+        )
+      }
+
+      if (!env.SELF_HOSTED && !ValidFileExtensions.includes(extension)) {
+        throw new BadRequestError(
+          `File "${file.name}" has an invalid extension: "${extension}"`
+        )
+      }
+
+      // filenames converted to UUIDs so they are unique
+      const processedFileName = `${uuid.v4()}.${extension}`
+
+      const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+      const response = await objectStore.upload({
+        bucket: ObjectStoreBuckets.APPS,
+        filename: s3Key,
+        path: file.path,
+        type: file.type,
+      })
+
+      return {
+        size: file.size,
+        name: file.name,
+        url: objectStore.getAppFileUrl(s3Key),
+        extension,
+        key: response.Key,
+      }
     })
-  })
-
-  ctx.body = await Promise.all(uploads)
+  )
 }
 
 export const deleteObjects = async function (ctx: Ctx) {

packages/server/src/api/routes/tests/attachment.spec.ts

@@ -0,0 +1,49 @@
+import * as setup from "./utilities"
+import { APIError } from "@budibase/types"
+
+describe("/api/applications/:appId/sync", () => {
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  describe("/api/attachments/process", () => {
+    it("should accept an image file upload", async () => {
+      let resp = await config.api.attachment.process(
+        "1px.jpg",
+        Buffer.from([0])
+      )
+      expect(resp.length).toBe(1)
+
+      let upload = resp[0]
+      expect(upload.url.endsWith(".jpg")).toBe(true)
+      expect(upload.extension).toBe("jpg")
+      expect(upload.size).toBe(1)
+      expect(upload.name).toBe("1px.jpg")
+    })
+
+    it("should reject an upload with a malicious file extension", async () => {
+      await config.withEnv({ SELF_HOSTED: undefined }, async () => {
+        let resp = (await config.api.attachment.process(
+          "ohno.exe",
+          Buffer.from([0]),
+          { expectStatus: 400 }
+        )) as unknown as APIError
+        expect(resp.message).toContain("invalid extension")
+      })
+    })
+
+    it("should reject an upload with no file", async () => {
+      let resp = (await config.api.attachment.process(
+        undefined as any,
+        undefined as any,
+        {
+          expectStatus: 400,
+        }
+      )) as unknown as APIError
+      expect(resp.message).toContain("No file provided")
+    })
+  })
+})

packages/server/src/api/routes/tests/static.spec.js

@@ -5,11 +5,15 @@ describe("/static", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let app
+  let cleanupEnv
 
-  afterAll(setup.afterAll)
+  afterAll(() => {
+    setup.afterAll()
+    cleanupEnv()
+  })
 
   beforeAll(async () => {
-    config.modeSelf()
+    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
     app = await config.init()
   })
 

packages/server/src/api/routes/tests/webhook.spec.ts

@@ -8,11 +8,15 @@ describe("/webhooks", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let webhook: Webhook
+  let cleanupEnv: () => void
 
-  afterAll(setup.afterAll)
+  afterAll(() => {
+    setup.afterAll()
+    cleanupEnv()
+  })
 
   const setupTest = async () => {
-    config.modeSelf()
+    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
     await config.init()
     const autoConfig = basicAutomation()
     autoConfig.definition.trigger.schema = {

packages/server/src/integrations/tests/googlesheets.spec.ts

@@ -35,13 +35,18 @@ import { FieldType, Table, TableSchema } from "@budibase/types"
 describe("Google Sheets Integration", () => {
   let integration: any,
     config = new TestConfiguration()
+  let cleanupEnv: () => void
 
   beforeAll(() => {
-    config.setGoogleAuth("test")
+    cleanupEnv = config.setEnv({
+      GOOGLE_CLIENT_ID: "test",
+      GOOGLE_CLIENT_SECRET: "test",
+    })
   })
 
   afterAll(async () => {
-    await config.end()
+    cleanupEnv()
+    config.end()
   })
 
   beforeEach(async () => {

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -58,6 +58,7 @@ import {
 } from "@budibase/types"
 
 import API from "./api"
+import { cloneDeep } from "lodash"
 
 type DefaultUserValues = {
   globalUserId: string
@@ -188,30 +189,38 @@ class TestConfiguration {
     }
   }
 
-  // MODES
-  setMultiTenancy = (value: boolean) => {
-    env._set("MULTI_TENANCY", value)
-    coreEnv._set("MULTI_TENANCY", value)
+  async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
+    let cleanup = this.setEnv(newEnvVars)
+    try {
+      await f()
+    } finally {
+      cleanup()
+    }
   }
 
-  setSelfHosted = (value: boolean) => {
-    env._set("SELF_HOSTED", value)
-    coreEnv._set("SELF_HOSTED", value)
-  }
+  /*
+   * Sets the environment variables to the given values and returns a function
+   * that can be called to reset the environment variables to their original values.
+   */
+  setEnv(newEnvVars: Partial<typeof env>): () => void {
+    const oldEnv = cloneDeep(env)
+    const oldCoreEnv = cloneDeep(coreEnv)
 
-  setGoogleAuth = (value: string) => {
-    env._set("GOOGLE_CLIENT_ID", value)
-    env._set("GOOGLE_CLIENT_SECRET", value)
-    coreEnv._set("GOOGLE_CLIENT_ID", value)
-    coreEnv._set("GOOGLE_CLIENT_SECRET", value)
-  }
+    let key: keyof typeof newEnvVars
+    for (key in newEnvVars) {
+      env._set(key, newEnvVars[key])
+      coreEnv._set(key, newEnvVars[key])
+    }
 
-  modeCloud = () => {
-    this.setSelfHosted(false)
-  }
+    return () => {
+      for (const [key, value] of Object.entries(oldEnv)) {
+        env._set(key, value)
+      }
 
-  modeSelf = () => {
-    this.setSelfHosted(true)
+      for (const [key, value] of Object.entries(oldCoreEnv)) {
+        coreEnv._set(key, value)
+      }
+    }
   }
 
   // UTILS

packages/server/src/tests/utilities/api/attachment.ts

@@ -0,0 +1,35 @@
+import {
+  APIError,
+  Datasource,
+  ProcessAttachmentResponse,
+} from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+import fs from "fs"
+
+export class AttachmentAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  process = async (
+    name: string,
+    file: Buffer | fs.ReadStream | string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<ProcessAttachmentResponse> => {
+    const result = await this.request
+      .post(`/api/attachments/process`)
+      .attach("file", file, name)
+      .set(this.config.defaultHeaders())
+
+    if (result.statusCode !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          result.statusCode
+        }, body: ${JSON.stringify(result.body)}`
+      )
+    }
+
+    return result.body
+  }
+}

packages/server/src/tests/utilities/api/index.ts

@@ -7,6 +7,7 @@ import { DatasourceAPI } from "./datasource"
 import { LegacyViewAPI } from "./legacyView"
 import { ScreenAPI } from "./screen"
 import { ApplicationAPI } from "./application"
+import { AttachmentAPI } from "./attachment"
 
 export default class API {
   table: TableAPI
@@ -17,6 +18,7 @@ export default class API {
   datasource: DatasourceAPI
   screen: ScreenAPI
   application: ApplicationAPI
+  attachment: AttachmentAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -27,5 +29,6 @@ export default class API {
     this.datasource = new DatasourceAPI(config)
     this.screen = new ScreenAPI(config)
     this.application = new ApplicationAPI(config)
+    this.attachment = new AttachmentAPI(config)
   }
 }

packages/shared-core/src/constants.ts

@@ -96,3 +96,45 @@ export enum BuilderSocketEvent {
 export const SocketSessionTTL = 60
 export const ValidQueryNameRegex = /^[^()]*$/
 export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g
+export const ValidFileExtensions = [
+  "avif",
+  "css",
+  "csv",
+  "docx",
+  "drawio",
+  "editorconfig",
+  "edl",
+  "enc",
+  "export",
+  "geojson",
+  "gif",
+  "htm",
+  "html",
+  "ics",
+  "iqy",
+  "jfif",
+  "jpeg",
+  "jpg",
+  "json",
+  "log",
+  "md",
+  "mid",
+  "odt",
+  "pdf",
+  "png",
+  "ris",
+  "rtf",
+  "svg",
+  "tex",
+  "toml",
+  "twig",
+  "txt",
+  "url",
+  "wav",
+  "webp",
+  "xls",
+  "xlsx",
+  "xml",
+  "yaml",
+  "yml",
+]

packages/types/src/api/web/app/attachment.ts

@@ -0,0 +1,9 @@
+export interface Upload {
+  size: number
+  name: string
+  url: string
+  extension: string
+  key: string
+}
+
+export type ProcessAttachmentResponse = Upload[]

packages/types/src/api/web/app/index.ts

@@ -5,3 +5,4 @@ export * from "./view"
 export * from "./rows"
 export * from "./table"
 export * from "./permission"
+export * from "./attachment"