diff --git a/packages/builder/package.json b/packages/builder/package.json index fcf3d7a9f6..d46504f918 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -63,7 +63,7 @@ } }, "dependencies": { - "@budibase/bbui": "^1.34.2", + "@budibase/bbui": "^1.34.6", "@budibase/client": "^0.1.21", "@budibase/colorpicker": "^1.0.1", "@fortawesome/fontawesome-free": "^5.14.0", diff --git a/packages/builder/src/components/common/Dropzone.svelte b/packages/builder/src/components/common/Dropzone.svelte index a356ff811f..53cd606143 100644 --- a/packages/builder/src/components/common/Dropzone.svelte +++ b/packages/builder/src/components/common/Dropzone.svelte @@ -1,299 +1,32 @@ -
- - - - -
- - + diff --git a/packages/builder/yarn.lock b/packages/builder/yarn.lock index 27bc9d6dbd..2a5ee3d275 100644 --- a/packages/builder/yarn.lock +++ b/packages/builder/yarn.lock @@ -709,10 +709,10 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" -"@budibase/bbui@^1.34.2": - version "1.34.2" - resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.34.2.tgz#e4fcc728dc8d51a918f8ebd5c3f0b0afacfa4047" - integrity sha512-6RusGPZnEpHx1gtGcjk/lFLgMgFdDpSIxB8v2MiA+kp+uP1pFlzegbaDh+/JXyqFwK7HO91I0yXXBoPjibi7Aw== +"@budibase/bbui@^1.34.6": + version "1.34.6" + resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.34.6.tgz#d94a9c0af52244ded20dfcd7c93dbd6b184460dc" + integrity sha512-FLYKst1WDjQWpZPOm5w31M5mpdc4FZaHNT5UPyE+LTOtVJquUPycyS1Y/lhGjt/QjwP/Gn8wSvwwsD0gCNJvvg== dependencies: sirv-cli "^0.4.6" svelte-flatpickr "^2.4.0" diff --git a/packages/server/package.json b/packages/server/package.json index 93d4ff9e7c..2d2deca1a9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -59,7 +59,7 @@ "joi": "^17.2.1", "jsonwebtoken": "^8.5.1", "koa": "^2.7.0", - "koa-body": "^4.1.0", + "koa-body": "^4.2.0", "koa-compress": "^4.0.1", "koa-pino-logger": "^3.0.0", "koa-send": "^5.0.0", diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index 91f28793da..2f89d97742 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -64,19 +64,30 @@ function walkDir(dirPath, callback) { } } -function prepareUploadForS3({ filePath, s3Key, metadata, s3 }) { - const fileExtension = [...filePath.split(".")].pop() - const fileBytes = fs.readFileSync(filePath) - return s3 +async function prepareUploadForS3({ s3Key, metadata, s3, file }) { + const extension = [...file.name.split(".")].pop() + const fileBytes = fs.readFileSync(file.path) + + const upload = await s3 .upload({ Key: s3Key, Body: fileBytes, - ContentType: CONTENT_TYPE_MAP[fileExtension.toLowerCase()], + ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()], Metadata: metadata, }) .promise() + + return { + size: file.size, + name: file.name, + extension, + url: upload.Location, + key: upload.Key, + } } +exports.prepareUploadForS3 = prepareUploadForS3 + exports.uploadAppAssets = async function({ appId, instanceId, @@ -107,7 +118,10 @@ exports.uploadAppAssets = async function({ // Upload HTML, CSS and JS for each page of the web app walkDir(`${appAssetsPath}/${page}`, function(filePath) { const appAssetUpload = prepareUploadForS3({ - filePath, + file: { + path: filePath, + name: [...filePath.split("/")].pop(), + }, s3Key: filePath.replace(appAssetsPath, `assets/${appId}`), s3, metadata: { accountId }, @@ -124,8 +138,8 @@ exports.uploadAppAssets = async function({ if (file.uploaded) continue const attachmentUpload = prepareUploadForS3({ - filePath: file.path, - s3Key: `assets/${appId}/attachments/${file.name}`, + file, + s3Key: `assets/${appId}/attachments/${file.processedFileName}`, s3, metadata: { accountId }, }) diff --git a/packages/server/src/api/controllers/static.js b/packages/server/src/api/controllers/static.js index fd4afb42e7..663c4c7257 100644 --- a/packages/server/src/api/controllers/static.js +++ b/packages/server/src/api/controllers/static.js @@ -4,6 +4,8 @@ const jwt = require("jsonwebtoken") const fetch = require("node-fetch") const fs = require("fs") const uuid = require("uuid") +const AWS = require("aws-sdk") +const { prepareUploadForS3 } = require("./deploy/aws") const { budibaseAppsDir, @@ -22,8 +24,12 @@ exports.serveBuilder = async function(ctx) { await send(ctx, ctx.file, { root: ctx.devPath || builderPath }) } -exports.processLocalFileUpload = async function(ctx) { - const { files } = ctx.request.body +exports.uploadFile = async function(ctx) { + let files + files = + ctx.request.files.file.length > 1 + ? Array.from(ctx.request.files.file) + : [ctx.request.files.file] const attachmentsPath = resolve( budibaseAppsDir(), @@ -31,52 +37,99 @@ exports.processLocalFileUpload = async function(ctx) { "attachments" ) + if (process.env.CLOUD) { + // remote upload + const s3 = new AWS.S3({ + params: { + Bucket: "prod-budi-app-assets", + }, + }) + + const uploads = files.map(file => { + const fileExtension = [...file.name.split(".")].pop() + const processedFileName = `${uuid.v4()}.${fileExtension}` + + return prepareUploadForS3({ + file, + s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`, + s3, + }) + }) + + ctx.body = await Promise.all(uploads) + return + } + + ctx.body = await processLocalFileUploads({ + files, + outputPath: attachmentsPath, + instanceId: ctx.user.instanceId, + }) +} + +async function processLocalFileUploads({ files, outputPath, instanceId }) { // create attachments dir if it doesnt exist - !fs.existsSync(attachmentsPath) && - fs.mkdirSync(attachmentsPath, { recursive: true }) + !fs.existsSync(outputPath) && fs.mkdirSync(outputPath, { recursive: true }) const filesToProcess = files.map(file => { - const fileExtension = [...file.path.split(".")].pop() + const fileExtension = [...file.name.split(".")].pop() // filenames converted to UUIDs so they are unique - const fileName = `${uuid.v4()}.${fileExtension}` + const processedFileName = `${uuid.v4()}.${fileExtension}` return { - ...file, - fileName, + name: file.name, + path: file.path, + size: file.size, + type: file.type, + processedFileName, extension: fileExtension, - outputPath: join(attachmentsPath, fileName), - url: join("/attachments", fileName), + outputPath: join(outputPath, processedFileName), + url: join("/attachments", processedFileName), } }) - const fileProcessOperations = filesToProcess.map(file => - fileProcessor.process(file) + const fileProcessOperations = filesToProcess.map(fileProcessor.process) + + const processedFiles = await Promise.all(fileProcessOperations) + + let pendingFileUploads + // local document used to track which files need to be uploaded + // db.get throws an error if the document doesn't exist + // need to use a promise to default + const db = new CouchDB(instanceId) + await db + .get("_local/fileuploads") + .then(data => { + pendingFileUploads = data + }) + .catch(() => { + pendingFileUploads = { _id: "_local/fileuploads", uploads: [] } + }) + + pendingFileUploads.uploads = [ + ...processedFiles, + ...pendingFileUploads.uploads, + ] + await db.put(pendingFileUploads) + + return processedFiles +} + +exports.performLocalFileProcessing = async function(ctx) { + const { files } = ctx.request.body + + const processedFileOutputPath = resolve( + budibaseAppsDir(), + ctx.user.appId, + "attachments" ) try { - const processedFiles = await Promise.all(fileProcessOperations) - - let pendingFileUploads - // local document used to track which files need to be uploaded - // db.get throws an error if the document doesn't exist - // need to use a promise to default - const db = new CouchDB(ctx.user.instanceId) - await db - .get("_local/fileuploads") - .then(data => { - pendingFileUploads = data - }) - .catch(() => { - pendingFileUploads = { _id: "_local/fileuploads", uploads: [] } - }) - - pendingFileUploads.uploads = [ - ...processedFiles, - ...pendingFileUploads.uploads, - ] - await db.put(pendingFileUploads) - - ctx.body = processedFiles + ctx.body = await processLocalFileUploads({ + files, + outputPath: processedFileOutputPath, + instanceId: ctx.user.instanceId, + }) } catch (err) { ctx.throw(500, err) } diff --git a/packages/server/src/api/routes/static.js b/packages/server/src/api/routes/static.js index 0ce6a62668..aa136a3d15 100644 --- a/packages/server/src/api/routes/static.js +++ b/packages/server/src/api/routes/static.js @@ -26,8 +26,9 @@ router .post( "/api/attachments/process", authorized(BUILDER), - controller.processLocalFileUpload + controller.performLocalFileProcessing ) + .post("/api/attachments/upload", controller.uploadFile) .get("/componentlibrary", controller.serveComponentLibrary) .get("/assets/:file*", controller.serveAppAsset) .get("/attachments/:file*", controller.serveAttachment) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index cbdf6cd865..de283385a8 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -530,10 +530,12 @@ "@types/events@*": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" + integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== "@types/formidable@^1.0.31": version "1.0.31" resolved "https://registry.yarnpkg.com/@types/formidable/-/formidable-1.0.31.tgz#274f9dc2d0a1a9ce1feef48c24ca0859e7ec947b" + integrity sha512-dIhM5t8lRP0oWe2HF8MuPvdd1TpPTjhDMAqemcq6oIZQCBQTovhBAdTQ5L5veJB4pdQChadmHuxtB0YzqvfU3Q== dependencies: "@types/events" "*" "@types/node" "*" @@ -3946,9 +3948,10 @@ kleur@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" -koa-body@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/koa-body/-/koa-body-4.1.1.tgz#50686d290891fc6f1acb986cf7cfcd605f855ef0" +koa-body@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/koa-body/-/koa-body-4.2.0.tgz#37229208b820761aca5822d14c5fc55cee31b26f" + integrity sha512-wdGu7b9amk4Fnk/ytH8GuWwfs4fsB5iNkY8kZPpgQVb04QZSv85T0M8reb+cJmvLE8cjPYvBzRikD3s6qz8OoA== dependencies: "@types/formidable" "^1.0.31" co-body "^5.1.1" diff --git a/packages/standard-components/package.json b/packages/standard-components/package.json index 7048f9af63..ba3305d293 100644 --- a/packages/standard-components/package.json +++ b/packages/standard-components/package.json @@ -36,6 +36,7 @@ "gitHead": "284cceb9b703c38566c6e6363c022f79a08d5691", "dependencies": { "@beyonk/svelte-googlemaps": "^2.2.0", + "@budibase/bbui": "^1.34.6", "@fortawesome/fontawesome-free": "^5.14.0", "@budibase/bbui": "^1.34.2", "britecharts": "^2.16.1", diff --git a/packages/standard-components/src/DataForm.svelte b/packages/standard-components/src/DataForm.svelte index ba5998e5c6..0b8267c87f 100644 --- a/packages/standard-components/src/DataForm.svelte +++ b/packages/standard-components/src/DataForm.svelte @@ -2,6 +2,7 @@ import { onMount } from "svelte" import { fade } from "svelte/transition" import { Label, DatePicker } from "@budibase/bbui" + import Dropzone from "./attachments/Dropzone.svelte" import debounce from "lodash.debounce" export let _bb @@ -54,8 +55,9 @@ const save = debounce(async () => { for (let field of fields) { // Assign defaults to empty fields to prevent validation issues - if (!(field in record)) + if (!(field in record)) { record[field] = DEFAULTS_FOR_TYPE[schema[field].type] + } } const SAVE_RECORD_URL = `/api/${model}/records` @@ -132,6 +134,8 @@ {:else if schema[field].type === 'string'} + {:else if schema[field].type === 'attachment'} + {/if}
diff --git a/packages/standard-components/src/DataTable.svelte b/packages/standard-components/src/DataTable.svelte index 10f132d017..fe967338f5 100644 --- a/packages/standard-components/src/DataTable.svelte +++ b/packages/standard-components/src/DataTable.svelte @@ -6,6 +6,7 @@ import fsort from "fast-sort" import fetchData from "./fetchData.js" import { isEmpty } from "lodash/fp" + import AttachmentList from "./attachments/AttachmentList.svelte" export let backgroundColor export let color @@ -17,6 +18,7 @@ let headers = [] let sort = {} let sorted = [] + let schema = {} $: cssVariables = { backgroundColor, @@ -83,7 +85,10 @@ {#each sorted as row (row._id)} {#each headers as header} - {#if row[header]} + + {#if Array.isArray(row[header])} + + {:else if row[header]} {row[header]} {/if} {/each} diff --git a/packages/standard-components/src/api.js b/packages/standard-components/src/api.js index da29c70578..45e7a8f134 100644 --- a/packages/standard-components/src/api.js +++ b/packages/standard-components/src/api.js @@ -1,7 +1,10 @@ -const apiCall = method => async (url, body) => { - const headers = { +const apiCall = method => async ( + url, + body, + headers = { "Content-Type": "application/json", } +) => { const response = await fetch(url, { method: method, body: body && JSON.stringify(body), diff --git a/packages/standard-components/src/attachments/AttachmentList.svelte b/packages/standard-components/src/attachments/AttachmentList.svelte new file mode 100644 index 0000000000..950c1e43b6 --- /dev/null +++ b/packages/standard-components/src/attachments/AttachmentList.svelte @@ -0,0 +1,64 @@ + + +
+ {#each files as file} + +
+ {#if FILE_TYPES.IMAGE.includes(file.extension.toLowerCase())} + + {:else} + + {/if} +
+ {file.name} + + {/each} +
+ + diff --git a/packages/standard-components/src/attachments/Dropzone.svelte b/packages/standard-components/src/attachments/Dropzone.svelte new file mode 100644 index 0000000000..9d68c920c2 --- /dev/null +++ b/packages/standard-components/src/attachments/Dropzone.svelte @@ -0,0 +1,35 @@ + + + diff --git a/packages/standard-components/src/attachments/fileTypes.js b/packages/standard-components/src/attachments/fileTypes.js new file mode 100644 index 0000000000..2ce6958f2d --- /dev/null +++ b/packages/standard-components/src/attachments/fileTypes.js @@ -0,0 +1,5 @@ +export const FILE_TYPES = { + IMAGE: ["png", "tiff", "gif", "raw", "jpg", "jpeg"], + CODE: ["js", "rs", "py", "java", "rb", "hs", "yml"], + DOCUMENT: ["odf", "docx", "doc", "pdf", "csv"], +}