Merge branch 'master' of github.com:Budibase/budibase into feature/sqs-table-cleanup
This commit is contained in:
commit
6ff00135bf
29 changed files with 649 additions and 112 deletions
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"version": "2.23.10",
|
||||
"version": "2.23.11",
|
||||
"npmClient": "yarn",
|
||||
"packages": [
|
||||
"packages/*",
|
||||
|
|
|
@ -29,6 +29,7 @@ const DefaultBucketName = {
|
|||
TEMPLATES: "templates",
|
||||
GLOBAL: "global",
|
||||
PLUGINS: "plugins",
|
||||
TEMP: "tmp-file-attachments",
|
||||
}
|
||||
|
||||
const selfHosted = !!parseInt(process.env.SELF_HOSTED || "")
|
||||
|
@ -146,6 +147,7 @@ const environment = {
|
|||
process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL,
|
||||
PLUGIN_BUCKET_NAME:
|
||||
process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
|
||||
TEMP_BUCKET_NAME: process.env.TEMP_BUCKET_NAME || DefaultBucketName.TEMP,
|
||||
USE_COUCH: process.env.USE_COUCH || true,
|
||||
MOCK_REDIS: process.env.MOCK_REDIS,
|
||||
DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
|
||||
|
|
|
@ -7,31 +7,41 @@ import tar from "tar-fs"
|
|||
import zlib from "zlib"
|
||||
import { promisify } from "util"
|
||||
import { join } from "path"
|
||||
import fs, { ReadStream } from "fs"
|
||||
import fs, { PathLike, ReadStream } from "fs"
|
||||
import env from "../environment"
|
||||
import { budibaseTempDir } from "./utils"
|
||||
import { bucketTTLConfig, budibaseTempDir } from "./utils"
|
||||
import { v4 } from "uuid"
|
||||
import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
|
||||
import fsp from "fs/promises"
|
||||
|
||||
const streamPipeline = promisify(stream.pipeline)
|
||||
// use this as a temporary store of buckets that are being created
|
||||
const STATE = {
|
||||
bucketCreationPromises: {},
|
||||
}
|
||||
const signedFilePrefix = "/files/signed"
|
||||
|
||||
type ListParams = {
|
||||
ContinuationToken?: string
|
||||
}
|
||||
|
||||
type UploadParams = {
|
||||
type BaseUploadParams = {
|
||||
bucket: string
|
||||
filename: string
|
||||
path: string
|
||||
type?: string | null
|
||||
// can be undefined, we will remove it
|
||||
metadata?: {
|
||||
[key: string]: string | undefined
|
||||
}
|
||||
metadata?: { [key: string]: string | undefined }
|
||||
body?: ReadableStream | Buffer
|
||||
ttl?: number
|
||||
addTTL?: boolean
|
||||
extra?: any
|
||||
}
|
||||
|
||||
type UploadParams = BaseUploadParams & {
|
||||
path?: string | PathLike
|
||||
}
|
||||
|
||||
type StreamUploadParams = BaseUploadParams & {
|
||||
stream: ReadStream
|
||||
}
|
||||
|
||||
const CONTENT_TYPE_MAP: any = {
|
||||
|
@ -41,6 +51,8 @@ const CONTENT_TYPE_MAP: any = {
|
|||
js: "application/javascript",
|
||||
json: "application/json",
|
||||
gz: "application/gzip",
|
||||
svg: "image/svg+xml",
|
||||
form: "multipart/form-data",
|
||||
}
|
||||
|
||||
const STRING_CONTENT_TYPES = [
|
||||
|
@ -105,7 +117,10 @@ export function ObjectStore(
|
|||
* Given an object store and a bucket name this will make sure the bucket exists,
|
||||
* if it does not exist then it will create it.
|
||||
*/
|
||||
export async function makeSureBucketExists(client: any, bucketName: string) {
|
||||
export async function createBucketIfNotExists(
|
||||
client: any,
|
||||
bucketName: string
|
||||
): Promise<{ created: boolean; exists: boolean }> {
|
||||
bucketName = sanitizeBucket(bucketName)
|
||||
try {
|
||||
await client
|
||||
|
@ -113,15 +128,16 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
|
|||
Bucket: bucketName,
|
||||
})
|
||||
.promise()
|
||||
return { created: false, exists: true }
|
||||
} catch (err: any) {
|
||||
const promises: any = STATE.bucketCreationPromises
|
||||
const doesntExist = err.statusCode === 404,
|
||||
noAccess = err.statusCode === 403
|
||||
if (promises[bucketName]) {
|
||||
await promises[bucketName]
|
||||
return { created: false, exists: true }
|
||||
} else if (doesntExist || noAccess) {
|
||||
if (doesntExist) {
|
||||
// bucket doesn't exist create it
|
||||
promises[bucketName] = client
|
||||
.createBucket({
|
||||
Bucket: bucketName,
|
||||
|
@ -129,13 +145,15 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
|
|||
.promise()
|
||||
await promises[bucketName]
|
||||
delete promises[bucketName]
|
||||
return { created: true, exists: false }
|
||||
} else {
|
||||
throw new Error("Access denied to object store bucket." + err)
|
||||
}
|
||||
} else {
|
||||
throw new Error("Unable to write to object store bucket.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads the contents of a file given the required parameters, useful when
|
||||
* temp files in use (for example file uploaded as an attachment).
|
||||
|
@ -146,12 +164,22 @@ export async function upload({
|
|||
path,
|
||||
type,
|
||||
metadata,
|
||||
body,
|
||||
ttl,
|
||||
}: UploadParams) {
|
||||
const extension = filename.split(".").pop()
|
||||
const fileBytes = fs.readFileSync(path)
|
||||
|
||||
const fileBytes = path ? (await fsp.open(path)).createReadStream() : body
|
||||
|
||||
const objectStore = ObjectStore(bucketName)
|
||||
await makeSureBucketExists(objectStore, bucketName)
|
||||
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
|
||||
|
||||
if (ttl && (bucketCreated.created || bucketCreated.exists)) {
|
||||
let ttlConfig = bucketTTLConfig(bucketName, ttl)
|
||||
if (objectStore.putBucketLifecycleConfiguration) {
|
||||
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
|
||||
}
|
||||
}
|
||||
|
||||
let contentType = type
|
||||
if (!contentType) {
|
||||
|
@ -174,6 +202,7 @@ export async function upload({
|
|||
}
|
||||
config.Metadata = metadata
|
||||
}
|
||||
|
||||
return objectStore.upload(config).promise()
|
||||
}
|
||||
|
||||
|
@ -181,14 +210,24 @@ export async function upload({
|
|||
* Similar to the upload function but can be used to send a file stream
|
||||
* through to the object store.
|
||||
*/
|
||||
export async function streamUpload(
|
||||
bucketName: string,
|
||||
filename: string,
|
||||
stream: ReadStream | ReadableStream,
|
||||
extra = {}
|
||||
) {
|
||||
export async function streamUpload({
|
||||
bucket: bucketName,
|
||||
stream,
|
||||
filename,
|
||||
type,
|
||||
extra,
|
||||
ttl,
|
||||
}: StreamUploadParams) {
|
||||
const extension = filename.split(".").pop()
|
||||
const objectStore = ObjectStore(bucketName)
|
||||
await makeSureBucketExists(objectStore, bucketName)
|
||||
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
|
||||
|
||||
if (ttl && (bucketCreated.created || bucketCreated.exists)) {
|
||||
let ttlConfig = bucketTTLConfig(bucketName, ttl)
|
||||
if (objectStore.putBucketLifecycleConfiguration) {
|
||||
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
|
||||
}
|
||||
}
|
||||
|
||||
// Set content type for certain known extensions
|
||||
if (filename?.endsWith(".js")) {
|
||||
|
@ -203,10 +242,18 @@ export async function streamUpload(
|
|||
}
|
||||
}
|
||||
|
||||
let contentType = type
|
||||
if (!contentType) {
|
||||
contentType = extension
|
||||
? CONTENT_TYPE_MAP[extension.toLowerCase()]
|
||||
: CONTENT_TYPE_MAP.txt
|
||||
}
|
||||
|
||||
const params = {
|
||||
Bucket: sanitizeBucket(bucketName),
|
||||
Key: sanitizeKey(filename),
|
||||
Body: stream,
|
||||
ContentType: contentType,
|
||||
...extra,
|
||||
}
|
||||
return objectStore.upload(params).promise()
|
||||
|
@ -286,7 +333,7 @@ export function getPresignedUrl(
|
|||
const signedUrl = new URL(url)
|
||||
const path = signedUrl.pathname
|
||||
const query = signedUrl.search
|
||||
return `/files/signed${path}${query}`
|
||||
return `${signedFilePrefix}${path}${query}`
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -341,7 +388,7 @@ export async function retrieveDirectory(bucketName: string, path: string) {
|
|||
*/
|
||||
export async function deleteFile(bucketName: string, filepath: string) {
|
||||
const objectStore = ObjectStore(bucketName)
|
||||
await makeSureBucketExists(objectStore, bucketName)
|
||||
await createBucketIfNotExists(objectStore, bucketName)
|
||||
const params = {
|
||||
Bucket: bucketName,
|
||||
Key: sanitizeKey(filepath),
|
||||
|
@ -351,7 +398,7 @@ export async function deleteFile(bucketName: string, filepath: string) {
|
|||
|
||||
export async function deleteFiles(bucketName: string, filepaths: string[]) {
|
||||
const objectStore = ObjectStore(bucketName)
|
||||
await makeSureBucketExists(objectStore, bucketName)
|
||||
await createBucketIfNotExists(objectStore, bucketName)
|
||||
const params = {
|
||||
Bucket: bucketName,
|
||||
Delete: {
|
||||
|
@ -412,7 +459,13 @@ export async function uploadDirectory(
|
|||
if (file.isDirectory()) {
|
||||
uploads.push(uploadDirectory(bucketName, local, path))
|
||||
} else {
|
||||
uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
|
||||
uploads.push(
|
||||
streamUpload({
|
||||
bucket: bucketName,
|
||||
filename: path,
|
||||
stream: fs.createReadStream(local),
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
await Promise.all(uploads)
|
||||
|
@ -467,3 +520,23 @@ export async function getReadStream(
|
|||
}
|
||||
return client.getObject(params).createReadStream()
|
||||
}
|
||||
|
||||
/*
|
||||
Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
|
||||
the bucket and the path from it
|
||||
*/
|
||||
export function extractBucketAndPath(
|
||||
url: string
|
||||
): { bucket: string; path: string } | null {
|
||||
const baseUrl = url.split("?")[0]
|
||||
|
||||
const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
|
||||
const match = baseUrl.match(regex)
|
||||
|
||||
if (match && match.groups) {
|
||||
const { bucket, path } = match.groups
|
||||
return { bucket, path }
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ import { join } from "path"
|
|||
import { tmpdir } from "os"
|
||||
import fs from "fs"
|
||||
import env from "../environment"
|
||||
import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
|
||||
|
||||
/****************************************************
|
||||
* NOTE: When adding a new bucket - name *
|
||||
|
@ -15,6 +16,7 @@ export const ObjectStoreBuckets = {
|
|||
TEMPLATES: env.TEMPLATES_BUCKET_NAME,
|
||||
GLOBAL: env.GLOBAL_BUCKET_NAME,
|
||||
PLUGINS: env.PLUGIN_BUCKET_NAME,
|
||||
TEMP: env.TEMP_BUCKET_NAME,
|
||||
}
|
||||
|
||||
const bbTmp = join(tmpdir(), ".budibase")
|
||||
|
@ -29,3 +31,27 @@ try {
|
|||
export function budibaseTempDir() {
|
||||
return bbTmp
|
||||
}
|
||||
|
||||
export const bucketTTLConfig = (
|
||||
bucketName: string,
|
||||
days: number
|
||||
): PutBucketLifecycleConfigurationRequest => {
|
||||
const lifecycleRule = {
|
||||
ID: `${bucketName}-ExpireAfter${days}days`,
|
||||
Prefix: "",
|
||||
Status: "Enabled",
|
||||
Expiration: {
|
||||
Days: days,
|
||||
},
|
||||
}
|
||||
const lifecycleConfiguration = {
|
||||
Rules: [lifecycleRule],
|
||||
}
|
||||
|
||||
const params = {
|
||||
Bucket: bucketName,
|
||||
LifecycleConfiguration: lifecycleConfiguration,
|
||||
}
|
||||
|
||||
return params
|
||||
}
|
||||
|
|
|
@ -4,3 +4,6 @@ export { generator } from "./structures"
|
|||
export * as testContainerUtils from "./testContainerUtils"
|
||||
export * as utils from "./utils"
|
||||
export * from "./jestUtils"
|
||||
import * as minio from "./minio"
|
||||
|
||||
export const objectStoreTestProviders = { minio }
|
||||
|
|
34
packages/backend-core/tests/core/utilities/minio.ts
Normal file
34
packages/backend-core/tests/core/utilities/minio.ts
Normal file
|
@ -0,0 +1,34 @@
|
|||
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
|
||||
import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
|
||||
import env from "../../../src/environment"
|
||||
|
||||
let container: StartedTestContainer | undefined
|
||||
|
||||
class ObjectStoreWaitStrategy extends AbstractWaitStrategy {
|
||||
async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
|
||||
const logs = Wait.forListeningPorts()
|
||||
await logs.waitUntilReady(container, boundPorts, startTime)
|
||||
}
|
||||
}
|
||||
|
||||
export async function start(): Promise<void> {
|
||||
container = await new GenericContainer("minio/minio")
|
||||
.withExposedPorts(9000)
|
||||
.withCommand(["server", "/data"])
|
||||
.withEnvironment({
|
||||
MINIO_ACCESS_KEY: "budibase",
|
||||
MINIO_SECRET_KEY: "budibase",
|
||||
})
|
||||
.withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000))
|
||||
.start()
|
||||
|
||||
const port = container.getMappedPort(9000)
|
||||
env._set("MINIO_URL", `http://0.0.0.0:${port}`)
|
||||
}
|
||||
|
||||
export async function stop() {
|
||||
if (container) {
|
||||
await container.stop()
|
||||
container = undefined
|
||||
}
|
||||
}
|
|
@ -32,6 +32,7 @@
|
|||
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
|
||||
import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte"
|
||||
import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte"
|
||||
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
|
||||
import { BindingHelpers, BindingType } from "components/common/bindings/utils"
|
||||
import {
|
||||
bindingsToCompletions,
|
||||
|
@ -356,7 +357,8 @@
|
|||
value.customType !== "queryParams" &&
|
||||
value.customType !== "cron" &&
|
||||
value.customType !== "triggerSchema" &&
|
||||
value.customType !== "automationFields"
|
||||
value.customType !== "automationFields" &&
|
||||
value.type !== "attachment"
|
||||
)
|
||||
}
|
||||
|
||||
|
@ -372,6 +374,15 @@
|
|||
console.error(error)
|
||||
}
|
||||
})
|
||||
const handleAttachmentParams = keyValuObj => {
|
||||
let params = {}
|
||||
if (keyValuObj?.length) {
|
||||
for (let param of keyValuObj) {
|
||||
params[param.url] = param.filename
|
||||
}
|
||||
}
|
||||
return params
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="fields">
|
||||
|
@ -437,6 +448,33 @@
|
|||
value={inputData[key]}
|
||||
options={Object.keys(table?.schema || {})}
|
||||
/>
|
||||
{:else if value.type === "attachment"}
|
||||
<div class="attachment-field-wrapper">
|
||||
<div class="label-wrapper">
|
||||
<Label>{label}</Label>
|
||||
</div>
|
||||
<div class="attachment-field-width">
|
||||
<KeyValueBuilder
|
||||
on:change={e =>
|
||||
onChange(
|
||||
{
|
||||
detail: e.detail.map(({ name, value }) => ({
|
||||
url: name,
|
||||
filename: value,
|
||||
})),
|
||||
},
|
||||
key
|
||||
)}
|
||||
object={handleAttachmentParams(inputData[key])}
|
||||
allowJS
|
||||
{bindings}
|
||||
keyBindings
|
||||
customButtonText={"Add attachment"}
|
||||
keyPlaceholder={"URL"}
|
||||
valuePlaceholder={"Filename"}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{:else if value.customType === "filters"}
|
||||
<ActionButton on:click={drawer.show}>Define filters</ActionButton>
|
||||
<Drawer bind:this={drawer} title="Filtering">
|
||||
|
@ -651,14 +689,22 @@
|
|||
}
|
||||
|
||||
.block-field {
|
||||
display: flex; /* Use Flexbox */
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
flex-direction: row; /* Arrange label and field side by side */
|
||||
align-items: center; /* Align vertically in the center */
|
||||
gap: 10px; /* Add some space between label and field */
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.attachment-field-width {
|
||||
margin-top: var(--spacing-xs);
|
||||
}
|
||||
|
||||
.label-wrapper {
|
||||
margin-top: var(--spacing-s);
|
||||
}
|
||||
|
||||
.test :global(.drawer) {
|
||||
width: 10000px !important;
|
||||
}
|
||||
|
|
|
@ -35,6 +35,8 @@
|
|||
export let bindingDrawerLeft
|
||||
export let allowHelpers = true
|
||||
export let customButtonText = null
|
||||
export let keyBindings = false
|
||||
export let allowJS = false
|
||||
export let compare = (option, value) => option === value
|
||||
|
||||
let fields = Object.entries(object || {}).map(([name, value]) => ({
|
||||
|
@ -116,12 +118,23 @@
|
|||
class:readOnly-menu={readOnly && showMenu}
|
||||
>
|
||||
{#each fields as field, idx}
|
||||
<Input
|
||||
placeholder={keyPlaceholder}
|
||||
readonly={readOnly}
|
||||
bind:value={field.name}
|
||||
on:blur={changed}
|
||||
/>
|
||||
{#if keyBindings}
|
||||
<DrawerBindableInput
|
||||
{bindings}
|
||||
placeholder={keyPlaceholder}
|
||||
on:blur={e => {
|
||||
field.name = e.detail
|
||||
changed()
|
||||
}}
|
||||
disabled={readOnly}
|
||||
value={field.name}
|
||||
{allowJS}
|
||||
{allowHelpers}
|
||||
drawerLeft={bindingDrawerLeft}
|
||||
/>
|
||||
{:else}
|
||||
<Input readonly={readOnly} bind:value={field.name} on:blur={changed} />
|
||||
{/if}
|
||||
{#if isJsonArray(field.value)}
|
||||
<Select readonly={true} value="Array" options={["Array"]} />
|
||||
{:else if options}
|
||||
|
@ -134,14 +147,14 @@
|
|||
{:else if bindings && bindings.length}
|
||||
<DrawerBindableInput
|
||||
{bindings}
|
||||
placeholder="Value"
|
||||
placeholder={valuePlaceholder}
|
||||
on:blur={e => {
|
||||
field.value = e.detail
|
||||
changed()
|
||||
}}
|
||||
disabled={readOnly}
|
||||
value={field.value}
|
||||
allowJS={false}
|
||||
{allowJS}
|
||||
{allowHelpers}
|
||||
drawerLeft={bindingDrawerLeft}
|
||||
/>
|
||||
|
|
|
@ -22,6 +22,7 @@ import {
|
|||
isJSBinding,
|
||||
decodeJSBinding,
|
||||
encodeJSBinding,
|
||||
getJsHelperList,
|
||||
} from "@budibase/string-templates"
|
||||
import { TableNames } from "./constants"
|
||||
import { JSONUtils, Constants } from "@budibase/frontend-core"
|
||||
|
@ -1210,9 +1211,32 @@ const shouldReplaceBinding = (currentValue, from, convertTo, binding) => {
|
|||
if (!currentValue?.includes(from)) {
|
||||
return false
|
||||
}
|
||||
if (convertTo === "readableBinding") {
|
||||
// Dont replace if the value already matches the readable binding
|
||||
// some cases we have the same binding for readable/runtime, specific logic for this
|
||||
const sameBindings = binding.runtimeBinding.includes(binding.readableBinding)
|
||||
const convertingToReadable = convertTo === "readableBinding"
|
||||
const helperNames = Object.keys(getJsHelperList())
|
||||
const matchedHelperNames = helperNames.filter(
|
||||
name => name.includes(from) && currentValue.includes(name)
|
||||
)
|
||||
// edge case - if the binding is part of a helper it may accidentally replace it
|
||||
if (matchedHelperNames.length > 0) {
|
||||
const indexStart = currentValue.indexOf(from),
|
||||
indexEnd = indexStart + from.length
|
||||
for (let helperName of matchedHelperNames) {
|
||||
const helperIndexStart = currentValue.indexOf(helperName),
|
||||
helperIndexEnd = helperIndexStart + helperName.length
|
||||
if (indexStart >= helperIndexStart && indexEnd <= helperIndexEnd) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (convertingToReadable && !sameBindings) {
|
||||
// Don't replace if the value already matches the readable binding
|
||||
return currentValue.indexOf(binding.readableBinding) === -1
|
||||
} else if (convertingToReadable) {
|
||||
// if the runtime and readable bindings are very similar we have to assume it should be replaced
|
||||
return true
|
||||
}
|
||||
// remove all the spaces, if the input is surrounded by spaces e.g. [ Auto ID ] then
|
||||
// this makes sure it is detected
|
||||
|
|
|
@ -9,7 +9,7 @@ const {
|
|||
ObjectStore,
|
||||
retrieve,
|
||||
uploadDirectory,
|
||||
makeSureBucketExists,
|
||||
createBucketIfNotExists,
|
||||
} = objectStore
|
||||
|
||||
const bucketList = Object.values(ObjectStoreBuckets)
|
||||
|
@ -61,7 +61,7 @@ export async function importObjects() {
|
|||
let count = 0
|
||||
for (let bucket of buckets) {
|
||||
const client = ObjectStore(bucket)
|
||||
await makeSureBucketExists(client, bucket)
|
||||
await createBucketIfNotExists(client, bucket)
|
||||
const files = await uploadDirectory(bucket, join(path, bucket), "/")
|
||||
count += files.length
|
||||
bar.update(count)
|
||||
|
|
|
@ -54,11 +54,9 @@ export async function downloadDockerCompose() {
|
|||
|
||||
export async function checkDockerConfigured() {
|
||||
const error =
|
||||
"docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
|
||||
"docker has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
|
||||
const docker = await lookpath("docker")
|
||||
const compose = await lookpath("docker-compose")
|
||||
const composeV2 = await lookpath("docker compose")
|
||||
if (!docker || (!compose && !composeV2)) {
|
||||
if (!docker) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit b55d5b32003e3e999a1cbf2e5f3e6ce8d71eace7
|
||||
Subproject commit dff7b5a9dd1fd770f8a48fb8e6df1740be605f18
|
|
@ -61,14 +61,17 @@
|
|||
"@google-cloud/firestore": "6.8.0",
|
||||
"@koa/router": "8.0.8",
|
||||
"@socket.io/redis-adapter": "^8.2.1",
|
||||
"@types/xml2js": "^0.4.14",
|
||||
"airtable": "0.10.1",
|
||||
"arangojs": "7.2.0",
|
||||
"archiver": "7.0.1",
|
||||
"aws-sdk": "2.1030.0",
|
||||
"bcrypt": "5.1.0",
|
||||
"bcryptjs": "2.4.3",
|
||||
"bl": "^6.0.12",
|
||||
"bull": "4.10.1",
|
||||
"chokidar": "3.5.3",
|
||||
"content-disposition": "^0.5.4",
|
||||
"cookies": "0.8.0",
|
||||
"csvtojson": "2.0.10",
|
||||
"curlconverter": "3.21.0",
|
||||
|
|
|
@ -7,6 +7,7 @@ import {
|
|||
AutomationStepType,
|
||||
AutomationIOType,
|
||||
AutomationFeature,
|
||||
AutomationCustomIOType,
|
||||
} from "@budibase/types"
|
||||
|
||||
export const definition: AutomationStepSchema = {
|
||||
|
@ -72,10 +73,10 @@ export const definition: AutomationStepSchema = {
|
|||
title: "Location",
|
||||
dependsOn: "addInvite",
|
||||
},
|
||||
url: {
|
||||
type: AutomationIOType.STRING,
|
||||
title: "URL",
|
||||
dependsOn: "addInvite",
|
||||
attachments: {
|
||||
type: AutomationIOType.ATTACHMENT,
|
||||
customType: AutomationCustomIOType.MULTI_ATTACHMENTS,
|
||||
title: "Attachments",
|
||||
},
|
||||
},
|
||||
required: ["to", "from", "subject", "contents"],
|
||||
|
@ -110,11 +111,13 @@ export async function run({ inputs }: AutomationStepInput) {
|
|||
summary,
|
||||
location,
|
||||
url,
|
||||
attachments,
|
||||
} = inputs
|
||||
if (!contents) {
|
||||
contents = "<h1>No content</h1>"
|
||||
}
|
||||
to = to || undefined
|
||||
|
||||
try {
|
||||
let response = await sendSmtpEmail({
|
||||
to,
|
||||
|
@ -124,6 +127,7 @@ export async function run({ inputs }: AutomationStepInput) {
|
|||
cc,
|
||||
bcc,
|
||||
automation: true,
|
||||
attachments,
|
||||
invite: addInvite
|
||||
? {
|
||||
startTime,
|
||||
|
|
|
@ -50,6 +50,10 @@ describe("test the outgoing webhook action", () => {
|
|||
cc: "cc",
|
||||
bcc: "bcc",
|
||||
addInvite: true,
|
||||
attachments: [
|
||||
{ url: "attachment1", filename: "attachment1.txt" },
|
||||
{ url: "attachment2", filename: "attachment2.txt" },
|
||||
],
|
||||
...invite,
|
||||
}
|
||||
let resp = generateResponse(inputs.to, inputs.from)
|
||||
|
@ -69,6 +73,10 @@ describe("test the outgoing webhook action", () => {
|
|||
bcc: "bcc",
|
||||
invite,
|
||||
automation: true,
|
||||
attachments: [
|
||||
{ url: "attachment1", filename: "attachment1.txt" },
|
||||
{ url: "attachment2", filename: "attachment2.txt" },
|
||||
],
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
@ -21,6 +21,10 @@ import { performance } from "perf_hooks"
|
|||
import FormData from "form-data"
|
||||
import { URLSearchParams } from "url"
|
||||
import { blacklist } from "@budibase/backend-core"
|
||||
import { handleFileResponse, handleXml } from "./utils"
|
||||
import { parse } from "content-disposition"
|
||||
import path from "path"
|
||||
import { Builder as XmlBuilder } from "xml2js"
|
||||
|
||||
const BodyTypes = {
|
||||
NONE: "none",
|
||||
|
@ -57,8 +61,6 @@ const coreFields = {
|
|||
},
|
||||
}
|
||||
|
||||
const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
|
||||
|
||||
const SCHEMA: Integration = {
|
||||
docs: "https://github.com/node-fetch/node-fetch",
|
||||
description:
|
||||
|
@ -129,42 +131,44 @@ class RestIntegration implements IntegrationBase {
|
|||
}
|
||||
|
||||
async parseResponse(response: any, pagination: PaginationConfig | null) {
|
||||
let data, raw, headers
|
||||
let data, raw, headers, filename
|
||||
|
||||
const contentType = response.headers.get("content-type") || ""
|
||||
const contentDisposition = response.headers.get("content-disposition") || ""
|
||||
if (
|
||||
contentDisposition.includes("attachment") ||
|
||||
contentDisposition.includes("form-data")
|
||||
) {
|
||||
filename =
|
||||
path.basename(parse(contentDisposition).parameters?.filename) || ""
|
||||
}
|
||||
|
||||
try {
|
||||
if (response.status === 204) {
|
||||
data = []
|
||||
raw = []
|
||||
} else if (contentType.includes("application/json")) {
|
||||
data = await response.json()
|
||||
raw = JSON.stringify(data)
|
||||
} else if (
|
||||
contentType.includes("text/xml") ||
|
||||
contentType.includes("application/xml")
|
||||
) {
|
||||
const rawXml = await response.text()
|
||||
data =
|
||||
(await xmlParser(rawXml, {
|
||||
explicitArray: false,
|
||||
trim: true,
|
||||
explicitRoot: false,
|
||||
})) || {}
|
||||
// there is only one structure, its an array, return the array so it appears as rows
|
||||
const keys = Object.keys(data)
|
||||
if (keys.length === 1 && Array.isArray(data[keys[0]])) {
|
||||
data = data[keys[0]]
|
||||
}
|
||||
raw = rawXml
|
||||
} else if (contentType.includes("application/pdf")) {
|
||||
data = await response.arrayBuffer() // Save PDF as ArrayBuffer
|
||||
raw = Buffer.from(data)
|
||||
if (filename) {
|
||||
return handleFileResponse(response, filename, this.startTimeMs)
|
||||
} else {
|
||||
data = await response.text()
|
||||
raw = data
|
||||
if (response.status === 204) {
|
||||
data = []
|
||||
raw = []
|
||||
} else if (contentType.includes("application/json")) {
|
||||
data = await response.json()
|
||||
raw = JSON.stringify(data)
|
||||
} else if (
|
||||
contentType.includes("text/xml") ||
|
||||
contentType.includes("application/xml")
|
||||
) {
|
||||
let xmlResponse = await handleXml(response)
|
||||
data = xmlResponse.data
|
||||
raw = xmlResponse.rawXml
|
||||
} else {
|
||||
data = await response.text()
|
||||
raw = data
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
throw "Failed to parse response body."
|
||||
throw `Failed to parse response body: ${err}`
|
||||
}
|
||||
|
||||
const size = formatBytes(
|
||||
response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
|
||||
)
|
||||
|
|
|
@ -13,9 +13,23 @@ jest.mock("node-fetch", () => {
|
|||
}))
|
||||
})
|
||||
|
||||
import fetch from "node-fetch"
|
||||
jest.mock("@budibase/backend-core", () => {
|
||||
const core = jest.requireActual("@budibase/backend-core")
|
||||
return {
|
||||
...core,
|
||||
context: {
|
||||
...core.context,
|
||||
getProdAppId: jest.fn(() => "app-id"),
|
||||
},
|
||||
}
|
||||
})
|
||||
jest.mock("uuid", () => ({ v4: () => "00000000-0000-0000-0000-000000000000" }))
|
||||
|
||||
import { default as RestIntegration } from "../rest"
|
||||
import { RestAuthType } from "@budibase/types"
|
||||
import fetch from "node-fetch"
|
||||
import { objectStoreTestProviders } from "@budibase/backend-core/tests"
|
||||
import { Readable } from "stream"
|
||||
|
||||
const FormData = require("form-data")
|
||||
const { URLSearchParams } = require("url")
|
||||
|
@ -611,4 +625,104 @@ describe("REST Integration", () => {
|
|||
expect(calledConfig.headers).toEqual({})
|
||||
expect(calledConfig.agent.options.rejectUnauthorized).toBe(false)
|
||||
})
|
||||
|
||||
describe("File Handling", () => {
|
||||
beforeAll(async () => {
|
||||
jest.unmock("aws-sdk")
|
||||
await objectStoreTestProviders.minio.start()
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await objectStoreTestProviders.minio.stop()
|
||||
})
|
||||
|
||||
it("uploads file to object store and returns signed URL", async () => {
|
||||
const responseData = Buffer.from("teest file contnt")
|
||||
const filename = "test.tar.gz"
|
||||
const contentType = "application/gzip"
|
||||
const mockReadable = new Readable()
|
||||
mockReadable.push(responseData)
|
||||
mockReadable.push(null)
|
||||
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
|
||||
Promise.resolve({
|
||||
headers: {
|
||||
raw: () => ({
|
||||
"content-type": [contentType],
|
||||
"content-disposition": [`attachment; filename="${filename}"`],
|
||||
}),
|
||||
get: (header: any) => {
|
||||
if (header === "content-type") return contentType
|
||||
if (header === "content-disposition")
|
||||
return `attachment; filename="${filename}"`
|
||||
},
|
||||
},
|
||||
body: mockReadable,
|
||||
})
|
||||
)
|
||||
|
||||
const query = {
|
||||
path: "api",
|
||||
}
|
||||
|
||||
const response = await config.integration.read(query)
|
||||
|
||||
expect(response.data).toEqual({
|
||||
size: responseData.byteLength,
|
||||
name: "00000000-0000-0000-0000-000000000000.tar.gz",
|
||||
url: expect.stringContaining(
|
||||
"/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.tar.gz"
|
||||
),
|
||||
extension: "tar.gz",
|
||||
key: expect.stringContaining(
|
||||
"app-id/00000000-0000-0000-0000-000000000000.tar.gz"
|
||||
),
|
||||
})
|
||||
})
|
||||
|
||||
it("uploads file with non ascii filename to object store and returns signed URL", async () => {
|
||||
const responseData = Buffer.from("teest file contnt")
|
||||
const contentType = "text/plain"
|
||||
const mockReadable = new Readable()
|
||||
mockReadable.push(responseData)
|
||||
mockReadable.push(null)
|
||||
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
|
||||
Promise.resolve({
|
||||
headers: {
|
||||
raw: () => ({
|
||||
"content-type": [contentType],
|
||||
"content-disposition": [
|
||||
// eslint-disable-next-line no-useless-escape
|
||||
`attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`,
|
||||
],
|
||||
}),
|
||||
get: (header: any) => {
|
||||
if (header === "content-type") return contentType
|
||||
if (header === "content-disposition")
|
||||
// eslint-disable-next-line no-useless-escape
|
||||
return `attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`
|
||||
},
|
||||
},
|
||||
body: mockReadable,
|
||||
})
|
||||
)
|
||||
|
||||
const query = {
|
||||
path: "api",
|
||||
}
|
||||
|
||||
const response = await config.integration.read(query)
|
||||
|
||||
expect(response.data).toEqual({
|
||||
size: responseData.byteLength,
|
||||
name: "00000000-0000-0000-0000-000000000000.pdf",
|
||||
url: expect.stringContaining(
|
||||
"/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.pdf"
|
||||
),
|
||||
extension: "pdf",
|
||||
key: expect.stringContaining(
|
||||
"app-id/00000000-0000-0000-0000-000000000000.pdf"
|
||||
),
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
@ -6,10 +6,15 @@ import {
|
|||
TableSourceType,
|
||||
FieldSchema,
|
||||
} from "@budibase/types"
|
||||
import { context, objectStore } from "@budibase/backend-core"
|
||||
import { v4 } from "uuid"
|
||||
import { parseStringPromise as xmlParser } from "xml2js"
|
||||
import { formatBytes } from "../../utilities"
|
||||
import bl from "bl"
|
||||
import env from "../../environment"
|
||||
import { DocumentType, SEPARATOR } from "../../db/utils"
|
||||
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
|
||||
import { helpers, utils } from "@budibase/shared-core"
|
||||
import env from "../../environment"
|
||||
import { Knex } from "knex"
|
||||
|
||||
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
|
||||
|
@ -467,3 +472,74 @@ export function getPrimaryDisplay(testValue: unknown): string | undefined {
|
|||
export function isValidFilter(value: any) {
|
||||
return value != null && value !== ""
|
||||
}
|
||||
|
||||
export async function handleXml(response: any) {
|
||||
let data,
|
||||
rawXml = await response.text()
|
||||
data =
|
||||
(await xmlParser(rawXml, {
|
||||
explicitArray: false,
|
||||
trim: true,
|
||||
explicitRoot: false,
|
||||
})) || {}
|
||||
// there is only one structure, its an array, return the array so it appears as rows
|
||||
const keys = Object.keys(data)
|
||||
if (keys.length === 1 && Array.isArray(data[keys[0]])) {
|
||||
data = data[keys[0]]
|
||||
}
|
||||
return { data, rawXml }
|
||||
}
|
||||
|
||||
export async function handleFileResponse(
|
||||
response: any,
|
||||
filename: string,
|
||||
startTime: number
|
||||
) {
|
||||
let presignedUrl,
|
||||
size = 0
|
||||
const fileExtension = filename.includes(".")
|
||||
? filename.split(".").slice(1).join(".")
|
||||
: ""
|
||||
|
||||
const processedFileName = `${v4()}.${fileExtension}`
|
||||
const key = `${context.getProdAppId()}/${processedFileName}`
|
||||
const bucket = objectStore.ObjectStoreBuckets.TEMP
|
||||
|
||||
const stream = response.body.pipe(bl((error, data) => data))
|
||||
|
||||
if (response.body) {
|
||||
const contentLength = response.headers.get("content-length")
|
||||
if (contentLength) {
|
||||
size = parseInt(contentLength, 10)
|
||||
} else {
|
||||
const chunks: Buffer[] = []
|
||||
for await (const chunk of response.body) {
|
||||
chunks.push(chunk)
|
||||
size += chunk.length
|
||||
}
|
||||
}
|
||||
|
||||
await objectStore.streamUpload({
|
||||
bucket,
|
||||
filename: key,
|
||||
stream,
|
||||
ttl: 1,
|
||||
type: response.headers["content-type"],
|
||||
})
|
||||
}
|
||||
presignedUrl = await objectStore.getPresignedUrl(bucket, key)
|
||||
return {
|
||||
data: {
|
||||
size,
|
||||
name: processedFileName,
|
||||
url: presignedUrl,
|
||||
extension: fileExtension,
|
||||
key: key,
|
||||
},
|
||||
info: {
|
||||
code: response.status,
|
||||
size: formatBytes(size.toString()),
|
||||
time: `${Math.round(performance.now() - startTime)}ms`,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
@ -106,22 +106,22 @@ export async function updateClientLibrary(appId: string) {
|
|||
}
|
||||
|
||||
// Upload latest manifest and client library
|
||||
const manifestUpload = objectStore.streamUpload(
|
||||
ObjectStoreBuckets.APPS,
|
||||
join(appId, "manifest.json"),
|
||||
fs.createReadStream(manifest),
|
||||
{
|
||||
const manifestUpload = objectStore.streamUpload({
|
||||
bucket: ObjectStoreBuckets.APPS,
|
||||
filename: join(appId, "manifest.json"),
|
||||
stream: fs.createReadStream(manifest),
|
||||
extra: {
|
||||
ContentType: "application/json",
|
||||
}
|
||||
)
|
||||
const clientUpload = objectStore.streamUpload(
|
||||
ObjectStoreBuckets.APPS,
|
||||
join(appId, "budibase-client.js"),
|
||||
fs.createReadStream(client),
|
||||
{
|
||||
},
|
||||
})
|
||||
const clientUpload = objectStore.streamUpload({
|
||||
bucket: ObjectStoreBuckets.APPS,
|
||||
filename: join(appId, "budibase-client.js"),
|
||||
stream: fs.createReadStream(client),
|
||||
extra: {
|
||||
ContentType: "application/javascript",
|
||||
}
|
||||
)
|
||||
},
|
||||
})
|
||||
|
||||
const manifestSrc = fs.promises.readFile(manifest, "utf8")
|
||||
|
||||
|
|
|
@ -8,7 +8,7 @@ import {
|
|||
logging,
|
||||
env as coreEnv,
|
||||
} from "@budibase/backend-core"
|
||||
import { Ctx, User, EmailInvite } from "@budibase/types"
|
||||
import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types"
|
||||
|
||||
interface Request {
|
||||
ctx?: Ctx
|
||||
|
@ -97,6 +97,7 @@ export async function sendSmtpEmail({
|
|||
bcc,
|
||||
automation,
|
||||
invite,
|
||||
attachments,
|
||||
}: {
|
||||
to: string
|
||||
from: string
|
||||
|
@ -105,6 +106,7 @@ export async function sendSmtpEmail({
|
|||
cc: string
|
||||
bcc: string
|
||||
automation: boolean
|
||||
attachments?: EmailAttachment[]
|
||||
invite?: EmailInvite
|
||||
}) {
|
||||
// tenant ID will be set in header
|
||||
|
@ -122,6 +124,7 @@ export async function sendSmtpEmail({
|
|||
purpose: "custom",
|
||||
automation,
|
||||
invite,
|
||||
attachments,
|
||||
},
|
||||
})
|
||||
)
|
||||
|
|
|
@ -16,7 +16,7 @@ import { setJSRunner, removeJSRunner } from "./helpers/javascript"
|
|||
import manifest from "./manifest.json"
|
||||
import { ProcessOptions } from "./types"
|
||||
|
||||
export { helpersToRemoveForJs } from "./helpers/list"
|
||||
export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list"
|
||||
export { FIND_ANY_HBS_REGEX } from "./utilities"
|
||||
export { setJSRunner, setOnErrorLog } from "./helpers/javascript"
|
||||
export { iifeWrapper } from "./iife"
|
||||
|
|
|
@ -10,6 +10,7 @@ export enum AutomationIOType {
|
|||
ARRAY = "array",
|
||||
JSON = "json",
|
||||
DATE = "date",
|
||||
ATTACHMENT = "attachment",
|
||||
}
|
||||
|
||||
export enum AutomationCustomIOType {
|
||||
|
@ -30,6 +31,7 @@ export enum AutomationCustomIOType {
|
|||
WEBHOOK_URL = "webhookUrl",
|
||||
AUTOMATION = "automation",
|
||||
AUTOMATION_FIELDS = "automationFields",
|
||||
MULTI_ATTACHMENTS = "multi_attachments",
|
||||
}
|
||||
|
||||
export enum AutomationTriggerStepId {
|
||||
|
@ -80,6 +82,11 @@ export interface EmailInvite {
|
|||
url?: string
|
||||
}
|
||||
|
||||
export interface EmailAttachment {
|
||||
url: string
|
||||
filename: string
|
||||
}
|
||||
|
||||
export interface SendEmailOpts {
|
||||
// workspaceId If finer grain controls being used then this will lookup config for workspace.
|
||||
workspaceId?: string
|
||||
|
@ -97,6 +104,7 @@ export interface SendEmailOpts {
|
|||
bcc?: boolean
|
||||
automation?: boolean
|
||||
invite?: EmailInvite
|
||||
attachments?: EmailAttachment[]
|
||||
}
|
||||
|
||||
export const AutomationStepIdArray = [
|
||||
|
|
|
@ -15,6 +15,7 @@ export async function sendEmail(ctx: BBContext) {
|
|||
bcc,
|
||||
automation,
|
||||
invite,
|
||||
attachments,
|
||||
} = ctx.request.body
|
||||
let user: any
|
||||
if (userId) {
|
||||
|
@ -31,6 +32,7 @@ export async function sendEmail(ctx: BBContext) {
|
|||
bcc,
|
||||
automation,
|
||||
invite,
|
||||
attachments,
|
||||
})
|
||||
ctx.body = {
|
||||
...response,
|
||||
|
|
|
@ -1,9 +1,15 @@
|
|||
jest.unmock("node-fetch")
|
||||
jest.unmock("aws-sdk")
|
||||
import { TestConfiguration } from "../../../../tests"
|
||||
import { EmailTemplatePurpose } from "../../../../constants"
|
||||
import { objectStoreTestProviders } from "@budibase/backend-core/tests"
|
||||
import { objectStore } from "@budibase/backend-core"
|
||||
import tk from "timekeeper"
|
||||
import { EmailAttachment } from "@budibase/types"
|
||||
|
||||
const fetch = require("node-fetch")
|
||||
|
||||
const nodemailer = require("nodemailer")
|
||||
const fetch = require("node-fetch")
|
||||
|
||||
// for the real email tests give them a long time to try complete/fail
|
||||
jest.setTimeout(30000)
|
||||
|
@ -12,14 +18,20 @@ describe("/api/global/email", () => {
|
|||
const config = new TestConfiguration()
|
||||
|
||||
beforeAll(async () => {
|
||||
tk.reset()
|
||||
await objectStoreTestProviders.minio.start()
|
||||
await config.beforeAll()
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await objectStoreTestProviders.minio.stop()
|
||||
await config.afterAll()
|
||||
})
|
||||
|
||||
async function sendRealEmail(purpose: string) {
|
||||
async function sendRealEmail(
|
||||
purpose: string,
|
||||
attachments?: EmailAttachment[]
|
||||
) {
|
||||
let response, text
|
||||
try {
|
||||
const timeout = () =>
|
||||
|
@ -35,8 +47,14 @@ describe("/api/global/email", () => {
|
|||
)
|
||||
await Promise.race([config.saveEtherealSmtpConfig(), timeout()])
|
||||
await Promise.race([config.saveSettingsConfig(), timeout()])
|
||||
|
||||
const res = await config.api.emails.sendEmail(purpose).timeout(20000)
|
||||
let res
|
||||
if (attachments) {
|
||||
res = await config.api.emails
|
||||
.sendEmail(purpose, attachments)
|
||||
.timeout(20000)
|
||||
} else {
|
||||
res = await config.api.emails.sendEmail(purpose).timeout(20000)
|
||||
}
|
||||
// ethereal hiccup, can't test right now
|
||||
if (res.status >= 300) {
|
||||
return
|
||||
|
@ -80,4 +98,25 @@ describe("/api/global/email", () => {
|
|||
it("should be able to send a password recovery email", async () => {
|
||||
await sendRealEmail(EmailTemplatePurpose.PASSWORD_RECOVERY)
|
||||
})
|
||||
|
||||
it("should be able to send an email with attachments", async () => {
|
||||
let bucket = "testbucket"
|
||||
let filename = "test.txt"
|
||||
await objectStore.upload({
|
||||
bucket,
|
||||
filename,
|
||||
body: Buffer.from("test data"),
|
||||
})
|
||||
let presignedUrl = await objectStore.getPresignedUrl(
|
||||
bucket,
|
||||
filename,
|
||||
60000
|
||||
)
|
||||
|
||||
let attachmentObject = {
|
||||
url: presignedUrl,
|
||||
filename,
|
||||
}
|
||||
await sendRealEmail(EmailTemplatePurpose.WELCOME, [attachmentObject])
|
||||
})
|
||||
})
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
import { EmailAttachment } from "@budibase/types"
|
||||
import TestConfiguration from "../TestConfiguration"
|
||||
import { TestAPI } from "./base"
|
||||
|
||||
|
@ -6,11 +7,12 @@ export class EmailAPI extends TestAPI {
|
|||
super(config)
|
||||
}
|
||||
|
||||
sendEmail = (purpose: string) => {
|
||||
sendEmail = (purpose: string, attachments?: EmailAttachment[]) => {
|
||||
return this.request
|
||||
.post(`/api/global/email/send`)
|
||||
.send({
|
||||
email: "test@example.com",
|
||||
attachments,
|
||||
purpose,
|
||||
tenantId: this.config.getTenantId(),
|
||||
userId: this.config.user?._id!,
|
||||
|
|
|
@ -4,8 +4,8 @@ process.env.JWT_SECRET = "test-jwtsecret"
|
|||
process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
|
||||
process.env.MULTI_TENANCY = "1"
|
||||
process.env.MINIO_URL = "http://localhost"
|
||||
process.env.MINIO_ACCESS_KEY = "test"
|
||||
process.env.MINIO_SECRET_KEY = "test"
|
||||
process.env.MINIO_ACCESS_KEY = "budibase"
|
||||
process.env.MINIO_SECRET_KEY = "budibase"
|
||||
process.env.PLATFORM_URL = "http://localhost:10000"
|
||||
process.env.INTERNAL_API_KEY = "tet"
|
||||
process.env.DISABLE_ACCOUNT_PORTAL = "0"
|
||||
|
|
|
@ -62,8 +62,8 @@ export function smtpEthereal(): SMTPConfig {
|
|||
from: "testfrom@example.com",
|
||||
secure: false,
|
||||
auth: {
|
||||
user: "wyatt.zulauf29@ethereal.email",
|
||||
pass: "tEwDtHBWWxusVWAPfa",
|
||||
user: "mortimer.leuschke@ethereal.email",
|
||||
pass: "5hSjsPbzRv7gEUsfzx",
|
||||
},
|
||||
connectionTimeout: 1000, // must be less than the jest default of 5000
|
||||
},
|
||||
|
|
|
@ -4,8 +4,10 @@ import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
|
|||
import { getSettingsTemplateContext } from "./templates"
|
||||
import { processString } from "@budibase/string-templates"
|
||||
import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
|
||||
import { configs, cache } from "@budibase/backend-core"
|
||||
import { configs, cache, objectStore } from "@budibase/backend-core"
|
||||
import ical from "ical-generator"
|
||||
import fetch from "node-fetch"
|
||||
import path from "path"
|
||||
|
||||
const nodemailer = require("nodemailer")
|
||||
|
||||
|
@ -162,6 +164,42 @@ export async function sendEmail(
|
|||
contents: opts?.contents,
|
||||
}),
|
||||
}
|
||||
if (opts?.attachments) {
|
||||
const attachments = await Promise.all(
|
||||
opts.attachments?.map(async attachment => {
|
||||
const isFullyFormedUrl =
|
||||
attachment.url.startsWith("http://") ||
|
||||
attachment.url.startsWith("https://")
|
||||
if (isFullyFormedUrl) {
|
||||
const response = await fetch(attachment.url)
|
||||
if (!response.ok) {
|
||||
throw new Error(`unexpected response ${response.statusText}`)
|
||||
}
|
||||
const fallbackFilename = path.basename(
|
||||
new URL(attachment.url).pathname
|
||||
)
|
||||
return {
|
||||
filename: attachment.filename || fallbackFilename,
|
||||
content: response?.body,
|
||||
}
|
||||
} else {
|
||||
const url = attachment.url
|
||||
const result = objectStore.extractBucketAndPath(url)
|
||||
if (result === null) {
|
||||
throw new Error("Invalid signed URL")
|
||||
}
|
||||
const { bucket, path } = result
|
||||
const readStream = await objectStore.getReadStream(bucket, path)
|
||||
const fallbackFilename = path.split("/").pop() || ""
|
||||
return {
|
||||
filename: attachment.filename || fallbackFilename,
|
||||
content: readStream,
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
message = { ...message, attachments }
|
||||
}
|
||||
|
||||
message = {
|
||||
...message,
|
||||
|
|
19
yarn.lock
19
yarn.lock
|
@ -6147,6 +6147,13 @@
|
|||
dependencies:
|
||||
"@types/webidl-conversions" "*"
|
||||
|
||||
"@types/xml2js@^0.4.14":
|
||||
version "0.4.14"
|
||||
resolved "https://registry.yarnpkg.com/@types/xml2js/-/xml2js-0.4.14.tgz#5d462a2a7330345e2309c6b549a183a376de8f9a"
|
||||
integrity sha512-4YnrRemBShWRO2QjvUin8ESA41rH+9nQGLUGZV/1IDhi3SL9OhdpNC/MrulTWuptXKwhx/aDxE7toV0f/ypIXQ==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/yargs-parser@*":
|
||||
version "21.0.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b"
|
||||
|
@ -7607,6 +7614,16 @@ bl@^4.0.3, bl@^4.1.0:
|
|||
inherits "^2.0.4"
|
||||
readable-stream "^3.4.0"
|
||||
|
||||
bl@^6.0.12:
|
||||
version "6.0.12"
|
||||
resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8"
|
||||
integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w==
|
||||
dependencies:
|
||||
"@types/readable-stream" "^4.0.0"
|
||||
buffer "^6.0.3"
|
||||
inherits "^2.0.4"
|
||||
readable-stream "^4.2.0"
|
||||
|
||||
bl@^6.0.3:
|
||||
version "6.0.9"
|
||||
resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.9.tgz#df8fcb2ef7be2e5ee8f65afa493502914e0d816f"
|
||||
|
@ -8781,7 +8798,7 @@ consolidate@^0.16.0:
|
|||
dependencies:
|
||||
bluebird "^3.7.2"
|
||||
|
||||
content-disposition@^0.5.2, content-disposition@^0.5.3, content-disposition@~0.5.2:
|
||||
content-disposition@^0.5.2, content-disposition@^0.5.3, content-disposition@^0.5.4, content-disposition@~0.5.2:
|
||||
version "0.5.4"
|
||||
resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe"
|
||||
integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==
|
||||
|
|
Loading…
Reference in a new issue