diff --git a/lerna.json b/lerna.json index 9c5a6c6bab..16dc73aa30 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.24.2", + "version": "2.25.0", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 6cea7efeba..4beb02c9c7 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -281,7 +281,7 @@ export function doInScimContext(task: any) { return newContext(updates, task) } -export async function ensureSnippetContext() { +export async function ensureSnippetContext(enabled = !env.isTest()) { const ctx = getCurrentContext() // If we've already added snippets to context, continue @@ -292,7 +292,7 @@ export async function ensureSnippetContext() { // Otherwise get snippets for this app and update context let snippets: Snippet[] | undefined const db = getAppDB() - if (db && !env.isTest()) { + if (db && enabled) { const app = await db.get(DocumentType.APP_METADATA) snippets = app.snippets } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index d220d0a8ac..ef351f7d4d 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -3,11 +3,11 @@ import { AllDocsResponse, AnyDocument, Database, - DatabaseOpts, - DatabaseQueryOpts, - DatabasePutOpts, DatabaseCreateIndexOpts, DatabaseDeleteIndexOpts, + DatabaseOpts, + DatabasePutOpts, + DatabaseQueryOpts, Document, isDocument, RowResponse, @@ -17,7 +17,7 @@ import { import { getCouchInfo } from "./connections" import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" -import { WriteStream, ReadStream } from "fs" +import { ReadStream, WriteStream } from "fs" import { newid } from "../../docIds/newid" import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" @@ -38,6 +38,39 @@ function buildNano(couchInfo: { url: string; cookie: string }) { type DBCall = () => Promise +class CouchDBError extends Error { + status: number + statusCode: number + reason: string + name: string + errid: string + error: string + description: string + + constructor( + message: string, + info: { + status: number | undefined + statusCode: number | undefined + name: string + errid: string + description: string + reason: string + error: string + } + ) { + super(message) + const statusCode = info.status || info.statusCode || 500 + this.status = statusCode + this.statusCode = statusCode + this.reason = info.reason + this.name = info.name + this.errid = info.errid + this.description = info.description + this.error = info.error + } +} + export function DatabaseWithConnection( dbName: string, connection: string, @@ -119,7 +152,7 @@ export class DatabaseImpl implements Database { } catch (err: any) { // Handling race conditions if (err.statusCode !== 412) { - throw err + throw new CouchDBError(err.message, err) } } } @@ -138,10 +171,9 @@ export class DatabaseImpl implements Database { if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) { await this.checkAndCreateDb() return await this.performCall(call) - } else if (err.statusCode) { - err.status = err.statusCode } - throw err + // stripping the error down the props which are safe/useful, drop everything else + throw new CouchDBError(`CouchDB error: ${err.message}`, err) } } @@ -288,7 +320,7 @@ export class DatabaseImpl implements Database { if (err.statusCode === 404) { return } else { - throw { ...err, status: err.statusCode } + throw new CouchDBError(err.message, err) } } } diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 2bef91ffef..0ac2c35179 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -13,13 +13,14 @@ import { bucketTTLConfig, budibaseTempDir } from "./utils" import { v4 } from "uuid" import { APP_PREFIX, APP_DEV_PREFIX } from "../db" import fsp from "fs/promises" +import { HeadObjectOutput } from "aws-sdk/clients/s3" const streamPipeline = promisify(stream.pipeline) // use this as a temporary store of buckets that are being created const STATE = { bucketCreationPromises: {}, } -const signedFilePrefix = "/files/signed" +export const SIGNED_FILE_PREFIX = "/files/signed" type ListParams = { ContinuationToken?: string @@ -40,8 +41,13 @@ type UploadParams = BaseUploadParams & { path?: string | PathLike } -type StreamUploadParams = BaseUploadParams & { - stream: ReadStream +export type StreamTypes = + | ReadStream + | NodeJS.ReadableStream + | ReadableStream + +export type StreamUploadParams = BaseUploadParams & { + stream?: StreamTypes } const CONTENT_TYPE_MAP: any = { @@ -174,11 +180,9 @@ export async function upload({ const objectStore = ObjectStore(bucketName) const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) - if (ttl && (bucketCreated.created || bucketCreated.exists)) { + if (ttl && bucketCreated.created) { let ttlConfig = bucketTTLConfig(bucketName, ttl) - if (objectStore.putBucketLifecycleConfiguration) { - await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() - } + await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() } let contentType = type @@ -222,11 +226,9 @@ export async function streamUpload({ const objectStore = ObjectStore(bucketName) const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) - if (ttl && (bucketCreated.created || bucketCreated.exists)) { + if (ttl && bucketCreated.created) { let ttlConfig = bucketTTLConfig(bucketName, ttl) - if (objectStore.putBucketLifecycleConfiguration) { - await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() - } + await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() } // Set content type for certain known extensions @@ -333,7 +335,7 @@ export function getPresignedUrl( const signedUrl = new URL(url) const path = signedUrl.pathname const query = signedUrl.search - return `${signedFilePrefix}${path}${query}` + return `${SIGNED_FILE_PREFIX}${path}${query}` } } @@ -521,6 +523,26 @@ export async function getReadStream( return client.getObject(params).createReadStream() } +export async function getObjectMetadata( + bucket: string, + path: string +): Promise { + bucket = sanitizeBucket(bucket) + path = sanitizeKey(path) + + const client = ObjectStore(bucket) + const params = { + Bucket: bucket, + Key: path, + } + + try { + return await client.headObject(params).promise() + } catch (err: any) { + throw new Error("Unable to retrieve metadata from object") + } +} + /* Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract the bucket and the path from it @@ -530,7 +552,9 @@ export function extractBucketAndPath( ): { bucket: string; path: string } | null { const baseUrl = url.split("?")[0] - const regex = new RegExp(`^${signedFilePrefix}/(?[^/]+)/(?.+)$`) + const regex = new RegExp( + `^${SIGNED_FILE_PREFIX}/(?[^/]+)/(?.+)$` + ) const match = baseUrl.match(regex) if (match && match.groups) { diff --git a/packages/backend-core/src/objectStore/utils.ts b/packages/backend-core/src/objectStore/utils.ts index 08b5238ff6..5b9c2e3646 100644 --- a/packages/backend-core/src/objectStore/utils.ts +++ b/packages/backend-core/src/objectStore/utils.ts @@ -1,9 +1,14 @@ -import { join } from "path" +import path, { join } from "path" import { tmpdir } from "os" import fs from "fs" import env from "../environment" import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3" - +import * as objectStore from "./objectStore" +import { + AutomationAttachment, + AutomationAttachmentContent, + BucketedContent, +} from "@budibase/types" /**************************************************** * NOTE: When adding a new bucket - name * * sure that S3 usages (like budibase-infra) * @@ -55,3 +60,50 @@ export const bucketTTLConfig = ( return params } + +async function processUrlAttachment( + attachment: AutomationAttachment +): Promise { + const response = await fetch(attachment.url) + if (!response.ok || !response.body) { + throw new Error(`Unexpected response ${response.statusText}`) + } + const fallbackFilename = path.basename(new URL(attachment.url).pathname) + return { + filename: attachment.filename || fallbackFilename, + content: response.body, + } +} + +export async function processObjectStoreAttachment( + attachment: AutomationAttachment +): Promise { + const result = objectStore.extractBucketAndPath(attachment.url) + + if (result === null) { + throw new Error("Invalid signed URL") + } + + const { bucket, path: objectPath } = result + const readStream = await objectStore.getReadStream(bucket, objectPath) + const fallbackFilename = path.basename(objectPath) + return { + bucket, + path: objectPath, + filename: attachment.filename || fallbackFilename, + content: readStream, + } +} + +export async function processAutomationAttachment( + attachment: AutomationAttachment +): Promise { + const isFullyFormedUrl = + attachment.url?.startsWith("http://") || + attachment.url?.startsWith("https://") + if (isFullyFormedUrl) { + return await processUrlAttachment(attachment) + } else { + return await processObjectStoreAttachment(attachment) + } +} diff --git a/packages/bbui/src/FancyForm/FancyInput.svelte b/packages/bbui/src/FancyForm/FancyInput.svelte index 0c58b9b045..f665fa5724 100644 --- a/packages/bbui/src/FancyForm/FancyInput.svelte +++ b/packages/bbui/src/FancyForm/FancyInput.svelte @@ -11,6 +11,7 @@ export let error = null export let validate = null export let suffix = null + export let validateOn = "change" const dispatch = createEventDispatcher() @@ -24,7 +25,16 @@ const newValue = e.target.value dispatch("change", newValue) value = newValue - if (validate) { + if (validate && (error || validateOn === "change")) { + error = validate(newValue) + } + } + + const onBlur = e => { + focused = false + const newValue = e.target.value + dispatch("blur", newValue) + if (validate && validateOn === "blur") { error = validate(newValue) } } @@ -61,7 +71,7 @@ type={type || "text"} on:input={onChange} on:focus={() => (focused = true)} - on:blur={() => (focused = false)} + on:blur={onBlur} class:placeholder bind:this={ref} /> diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 2d2022299c..0cf0f6c740 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -358,7 +358,8 @@ value.customType !== "cron" && value.customType !== "triggerSchema" && value.customType !== "automationFields" && - value.type !== "attachment" + value.type !== "attachment" && + value.type !== "attachment_single" ) } diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index 0d15df6c87..ab020aad08 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -2,6 +2,8 @@ import { tables } from "stores/builder" import { Select, Checkbox, Label } from "@budibase/bbui" import { createEventDispatcher } from "svelte" + import { FieldType } from "@budibase/types" + import RowSelectorTypes from "./RowSelectorTypes.svelte" import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte" import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte" @@ -14,7 +16,6 @@ export let bindings export let isTestModal export let isUpdateRow - $: parsedBindings = bindings.map(binding => { let clone = Object.assign({}, binding) clone.icon = "ShareAndroid" @@ -26,15 +27,19 @@ $: { table = $tables.list.find(table => table._id === value?.tableId) - schemaFields = Object.entries(table?.schema ?? {}) - // surface the schema so the user can see it in the json - schemaFields.map(([, schema]) => { + + // Just sorting attachment types to the bottom here for a cleaner UX + schemaFields = Object.entries(table?.schema ?? {}).sort( + ([, schemaA], [, schemaB]) => + (schemaA.type === "attachment") - (schemaB.type === "attachment") + ) + + schemaFields.forEach(([, schema]) => { if (!schema.autocolumn && !value[schema.name]) { value[schema.name] = "" } }) } - const onChangeTable = e => { value["tableId"] = e.detail dispatch("change", value) @@ -114,10 +119,16 @@ {#if schemaFields.length} {#each schemaFields as [field, schema]} - {#if !schema.autocolumn && schema.type !== "attachment"} -
+ {#if !schema.autocolumn} +
-
+
{#if isTestModal} import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui" + import { FieldType } from "@budibase/types" import LinkedRowSelector from "components/common/LinkedRowSelector.svelte" import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte" import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte" import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte" import Editor from "components/integration/QueryEditor.svelte" + import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" export let onChange export let field @@ -22,6 +24,27 @@ function schemaHasOptions(schema) { return !!schema.constraints?.inclusion?.length } + + const handleAttachmentParams = keyValuObj => { + let params = {} + + if ( + schema.type === FieldType.ATTACHMENT_SINGLE && + Object.keys(keyValuObj).length === 0 + ) { + return [] + } + if (!Array.isArray(keyValuObj)) { + keyValuObj = [keyValuObj] + } + + if (keyValuObj.length) { + for (let param of keyValuObj) { + params[param.url] = param.filename + } + } + return params + } {#if schemaHasOptions(schema) && schema.type !== "array"} @@ -77,6 +100,35 @@ on:change={e => onChange(e, field)} useLabel={false} /> +{:else if schema.type === FieldType.ATTACHMENTS || schema.type === FieldType.ATTACHMENT_SINGLE} +
+ + onChange( + { + detail: + schema.type === FieldType.ATTACHMENT_SINGLE + ? e.detail.length > 0 + ? { url: e.detail[0].name, filename: e.detail[0].value } + : {} + : e.detail.map(({ name, value }) => ({ + url: name, + filename: value, + })), + }, + field + )} + object={handleAttachmentParams(value[field])} + allowJS + {bindings} + keyBindings + customButtonText={"Add attachment"} + keyPlaceholder={"URL"} + valuePlaceholder={"Filename"} + actionButtonDisabled={schema.type === FieldType.ATTACHMENT_SINGLE && + Object.keys(value[field]).length >= 1} + /> +
{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)} {/if} + + diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index e3937ab772..decf77069f 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -1,7 +1,9 @@ - + {text} - - dispatch("change", tempValue)} - > -
- (tempValue = e.detail)} - /> -
- - - + + + + (tempValue = e.detail)} + {bindings} + /> + + diff --git a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte index 8ce9dda209..fb448cca8d 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte @@ -4,6 +4,7 @@ readableToRuntimeBinding, runtimeToReadableBinding, } from "dataBinding" + import { FieldType } from "@budibase/types" import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" import { createEventDispatcher, setContext } from "svelte" @@ -102,6 +103,8 @@ longform: value => !isJSBinding(value), json: value => !isJSBinding(value), boolean: isValidBoolean, + attachment: false, + attachment_single: false, } const isValid = value => { @@ -116,7 +119,16 @@ if (type === "json" && !isJSBinding(value)) { return "json-slot-icon" } - if (!["string", "number", "bigint", "barcodeqr"].includes(type)) { + if ( + ![ + "string", + "number", + "bigint", + "barcodeqr", + "attachment", + "attachment_single", + ].includes(type) + ) { return "slot-icon" } return "" @@ -157,7 +169,7 @@ {updateOnChange} /> {/if} - {#if !disabled && type !== "formula"} + {#if !disabled && type !== "formula" && !disabled && type !== FieldType.ATTACHMENTS && !disabled && type !== FieldType.ATTACHMENT_SINGLE}
{ diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte index 5ed18a970a..6f69e71ccb 100644 --- a/packages/builder/src/components/integration/KeyValueBuilder.svelte +++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte @@ -37,6 +37,7 @@ export let customButtonText = null export let keyBindings = false export let allowJS = false + export let actionButtonDisabled = false export let compare = (option, value) => option === value let fields = Object.entries(object || {}).map(([name, value]) => ({ @@ -189,7 +190,14 @@ {/if} {#if !readOnly && !noAddButton}
- + {#if customButtonText} {customButtonText} {:else} diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 1c569cb459..2e61238cc8 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -289,6 +289,7 @@ OperatorOptions.ContainsAny.value, ].includes(filter.operator)} disabled={filter.noValue} + type={filter.valueType} /> {:else} @@ -325,8 +326,6 @@