diff --git a/.github/workflows/force-release.yml b/.github/workflows/force-release.yml index 8a9d444f51..3d96d51484 100644 --- a/.github/workflows/force-release.yml +++ b/.github/workflows/force-release.yml @@ -9,7 +9,7 @@ on: jobs: ensure-is-master-tag: name: Ensure is a master tag - runs-on: qa-arc-runner-set + runs-on: ubuntu-latest steps: - name: Checkout monorepo uses: actions/checkout@v4 diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml new file mode 100644 index 0000000000..e2fa9f2515 --- /dev/null +++ b/.github/workflows/pr-labeler.yml @@ -0,0 +1,38 @@ +name: PR labeler + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +on: + pull_request: + types: [opened, synchronize] + +jobs: + size-labeler: + runs-on: ubuntu-latest + steps: + - uses: codelytv/pr-size-labeler@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + xs_max_size: "10" + s_max_size: "100" + m_max_size: "500" + l_max_size: "1000" + fail_if_xl: "false" + files_to_ignore: "yarn.lock" + + team-labeler: + runs-on: ubuntu-latest + if: ${{ github.event.action == 'opened' }} + steps: + - uses: rodrigoarias/auto-label-per-user@v1.0.0 + with: + git-token: ${{ secrets.GITHUB_TOKEN }} + user-team-map: | + { + "adrinr": "firestorm", + "samwho": "firestorm", + "PClmnt": "firestorm", + "mike12345567": "firestorm" + } diff --git a/charts/budibase/Chart.lock b/charts/budibase/Chart.lock index 3ee752a362..8ff1d36941 100644 --- a/charts/budibase/Chart.lock +++ b/charts/budibase/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: couchdb repository: https://apache.github.io/couchdb-helm - version: 4.3.0 -digest: sha256:94449a7f195b186f5af33ec5aa66d58b36bede240fae710f021ca87837b30606 -generated: "2023-11-20T17:43:02.777596Z" + version: 4.5.6 +digest: sha256:405f098633e632d6f4e140175f156ed4f02918b0d89193f1b66c9cbea211d6c9 +generated: "2024-06-05T14:41:05.979052+01:00" diff --git a/charts/budibase/Chart.yaml b/charts/budibase/Chart.yaml index e2c9378f2c..a16eba8982 100644 --- a/charts/budibase/Chart.yaml +++ b/charts/budibase/Chart.yaml @@ -17,6 +17,6 @@ version: 0.0.0 appVersion: 0.0.0 dependencies: - name: couchdb - version: 4.3.0 + version: 4.5.6 repository: https://apache.github.io/couchdb-helm condition: services.couchdb.enabled diff --git a/charts/budibase/README.md b/charts/budibase/README.md index 207992087d..c944ed1397 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -112,7 +112,9 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | awsAlbIngress.enabled | bool | `false` | Whether to create an ALB Ingress resource pointing to the Budibase proxy. Requires the AWS ALB Ingress Controller. | | couchdb.clusterSize | int | `1` | The number of replicas to run in the CouchDB cluster. We set this to 1 by default to make things simpler, but you can set it to 3 if you need a high-availability CouchDB cluster. | | couchdb.couchdbConfig.couchdb.uuid | string | `"budibase-couchdb"` | Unique identifier for this CouchDB server instance. You shouldn't need to change this. | +| couchdb.extraPorts[0] | object | `{"containerPort":4984,"name":"sqs"}` | Extra ports to expose on the CouchDB service. We expose the SQS port by default, but you can add more ports here if you need to. | | couchdb.image | object | `{}` | We use a custom CouchDB image for running Budibase and we don't support using any other CouchDB image. You shouldn't change this, and if you do we can't guarantee that Budibase will work. | +| couchdb.service.extraPorts[0] | object | `{"name":"sqs","port":4984,"protocol":"TCP","targetPort":4984}` | Extra ports to expose on the CouchDB service. We expose the SQS port by default, but you can add more ports here if you need to. | | globals.apiEncryptionKey | string | `""` | Used for encrypting API keys and environment variables when stored in the database. You don't need to set this if `createSecrets` is true. | | globals.appVersion | string | `""` | The version of Budibase to deploy. Defaults to what's specified by {{ .Chart.AppVersion }}. Ends up being used as the image version tag for the apps, proxy, and worker images. | | globals.automationMaxIterations | string | `"200"` | The maximum number of iterations allows for an automation loop step. You can read more about looping here: . | @@ -135,6 +137,8 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | globals.smtp.password | string | `""` | The password to use when authenticating with your SMTP server. | | globals.smtp.port | string | `"587"` | The port of your SMTP server. | | globals.smtp.user | string | `""` | The username to use when authenticating with your SMTP server. | +| globals.sqs.enabled | bool | `false` | Whether to use the CouchDB "structured query service" or not. This is disabled by default for now, but will become the default in a future release. | +| globals.tempBucketName | string | `""` | | | globals.tenantFeatureFlags | string | `"*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"` | Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be changed. | | imagePullSecrets | list | `[]` | Passed to all pods created by this chart. Should not ordinarily need to be changed. | | ingress.className | string | `""` | What ingress class to use. | @@ -152,6 +156,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.apps.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the apps service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the apps pods. | | services.apps.extraContainers | list | `[]` | Additional containers to be added to the apps pod. | | services.apps.extraEnv | list | `[]` | Extra environment variables to set for apps pods. Takes a list of name=value pairs. | +| services.apps.extraEnvFromSecret | list | `[]` | Name of the K8s Secret in the same namespace which contains the extra environment variables. This can be used to avoid storing sensitive information in the values.yaml file. | | services.apps.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main apps container. | | services.apps.extraVolumes | list | `[]` | Additional volumes to the apps pod. | | services.apps.httpLogging | int | `1` | Whether or not to log HTTP requests to the apps service. | @@ -168,6 +173,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. | | services.automationWorkers.extraContainers | list | `[]` | Additional containers to be added to the automationWorkers pod. | | services.automationWorkers.extraEnv | list | `[]` | Extra environment variables to set for automation worker pods. Takes a list of name=value pairs. | +| services.automationWorkers.extraEnvFromSecret | list | `[]` | Name of the K8s Secret in the same namespace which contains the extra environment variables. This can be used to avoid storing sensitive information in the values.yaml file. | | services.automationWorkers.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main automationWorkers container. | | services.automationWorkers.extraVolumes | list | `[]` | Additional volumes to the automationWorkers pod. | | services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | @@ -195,7 +201,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.objectStore.region | string | `""` | AWS_REGION if using S3 | | services.objectStore.resources | object | `{}` | The resources to use for Minio pods. See for more information on how to set these. | | services.objectStore.secretKey | string | `""` | AWS_SECRET_ACCESS_KEY if using S3 | -| services.objectStore.storage | string | `"100Mi"` | How much storage to give Minio in its PersistentVolumeClaim. | +| services.objectStore.storage | string | `"2Gi"` | How much storage to give Minio in its PersistentVolumeClaim. | | services.objectStore.storageClass | string | `""` | If defined, storageClassName: If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. | | services.objectStore.url | string | `"http://minio-service:9000"` | URL to use for object storage. Only change this if you're using an external object store, such as S3. Remember to set `minio: false` if you do this. | | services.proxy.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the proxy service. | @@ -227,6 +233,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.worker.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the worker pods. | | services.worker.extraContainers | list | `[]` | Additional containers to be added to the worker pod. | | services.worker.extraEnv | list | `[]` | Extra environment variables to set for worker pods. Takes a list of name=value pairs. | +| services.worker.extraEnvFromSecret | list | `[]` | Name of the K8s Secret in the same namespace which contains the extra environment variables. This can be used to avoid storing sensitive information in the values.yaml file. | | services.worker.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main worker container. | | services.worker.extraVolumes | list | `[]` | Additional volumes to the worker pod. | | services.worker.httpLogging | int | `1` | Whether or not to log HTTP requests to the worker service. | diff --git a/charts/budibase/charts/couchdb-4.3.0.tgz b/charts/budibase/charts/couchdb-4.3.0.tgz deleted file mode 100644 index d3cce28ee6..0000000000 Binary files a/charts/budibase/charts/couchdb-4.3.0.tgz and /dev/null differ diff --git a/charts/budibase/charts/couchdb-4.5.6.tgz b/charts/budibase/charts/couchdb-4.5.6.tgz new file mode 100644 index 0000000000..a9211ed279 Binary files /dev/null and b/charts/budibase/charts/couchdb-4.5.6.tgz differ diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index ed7166ec5d..6b4fbe5dda 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -42,6 +42,14 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + {{ if .Values.globals.sqs.enabled }} + - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url }} + {{ else }} + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: @@ -198,10 +206,21 @@ spec: - name: APP_FEATURES value: "api" {{- end }} + {{- if .Values.globals.sqs.enabled }} + - name: SQS_SEARCH_ENABLE + value: "true" + {{- end }} {{- range .Values.services.apps.extraEnv }} - name: {{ .name }} value: {{ .value | quote }} {{- end }} + {{- range .Values.services.apps.extraEnvFromSecret}} + - name: {{ .name }} + valueFrom: + secretKeyRef: + name: {{ .secretName }} + key: {{ .secretKey | quote }} + {{- end}} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.apps.startupProbe }} diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 3c6f94ae9e..38a384626e 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -201,6 +201,13 @@ spec: - name: {{ .name }} value: {{ .value | quote }} {{- end }} + {{- range .Values.services.automationWorkers.extraEnvFromSecret}} + - name: {{ .name }} + valueFrom: + secretKeyRef: + name: {{ .secretName }} + key: {{ .secretKey | quote }} + {{- end}} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always @@ -272,4 +279,4 @@ spec: {{- toYaml .Values.services.automationWorkers.extraVolumes | nindent 8 }} {{ end }} status: {} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 66a9bb6c14..ecab8dc645 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -56,6 +56,14 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + {{ if .Values.globals.sqs.enabled }} + - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url }} + {{ else }} + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} + {{ end }} - name: API_ENCRYPTION_KEY value: {{ .Values.globals.apiEncryptionKey | quote }} - name: HTTP_LOGGING @@ -184,10 +192,21 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} + {{- if .Values.globals.sqs.enabled }} + - name: SQS_SEARCH_ENABLE + value: "true" + {{- end }} {{- range .Values.services.worker.extraEnv }} - name: {{ .name }} value: {{ .value | quote }} {{- end }} + {{- range .Values.services.worker.extraEnvFromSecret}} + - name: {{ .name }} + valueFrom: + secretKeyRef: + name: {{ .secretName }} + key: {{ .secretKey | quote }} + {{- end}} image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.worker.startupProbe }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 27037cdaa8..4e80be7322 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -138,6 +138,15 @@ globals: # -- The password to use when authenticating with your SMTP server. password: "" + sqs: + # -- Whether to use the CouchDB "structured query service" or not. This is disabled by + # default for now, but will become the default in a future release. + enabled: false + # @ignore + url: "" + # @ignore + port: "4984" + services: # -- The DNS suffix to use for service discovery. You only need to change this # if you've configured your cluster to use a different DNS suffix. @@ -240,6 +249,13 @@ services: # -- Extra environment variables to set for apps pods. Takes a list of # name=value pairs. extraEnv: [] + # -- Name of the K8s Secret in the same namespace which contains the extra environment variables. + # This can be used to avoid storing sensitive information in the values.yaml file. + extraEnvFromSecret: [] + # - name: MY_SECRET_KEY + # secretName : my-secret + # secretKey: my-secret-key + # -- Startup probe configuration for apps pods. You shouldn't need to # change this, but if you want to you can find more information here: # @@ -323,6 +339,13 @@ services: # -- Extra environment variables to set for automation worker pods. Takes a list of # name=value pairs. extraEnv: [] + # -- Name of the K8s Secret in the same namespace which contains the extra environment variables. + # This can be used to avoid storing sensitive information in the values.yaml file. + extraEnvFromSecret: [] + # - name: MY_SECRET_KEY + # secretName : my-secret + # secretKey: my-secret-key + # -- Startup probe configuration for automation worker pods. You shouldn't # need to change this, but if you want to you can find more information # here: @@ -408,6 +431,13 @@ services: # -- Extra environment variables to set for worker pods. Takes a list of # name=value pairs. extraEnv: [] + # -- Name of the K8s Secret in the same namespace which contains the extra environment variables. + # This can be used to avoid storing sensitive information in the values.yaml file. + extraEnvFromSecret: [] + # - name: MY_SECRET_KEY + # secretName : my-secret + # secretKey: my-secret-key + # -- Startup probe configuration for worker pods. You shouldn't need to # change this, but if you want to you can find more information here: # @@ -611,10 +641,25 @@ couchdb: # @ignore repository: budibase/couchdb # @ignore - tag: v3.2.1 + tag: v3.3.3 # @ignore pullPolicy: Always + extraPorts: + # -- Extra ports to expose on the CouchDB service. We expose the SQS port + # by default, but you can add more ports here if you need to. + - name: sqs + containerPort: 4984 + + service: + extraPorts: + # -- Extra ports to expose on the CouchDB service. We expose the SQS port + # by default, but you can add more ports here if you need to. + - name: sqs + port: 4984 + targetPort: 4984 + protocol: TCP + # @ignore # This should remain false. We ship Clouseau ourselves as part of the # budibase/couchdb image, and it's not possible to disable it because it's a diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 12b8df049f..59722dac5c 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -74,6 +74,7 @@ http { add_header X-Content-Type-Options nosniff always; add_header X-XSS-Protection "1; mode=block" always; add_header Content-Security-Policy "${csp_default}; ${csp_script}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; # upstreams set $apps ${APPS_UPSTREAM_URL}; diff --git a/lerna.json b/lerna.json index 335df975af..c551cf6bd9 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.27.5", + "version": "2.28.4", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/package.json b/package.json index 08176fae90..5377dfc5a1 100644 --- a/package.json +++ b/package.json @@ -37,8 +37,8 @@ "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", "build:apps": "yarn build --scope @budibase/server --scope @budibase/worker", "build:cli": "yarn build --scope @budibase/cli", - "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", - "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal --scope @budibase/account-portal-server --scope @budibase/account-portal-ui", + "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", + "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run --concurrency 2 check:types", "build:sdk": "lerna run --stream build:sdk", diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 8194d1aabf..4db63ad695 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -8,6 +8,7 @@ import { DatabaseOpts, DatabasePutOpts, DatabaseQueryOpts, + DBError, Document, isDocument, RowResponse, @@ -41,7 +42,7 @@ function buildNano(couchInfo: { url: string; cookie: string }) { type DBCall = () => Promise -class CouchDBError extends Error { +class CouchDBError extends Error implements DBError { status: number statusCode: number reason: string @@ -328,7 +329,14 @@ export class DatabaseImpl implements Database { async sqlDiskCleanup(): Promise { const dbName = this.name const url = `/${dbName}/_cleanup` - return await this._sqlQuery(url, "POST") + try { + await this._sqlQuery(url, "POST") + } catch (err: any) { + // hack for now - SQS throws a 500 when there is nothing to clean-up + if (err.status !== 500) { + throw err + } + } } // removes a document from sqlite @@ -352,18 +360,15 @@ export class DatabaseImpl implements Database { } async destroy() { + if (env.SQS_SEARCH_ENABLE && (await this.exists(SQLITE_DESIGN_DOC_ID))) { + // delete the design document, then run the cleanup operation + const definition = await this.get(SQLITE_DESIGN_DOC_ID) + // remove all tables - save the definition then trigger a cleanup + definition.sql.tables = {} + await this.put(definition) + await this.sqlDiskCleanup() + } try { - if (env.SQS_SEARCH_ENABLE) { - // delete the design document, then run the cleanup operation - try { - const definition = await this.get( - SQLITE_DESIGN_DOC_ID - ) - await this.remove(SQLITE_DESIGN_DOC_ID, definition._rev) - } finally { - await this.sqlDiskCleanup() - } - } return await this.nano().db.destroy(this.name) } catch (err: any) { // didn't exist, don't worry diff --git a/packages/backend-core/src/middleware/joi-validator.ts b/packages/backend-core/src/middleware/joi-validator.ts index ac8064a512..5047cdbbc1 100644 --- a/packages/backend-core/src/middleware/joi-validator.ts +++ b/packages/backend-core/src/middleware/joi-validator.ts @@ -3,7 +3,8 @@ import { Ctx } from "@budibase/types" function validate( schema: Joi.ObjectSchema | Joi.ArraySchema, - property: string + property: string, + opts: { errorPrefix: string } = { errorPrefix: `Invalid ${property}` } ) { // Return a Koa middleware function return (ctx: Ctx, next: any) => { @@ -29,16 +30,26 @@ function validate( const { error } = schema.validate(params) if (error) { - ctx.throw(400, `Invalid ${property} - ${error.message}`) + let message = error.message + if (opts.errorPrefix) { + message = `Invalid ${property} - ${message}` + } + ctx.throw(400, message) } return next() } } -export function body(schema: Joi.ObjectSchema | Joi.ArraySchema) { - return validate(schema, "body") +export function body( + schema: Joi.ObjectSchema | Joi.ArraySchema, + opts?: { errorPrefix: string } +) { + return validate(schema, "body", opts) } -export function params(schema: Joi.ObjectSchema | Joi.ArraySchema) { - return validate(schema, "params") +export function params( + schema: Joi.ObjectSchema | Joi.ArraySchema, + opts?: { errorPrefix: string } +) { + return validate(schema, "params", opts) } diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 0ac2c35179..de94e3968b 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -14,6 +14,7 @@ import { v4 } from "uuid" import { APP_PREFIX, APP_DEV_PREFIX } from "../db" import fsp from "fs/promises" import { HeadObjectOutput } from "aws-sdk/clients/s3" +import { ReadableStream } from "stream/web" const streamPipeline = promisify(stream.pipeline) // use this as a temporary store of buckets that are being created @@ -41,10 +42,7 @@ type UploadParams = BaseUploadParams & { path?: string | PathLike } -export type StreamTypes = - | ReadStream - | NodeJS.ReadableStream - | ReadableStream +export type StreamTypes = ReadStream | NodeJS.ReadableStream export type StreamUploadParams = BaseUploadParams & { stream?: StreamTypes @@ -222,6 +220,9 @@ export async function streamUpload({ extra, ttl, }: StreamUploadParams) { + if (!stream) { + throw new Error("Stream to upload is invalid/undefined") + } const extension = filename.split(".").pop() const objectStore = ObjectStore(bucketName) const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) @@ -251,14 +252,27 @@ export async function streamUpload({ : CONTENT_TYPE_MAP.txt } + const bucket = sanitizeBucket(bucketName), + objKey = sanitizeKey(filename) const params = { - Bucket: sanitizeBucket(bucketName), - Key: sanitizeKey(filename), + Bucket: bucket, + Key: objKey, Body: stream, ContentType: contentType, ...extra, } - return objectStore.upload(params).promise() + + const details = await objectStore.upload(params).promise() + const headDetails = await objectStore + .headObject({ + Bucket: bucket, + Key: objKey, + }) + .promise() + return { + ...details, + ContentLength: headDetails.ContentLength, + } } /** diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 1838eed92f..f633d0885e 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -21,6 +21,7 @@ let cleanupInterval: NodeJS.Timeout async function cleanup() { for (let queue of QUEUES) { await queue.clean(CLEANUP_PERIOD_MS, "completed") + await queue.clean(CLEANUP_PERIOD_MS, "failed") } } diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 85ae1924d0..5d2e4ee28d 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -15,6 +15,7 @@ Checkbox, DatePicker, DrawerContent, + Toggle, } from "@budibase/bbui" import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte" import { automationStore, selectedAutomation, tables } from "stores/builder" @@ -118,7 +119,6 @@ searchableSchema: true, }).schema } - try { if (isTestModal) { let newTestData = { schema } @@ -385,6 +385,16 @@ return params } + function toggleAttachmentBinding(e, key) { + onChange( + { + detail: "", + }, + key + ) + onChange({ detail: { useAttachmentBinding: e.detail } }, "meta") + } + onMount(async () => { try { await environment.loadVariables() @@ -462,27 +472,64 @@
-
- - onChange( - { - detail: e.detail.map(({ name, value }) => ({ - url: name, - filename: value, - })), - }, - key - )} - object={handleAttachmentParams(inputData[key])} - allowJS - {bindings} - keyBindings - customButtonText={"Add attachment"} - keyPlaceholder={"URL"} - valuePlaceholder={"Filename"} +
+ toggleAttachmentBinding(e, key)} />
+ +
+ {#if !inputData?.meta?.useAttachmentBinding} + + onChange( + { + detail: e.detail.map(({ name, value }) => ({ + url: name, + filename: value, + })), + }, + key + )} + object={handleAttachmentParams(inputData[key])} + allowJS + {bindings} + keyBindings + customButtonText={"Add attachment"} + keyPlaceholder={"URL"} + valuePlaceholder={"Filename"} + /> + {:else if isTestModal} + onChange(e, key)} + {bindings} + updateOnChange={false} + /> + {:else} +
+ onChange(e, key)} + {bindings} + updateOnChange={false} + placeholder={value.customType === "queryLimit" + ? queryLimit + : ""} + drawerLeft="260px" + /> +
+ {/if} +
{:else if value.customType === "filters"} Define filters diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index b5a54138ca..bd3bcda774 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -10,12 +10,12 @@ import { TableNames } from "constants" const dispatch = createEventDispatcher() - export let value export let meta export let bindings export let isTestModal export let isUpdateRow + $: parsedBindings = bindings.map(binding => { let clone = Object.assign({}, binding) clone.icon = "ShareAndroid" @@ -94,17 +94,22 @@ dispatch("change", newValue) } - const onChangeSetting = (e, field) => { - let fields = {} - fields[field] = { - clearRelationships: e.detail, + const onChangeSetting = (field, key, value) => { + let newField = {} + newField[field] = { + [key]: value, } + + let updatedFields = { + ...meta?.fields, + ...newField, + } + dispatch("change", { key: "meta", - fields, + fields: updatedFields, }) } - // Ensure any nullish tableId values get set to empty string so // that the select works $: if (value?.tableId == null) value = { tableId: "" } @@ -157,6 +162,9 @@ bindings={parsedBindings} {value} {onChange} + useAttachmentBinding={meta?.fields?.[field] + ?.useAttachmentBinding} + {onChangeSetting} /> {/if} @@ -167,7 +175,8 @@ value={meta.fields?.[field]?.clearRelationships} text={"Clear relationships if empty?"} size={"S"} - on:change={e => onChangeSetting(e, field)} + on:change={e => + onChangeSetting(field, "clearRelationships", e.detail)} /> {/if} diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte index 0a27360347..a43ff35c80 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte @@ -1,5 +1,11 @@ {#if schemaHasOptions(schema) && schema.type !== "array"} @@ -108,38 +131,65 @@ useLabel={false} /> {:else if attachmentTypes.includes(schema.type)} -
- - onChange( - { - detail: - schema.type === FieldType.ATTACHMENT_SINGLE || - schema.type === FieldType.SIGNATURE_SINGLE - ? e.detail.length > 0 - ? { - url: e.detail[0].name, - filename: e.detail[0].value, - } - : {} - : e.detail.map(({ name, value }) => ({ - url: name, - filename: value, - })), - }, - field - )} - object={handleAttachmentParams(value[field])} - allowJS - {bindings} - keyBindings - customButtonText={"Add attachment"} - keyPlaceholder={"URL"} - valuePlaceholder={"Filename"} - actionButtonDisabled={(schema.type === FieldType.ATTACHMENT_SINGLE || - schema.type === FieldType.SIGNATURE) && - Object.keys(value[field]).length >= 1} - /> +
+
+ handleToggleChange(field, e)} + /> +
+ {#if !useAttachmentBinding} +
+ { + onChange( + { + detail: + schema.type === FieldType.ATTACHMENT_SINGLE || + schema.type === FieldType.SIGNATURE_SINGLE + ? e.detail.length > 0 + ? { + url: e.detail[0].name, + filename: e.detail[0].value, + } + : {} + : e.detail.map(({ name, value }) => ({ + url: name, + filename: value, + })), + }, + field + ) + }} + object={handleAttachmentParams(value[field])} + allowJS + {bindings} + keyBindings + customButtonText={"Add attachment"} + keyPlaceholder={"URL"} + valuePlaceholder={"Filename"} + actionButtonDisabled={(schema.type === FieldType.ATTACHMENT_SINGLE || + schema.type === FieldType.SIGNATURE) && + Object.keys(value[field]).length >= 1} + /> +
+ {:else} +
+ onChange(e, field)} + type="string" + bindings={parsedBindings} + allowJS={true} + updateOnChange={false} + title={schema.name} + /> +
+ {/if}
{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)} - .attachment-field-spacinng { + .attachment-field-spacing, + .json-input-spacing { margin-top: var(--spacing-s); margin-bottom: var(--spacing-l); } diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte index 3b628c7b53..646b764a2c 100644 --- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte @@ -1,6 +1,6 @@
@@ -47,6 +57,16 @@ />