diff --git a/.eslintrc.json b/.eslintrc.json index fce6a9e27d..2c810eecc5 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -42,9 +42,18 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", "local-rules/no-budibase-imports": "error", - "local-rules/no-console-error": "error" + "local-rules/no-console-error": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], + "local-rules/no-budibase-imports": "error" } }, { @@ -60,7 +69,15 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], "local-rules/no-test-com": "error", "local-rules/email-domain-example-com": "error", "no-console": "warn", @@ -90,7 +107,8 @@ { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", - "destructuredArrayIgnorePattern": "^_" + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true } ], "import/no-relative-packages": "error", diff --git a/.vscode/settings.json b/.vscode/settings.json index e22d5a8866..0723219a8b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,5 +24,8 @@ }, "[svelte]": { "editor.defaultFormatter": "svelte.svelte-vscode" + }, + "[handlebars]": { + "editor.formatOnSave": false } } diff --git a/charts/budibase/README.md b/charts/budibase/README.md index dea7d1dbae..207992087d 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -150,6 +150,10 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.apps.autoscaling.maxReplicas | int | `10` | | | services.apps.autoscaling.minReplicas | int | `1` | | | services.apps.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the apps service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the apps pods. | +| services.apps.extraContainers | list | `[]` | Additional containers to be added to the apps pod. | +| services.apps.extraEnv | list | `[]` | Extra environment variables to set for apps pods. Takes a list of name=value pairs. | +| services.apps.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main apps container. | +| services.apps.extraVolumes | list | `[]` | Additional volumes to the apps pod. | | services.apps.httpLogging | int | `1` | Whether or not to log HTTP requests to the apps service. | | services.apps.livenessProbe | object | HTTP health checks. | Liveness probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.apps.logLevel | string | `"info"` | The log level for the apps service. | @@ -162,6 +166,10 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.automationWorkers.autoscaling.minReplicas | int | `1` | | | services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. | | services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. | +| services.automationWorkers.extraContainers | list | `[]` | Additional containers to be added to the automationWorkers pod. | +| services.automationWorkers.extraEnv | list | `[]` | Extra environment variables to set for automation worker pods. Takes a list of name=value pairs. | +| services.automationWorkers.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main automationWorkers container. | +| services.automationWorkers.extraVolumes | list | `[]` | Additional volumes to the automationWorkers pod. | | services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. | | services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | @@ -180,6 +188,9 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.objectStore.cloudfront.cdn | string | `""` | Set the url of a distribution to enable cloudfront. | | services.objectStore.cloudfront.privateKey64 | string | `""` | Base64 encoded private key for the above public key. | | services.objectStore.cloudfront.publicKeyId | string | `""` | ID of public key stored in cloudfront. | +| services.objectStore.extraContainers | list | `[]` | Additional containers to be added to the objectStore pod. | +| services.objectStore.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main objectStore container. | +| services.objectStore.extraVolumes | list | `[]` | Additional volumes to the objectStore pod. | | services.objectStore.minio | bool | `true` | Set to false if using another object store, such as S3. You will need to set `services.objectStore.url` to point to your bucket if you do this. | | services.objectStore.region | string | `""` | AWS_REGION if using S3 | | services.objectStore.resources | object | `{}` | The resources to use for Minio pods. See for more information on how to set these. | @@ -191,12 +202,19 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.proxy.autoscaling.maxReplicas | int | `10` | | | services.proxy.autoscaling.minReplicas | int | `1` | | | services.proxy.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the proxy service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the proxy pods. | +| services.proxy.extraContainers | list | `[]` | | +| services.proxy.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main proxy container. | +| services.proxy.extraVolumes | list | `[]` | Additional volumes to the proxy pod. | | services.proxy.livenessProbe | object | HTTP health checks. | Liveness probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.proxy.readinessProbe | object | HTTP health checks. | Readiness probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.proxy.replicaCount | int | `1` | The number of proxy replicas to run. | | services.proxy.resources | object | `{}` | The resources to use for proxy pods. See for more information on how to set these. | | services.proxy.startupProbe | object | HTTP health checks. | Startup probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.redis.enabled | bool | `true` | Whether or not to deploy a Redis pod into your cluster. | +| services.redis.extraContainers | list | `[]` | Additional containers to be added to the redis pod. | +| services.redis.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main redis container. | +| services.redis.extraVolumes | list | `[]` | Additional volumes to the redis pod. | +| services.redis.image | string | `"redis"` | The Redis image to use. | | services.redis.password | string | `"budibase"` | The password to use when connecting to Redis. It's recommended that you change this from the default if you're running Redis in-cluster. | | services.redis.port | int | `6379` | Port to expose Redis on. | | services.redis.resources | object | `{}` | The resources to use for Redis pods. See for more information on how to set these. | @@ -207,6 +225,10 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.worker.autoscaling.maxReplicas | int | `10` | | | services.worker.autoscaling.minReplicas | int | `1` | | | services.worker.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the worker pods. | +| services.worker.extraContainers | list | `[]` | Additional containers to be added to the worker pod. | +| services.worker.extraEnv | list | `[]` | Extra environment variables to set for worker pods. Takes a list of name=value pairs. | +| services.worker.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main worker container. | +| services.worker.extraVolumes | list | `[]` | Additional volumes to the worker pod. | | services.worker.httpLogging | int | `1` | Whether or not to log HTTP requests to the worker service. | | services.worker.livenessProbe | object | HTTP health checks. | Liveness probe configuration for worker pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.worker.logLevel | string | `"info"` | The log level for the worker service. | @@ -225,4 +247,4 @@ $ helm uninstall --namespace budibase budibase ``` ---------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.11.3](https://github.com/norwoodj/helm-docs/releases/v1.11.3) +Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index c7c4481122..b380908dd1 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -235,6 +235,13 @@ spec: args: {{- toYaml .Values.services.apps.args | nindent 10 }} {{ end }} + {{ if .Values.services.apps.extraVolumeMounts }} + volumeMounts: + {{- toYaml .Values.services.apps.extraVolumeMounts | nindent 10 }} + {{- end }} + {{- if .Values.services.apps.extraContainers }} + {{- toYaml .Values.services.apps.extraContainers | nindent 6 }} + {{- end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -258,4 +265,8 @@ spec: - name: ndots value: {{ .Values.services.apps.ndots | quote }} {{ end }} + {{ if .Values.services.apps.extraVolumes }} + volumes: + {{- toYaml .Values.services.apps.extraVolumes | nindent 6 }} + {{- end }} status: {} diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 36c3a8ffbf..51fa9ee4bb 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -235,6 +235,13 @@ spec: args: {{- toYaml .Values.services.automationWorkers.args | nindent 10 }} {{ end }} + {{ if .Values.services.automationWorkers.extraVolumeMounts }} + volumeMounts: + {{- toYaml .Values.services.automationWorkers.extraVolumeMounts | nindent 10 }} + {{ end }} + {{- if .Values.services.automationWorkers.extraContainers }} + {{- toYaml .Values.services.automationWorkers.extraContainers | nindent 6 }} + {{- end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -258,5 +265,9 @@ spec: - name: ndots value: {{ .Values.services.automationWorkers.ndots | quote }} {{ end }} + {{ if .Values.services.automationWorkers.extraVolumes }} + volumes: + {{- toYaml .Values.services.automationWorkers.extraVolumes | nindent 8 }} + {{ end }} status: {} {{- end }} \ No newline at end of file diff --git a/charts/budibase/templates/minio-service-deployment.yaml b/charts/budibase/templates/minio-service-deployment.yaml index 28e8eb9991..901ead2b46 100644 --- a/charts/budibase/templates/minio-service-deployment.yaml +++ b/charts/budibase/templates/minio-service-deployment.yaml @@ -54,6 +54,12 @@ spec: volumeMounts: - mountPath: /data name: minio-data + {{ if .Values.services.objectStore.extraVolumeMounts }} + {{- toYaml .Values.services.objectStore.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.services.objectStore.extraContainers }} + {{- toYaml .Values.services.objectStore.extraContainers | nindent 6 }} + {{- end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -75,5 +81,8 @@ spec: - name: minio-data persistentVolumeClaim: claimName: minio-data + {{ if .Values.services.objectStore.extraVolumes }} + {{- toYaml .Values.services.objectStore.extraVolumes | nindent 6 }} + {{- end }} status: {} {{- end }} diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml index 233028cafe..d5ea696431 100644 --- a/charts/budibase/templates/proxy-service-deployment.yaml +++ b/charts/budibase/templates/proxy-service-deployment.yaml @@ -82,7 +82,13 @@ spec: resources: {{- toYaml . | nindent 10 }} {{ end }} + {{ if .Values.services.proxy.extraVolumeMounts }} volumeMounts: + {{- toYaml .Values.services.proxy.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.services.proxy.extraContainers }} + {{- toYaml .Values.services.proxy.extraContainers | nindent 6 }} + {{- end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -108,7 +114,10 @@ spec: args: {{- toYaml .Values.services.proxy.args | nindent 8 }} {{ end }} + {{ if .Values.services.proxy.extraVolumes }} volumes: + {{- toYaml .Values.services.proxy.extraVolumes | nindent 6 }} + {{ end }} {{ if .Values.services.proxy.ndots }} dnsConfig: options: diff --git a/charts/budibase/templates/redis-service-deployment.yaml b/charts/budibase/templates/redis-service-deployment.yaml index bca40d2237..9ad12e0167 100644 --- a/charts/budibase/templates/redis-service-deployment.yaml +++ b/charts/budibase/templates/redis-service-deployment.yaml @@ -22,7 +22,7 @@ spec: - redis-server - --requirepass - {{ .Values.services.redis.password }} - image: redis + image: {{ .Values.services.redis.image }} imagePullPolicy: "" name: redis-service ports: @@ -34,6 +34,12 @@ spec: volumeMounts: - mountPath: /data name: redis-data + {{ if .Values.services.redis.extraVolumeMounts }} + {{- toYaml .Values.services.redis.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.services.redis.extraContainers }} + {{- toYaml .Values.services.redis.extraContainers | nindent 6 }} + {{- end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -55,6 +61,9 @@ spec: - name: redis-data persistentVolumeClaim: claimName: redis-data + {{ if .Values.services.redis.extraVolumes }} + {{- toYaml .Values.services.redis.extraVolumes | nindent 6 }} + {{- end }} status: {} {{- end }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 2f97508ae3..e37b2bc0e4 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -221,6 +221,13 @@ spec: args: {{- toYaml .Values.services.worker.args | nindent 10 }} {{ end }} + {{ if .Values.services.worker.extraVolumeMounts }} + volumeMounts: + {{- toYaml .Values.services.worker.extraVolumeMounts | nindent 10 }} + {{- end }} + {{- if .Values.services.worker.extraContainers }} + {{- toYaml .Values.services.worker.extraContainers | nindent 6 }} + {{- end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -244,4 +251,8 @@ spec: - name: ndots value: {{ .Values.services.worker.ndots | quote }} {{ end }} + {{ if .Values.services.worker.extraVolumes }} + volumes: + {{- toYaml .Values.services.worker.extraVolumes | nindent 6 }} + {{- end }} status: {} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 19b6c22d6c..9ace768625 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -206,6 +206,20 @@ services: # for autoscaling to work, you will need to have metrics-server # configured, and resources set for the proxy pods. targetCPUUtilizationPercentage: 80 + # -- Additional containers to be added to the proxy pod. + extraContainers: [] + # - name: my-sidecar + # image: myimage:latest + + # -- Additional volumeMounts to the main proxy container. + extraVolumeMounts: [] + # - name: my-volume + # mountPath: /path/to/mount + + # -- Additional volumes to the proxy pod. + extraVolumes: [] + # - name: my-volume + # emptyDir: {} apps: # @ignore (you shouldn't need to change this) @@ -274,6 +288,20 @@ services: # autoscaling to work, you will need to have metrics-server configured, # and resources set for the apps pods. targetCPUUtilizationPercentage: 80 + # -- Additional containers to be added to the apps pod. + extraContainers: [] + # - name: my-sidecar + # image: myimage:latest + + # -- Additional volumeMounts to the main apps container. + extraVolumeMounts: [] + # - name: my-volume + # mountPath: /path/to/mount + + # -- Additional volumes to the apps pod. + extraVolumes: [] + # - name: my-volume + # emptyDir: {} automationWorkers: # -- Whether or not to enable the automation worker service. If you disable this, @@ -346,6 +374,20 @@ services: # Note that for autoscaling to work, you will need to have metrics-server # configured, and resources set for the automation worker pods. targetCPUUtilizationPercentage: 80 + # -- Additional containers to be added to the automationWorkers pod. + extraContainers: [] + # - name: my-sidecar + # image: myimage:latest + + # -- Additional volumeMounts to the main automationWorkers container. + extraVolumeMounts: [] + # - name: my-volume + # mountPath: /path/to/mount + + # -- Additional volumes to the automationWorkers pod. + extraVolumes: [] + # - name: my-volume + # emptyDir: {} worker: # @ignore (you shouldn't need to change this) @@ -414,6 +456,20 @@ services: # for autoscaling to work, you will need to have metrics-server # configured, and resources set for the worker pods. targetCPUUtilizationPercentage: 80 + # -- Additional containers to be added to the worker pod. + extraContainers: [] + # - name: my-sidecar + # image: myimage:latest + + # -- Additional volumeMounts to the main worker container. + extraVolumeMounts: [] + # - name: my-volume + # mountPath: /path/to/mount + + # -- Additional volumes to the worker pod. + extraVolumes: [] + # - name: my-volume + # emptyDir: {} couchdb: # -- Whether or not to spin up a CouchDB instance in your cluster. True by @@ -440,6 +496,8 @@ services: resources: {} redis: + # -- The Redis image to use. + image: redis # -- Whether or not to deploy a Redis pod into your cluster. enabled: true # -- Port to expose Redis on. @@ -463,6 +521,20 @@ services: # # for more information on how to set these. resources: {} + # -- Additional containers to be added to the redis pod. + extraContainers: [] + # - name: my-sidecar + # image: myimage:latest + + # -- Additional volumeMounts to the main redis container. + extraVolumeMounts: [] + # - name: my-volume + # mountPath: /path/to/mount + + # -- Additional volumes to the redis pod. + extraVolumes: [] + # - name: my-volume + # emptyDir: {} objectStore: # -- Set to false if using another object store, such as S3. You will need @@ -488,7 +560,7 @@ services: # do this. url: "http://minio-service:9000" # -- How much storage to give Minio in its PersistentVolumeClaim. - storage: 100Mi + storage: 2Gi # -- If defined, storageClassName: If set to "-", # storageClassName: "", which disables dynamic provisioning If undefined # (the default) or set to null, no storageClassName spec is set, choosing @@ -505,6 +577,20 @@ services: publicKeyId: "" # -- Base64 encoded private key for the above public key. privateKey64: "" + # -- Additional containers to be added to the objectStore pod. + extraContainers: [] + # - name: my-sidecar + # image: myimage:latest + + # -- Additional volumeMounts to the main objectStore container. + extraVolumeMounts: [] + # - name: my-volume + # mountPath: /path/to/mount + + # -- Additional volumes to the objectStore pod. + extraVolumes: [] + # - name: my-volume + # emptyDir: {} # Override values in couchDB subchart. We're only specifying the values we're changing. # If you want to see all of the available values, see: diff --git a/hosting/.env b/hosting/.env index 8a0756c0e3..173d409d04 100644 --- a/hosting/.env +++ b/hosting/.env @@ -17,6 +17,7 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 +COUCH_DB_SQS_PORT=4006 REDIS_PORT=6379 WATCHTOWER_PORT=6161 BUDIBASE_ENVIRONMENT=PRODUCTION @@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD= # A path that is watched for plugin bundles. Any bundles found are imported automatically/ PLUGINS_DIR= -ROLLING_LOG_MAX_SIZE= \ No newline at end of file +ROLLING_LOG_MAX_SIZE= diff --git a/lerna.json b/lerna.json index 78a3aa13e9..e8bcd4429c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.4", + "version": "2.23.10", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/nx.json b/nx.json index 618395ec90..8ba8798946 100644 --- a/nx.json +++ b/nx.json @@ -9,10 +9,7 @@ }, "targetDefaults": { "build": { - "inputs": [ - "{workspaceRoot}/scripts/build.js", - "{workspaceRoot}/lerna.json" - ] + "inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"] } } } diff --git a/package.json b/package.json index 2816247939..e60a086e17 100644 --- a/package.json +++ b/package.json @@ -56,9 +56,10 @@ "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server", "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server", "dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server", + "dev:camunda": "./scripts/deploy-camunda.sh", "dev:all": "yarn run kill-all && lerna run --stream dev", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "./scripts/devDocker.sh", "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream", "lint:eslint": "eslint packages --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", diff --git a/packages/account-portal b/packages/account-portal index bd0e01d639..c167c331ff 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit bd0e01d639ec3b2547e7c859a1c43b622dce8344 +Subproject commit c167c331ff9b8161fc18e2ecbaaf1ea5815ba964 diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 4a2cfd34e2..d9dddd0097 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -8,19 +8,9 @@ import { SearchParams, WithRequired, } from "@budibase/types" +import { dataFilters } from "@budibase/shared-core" -const QUERY_START_REGEX = /\d[0-9]*:/g - -export function removeKeyNumbering(key: any): string { - if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { - const parts = key.split(":") - // remove the number - parts.shift() - return parts.join(":") - } else { - return key - } -} +export const removeKeyNumbering = dataFilters.removeKeyNumbering /** * Class to build lucene query URLs. diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2da2a77d67..8dbc904643 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -107,7 +107,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", - COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 87e43b324d..333accc985 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -115,7 +115,6 @@ class InMemoryQueue implements Partial { * a JSON message as this is required by Bull. * @param repeat serves no purpose for the import queue. */ - // eslint-disable-next-line no-unused-vars async add(data: any, opts?: JobOptions) { const jobId = opts?.jobId?.toString() if (jobId && this._queuedJobIds.has(jobId)) { @@ -166,8 +165,7 @@ class InMemoryQueue implements Partial { return [] } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async removeJobs(pattern: string) { + async removeJobs(_pattern: string) { // no-op } diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 6165a68e57..f77c6385ba 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -50,6 +50,8 @@ type CreateAdminUserOpts = { hashPassword?: boolean requirePassword?: boolean skipPasswordValidation?: boolean + firstName?: string + lastName?: string } type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn } @@ -517,6 +519,8 @@ export class UserDB { global: true, }, tenantId, + firstName: opts?.firstName, + lastName: opts?.lastName, } if (opts?.ssoId) { user.ssoId = opts.ssoId diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 48920a3771..7d62a6ef39 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -17,8 +17,8 @@ import { ContextUser, CouchFindOptions, DatabaseQueryOpts, - SearchQuery, - SearchQueryOperators, + SearchFilters, + SearchFilterOperator, SearchUsersRequest, User, } from "@budibase/types" @@ -44,11 +44,11 @@ function removeUserPassword(users: User | User[]) { return users } -export function isSupportedUserSearch(query: SearchQuery) { +export function isSupportedUserSearch(query: SearchFilters) { const allowed = [ - { op: SearchQueryOperators.STRING, key: "email" }, - { op: SearchQueryOperators.EQUAL, key: "_id" }, - { op: SearchQueryOperators.ONE_OF, key: "_id" }, + { op: SearchFilterOperator.STRING, key: "email" }, + { op: SearchFilterOperator.EQUAL, key: "_id" }, + { op: SearchFilterOperator.ONE_OF, key: "_id" }, ] for (let [key, operation] of Object.entries(query)) { if (typeof operation !== "object") { diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 0632993cf0..6434c7710d 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -14,6 +14,7 @@ notifications, Checkbox, DatePicker, + DrawerContent, } from "@budibase/bbui" import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte" import { automationStore, selectedAutomation, tables } from "stores/builder" @@ -37,7 +38,7 @@ hbAutocomplete, EditorModes, } from "components/common/CodeEditor" - import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte" + import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte" import { LuceneUtils, Utils } from "@budibase/frontend-core" import { getSchemaForDatasourcePlus, @@ -442,15 +443,16 @@ - (tempFilters = e.detail)} - /> + + (tempFilters = e.detail)} + /> + {:else if value.customType === "password"} import { createEventDispatcher } from "svelte" import { ActionButton, Modal, ModalContent } from "@budibase/bbui" - import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte" + import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte" export let schema export let filters @@ -40,7 +40,7 @@ onConfirm={() => dispatch("change", tempValue)} >
- t.toLowerCase()) + return Object.entries(FIELDS) + .filter(([fieldType]) => + possibleTypes.includes(fieldType.toLowerCase()) + ) + .map(([_, fieldDefinition]) => fieldDefinition) } const isUsers = @@ -632,7 +626,7 @@ />
- {:else if editableColumn.type === FieldType.LINK} + {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn} diff --git a/packages/builder/src/components/common/users/PasswordRepeatInput.svelte b/packages/builder/src/components/common/users/PasswordRepeatInput.svelte index 496bee14ec..4a453ef049 100644 --- a/packages/builder/src/components/common/users/PasswordRepeatInput.svelte +++ b/packages/builder/src/components/common/users/PasswordRepeatInput.svelte @@ -9,7 +9,6 @@ "", requiredValidator ) - // eslint-disable-next-line no-unused-vars const [repeatPassword, _, repeatTouched] = createValidationStore( "", requiredValidator diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DownloadFile.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DownloadFile.svelte new file mode 100644 index 0000000000..babaf2815b --- /dev/null +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DownloadFile.svelte @@ -0,0 +1,110 @@ + + +
+ + table.label} + getOptionValue={table => table.resourceId} + /> + + onFieldChange(filter, e.detail)} - placeholder="Column" - /> - - {:else if ["options", "array"].includes(filter.type)} - - {:else if filter.type === "boolean"} - - {:else if filter.type === "datetime"} - - {:else} - - {/if} -
- duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> -
- {/each} -
- {/if} -
- -
- - - - + +
+ Results are filtered to only those which match all of the following + constraints. +
+ diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte index 644630810d..3489fd809c 100644 --- a/packages/client/src/components/app/forms/AttachmentField.svelte +++ b/packages/client/src/components/app/forms/AttachmentField.svelte @@ -58,17 +58,6 @@ } } - const deleteAttachments = async fileList => { - try { - return await API.deleteAttachments({ - keys: fileList, - tableId: formContext?.dataSource?.tableId, - }) - } catch (error) { - return [] - } - } - const handleChange = e => { const value = fieldApiMapper.set(e.detail) const changed = fieldApi.setValue(value) @@ -98,7 +87,6 @@ error={fieldState.error} on:change={handleChange} {processFiles} - {deleteAttachments} {handleFileTooLarge} {handleTooManyFiles} {maximum} diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js index 68478b76ac..d883ee1b55 100644 --- a/packages/client/src/utils/buttonActions.js +++ b/packages/client/src/utils/buttonActions.js @@ -1,5 +1,6 @@ import { get } from "svelte/store" import download from "downloadjs" +import { downloadStream } from "@budibase/frontend-core" import { routeStore, builderStore, @@ -400,6 +401,51 @@ const closeSidePanelHandler = () => { sidePanelStore.actions.close() } +const downloadFileHandler = async action => { + const { url, fileName } = action.parameters + try { + const { type } = action.parameters + if (type === "attachment") { + const { tableId, rowId, attachmentColumn } = action.parameters + const res = await API.downloadAttachment( + tableId, + rowId, + attachmentColumn, + { suppressErrors: true } + ) + await downloadStream(res) + return + } + + const response = await fetch(url) + + if (!response.ok) { + notificationStore.actions.error( + `Failed to download from '${url}'. Server returned status code: ${response.status}` + ) + return + } + + const objectUrl = URL.createObjectURL(await response.blob()) + + const link = document.createElement("a") + link.href = objectUrl + link.download = fileName + link.click() + + URL.revokeObjectURL(objectUrl) + } catch (e) { + console.error(e) + if (e.status) { + notificationStore.actions.error( + `Failed to download from '${url}'. Server returned status code: ${e.status}` + ) + } else { + notificationStore.actions.error(`Failed to download from '${url}'.`) + } + } +} + const handlerMap = { ["Fetch Row"]: fetchRowHandler, ["Save Row"]: saveRowHandler, @@ -418,6 +464,7 @@ const handlerMap = { ["Prompt User"]: promptUserHandler, ["Open Side Panel"]: openSidePanelHandler, ["Close Side Panel"]: closeSidePanelHandler, + ["Download File"]: downloadFileHandler, } const confirmTextMap = { diff --git a/packages/client/tsconfig.json b/packages/client/tsconfig.json new file mode 100644 index 0000000000..81f1657f48 --- /dev/null +++ b/packages/client/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "allowJs": true, + "strict": true, + "outDir": "dist", + "paths": { + "@budibase/*": [ + "../*/src/index.ts", + "../*/src/index.js", + "../*", + "../../node_modules/@budibase/*" + ], + "*": ["./src/*"] + } + }, + "include": ["src/**/*"] +} diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 4ca88de8f2..3f97573d4a 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -11,6 +11,7 @@ "@budibase/types": "0.0.0", "dayjs": "^1.10.8", "lodash": "4.17.21", + "shortid": "2.2.15", "socket.io-client": "^4.6.1" } } diff --git a/packages/frontend-core/src/api/attachments.js b/packages/frontend-core/src/api/attachments.js index f79b461574..72f280d99d 100644 --- a/packages/frontend-core/src/api/attachments.js +++ b/packages/frontend-core/src/api/attachments.js @@ -61,31 +61,17 @@ export const buildAttachmentEndpoints = API => { }) return { publicUrl } }, - /** - * Deletes attachments from the bucket. - * @param keys the attachments to delete - * @param tableId the associated table ID + * Download an attachment from a row given its column name. + * @param datasourceId the ID of the datasource to download from + * @param rowId the ID of the row to download from + * @param columnName the column name to download */ - deleteAttachments: async ({ keys, tableId }) => { - return await API.post({ - url: `/api/attachments/${tableId}/delete`, - body: { - keys, - }, - }) - }, - - /** - * Deletes attachments from the builder bucket. - * @param keys the attachments to delete - */ - deleteBuilderAttachments: async keys => { - return await API.post({ - url: `/api/attachments/delete`, - body: { - keys, - }, + downloadAttachment: async (datasourceId, rowId, columnName, options) => { + return await API.get({ + url: `/api/${datasourceId}/rows/${rowId}/attachment/${columnName}`, + parseResponse: response => response, + suppressErrors: options?.suppressErrors, }) }, } diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte similarity index 50% rename from packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte rename to packages/frontend-core/src/components/FilterBuilder.svelte index 7f1ee8010d..1b252d5b06 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -4,33 +4,36 @@ Button, Combobox, DatePicker, - DrawerContent, Icon, Input, - Label, Layout, - Multiselect, Select, + Label, + Multiselect, } from "@budibase/bbui" - import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" - import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" + import { FieldType, SearchFilterOperator } from "@budibase/types" import { generate } from "shortid" - import { Constants, LuceneUtils } from "@budibase/frontend-core" - import { getFields } from "helpers/searchFields" - import { FieldType } from "@budibase/types" - import { createEventDispatcher, onMount } from "svelte" + import { LuceneUtils, Constants } from "@budibase/frontend-core" + import { getContext } from "svelte" import FilterUsers from "./FilterUsers.svelte" + const { OperatorOptions } = Constants + export let schemaFields export let filters = [] - export let bindings = [] - export let panel = ClientBindingPanel - export let allowBindings = true export let datasource + export let behaviourFilters = false + export let allowBindings = false + export let filtersLabel = "Filters" + + $: matchAny = filters?.find(filter => filter.operator === "allOr") != null + $: onEmptyFilter = + filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" + + $: fieldFilters = filters.filter( + filter => filter.operator !== "allOr" && !filter.onEmptyFilter + ) - const dispatch = createEventDispatcher() - const { OperatorOptions } = Constants - const KeyedFieldRegex = /\d[0-9]*:/g const behaviourOptions = [ { value: "and", label: "Match all filters" }, { value: "or", label: "Match any filter" }, @@ -40,62 +43,18 @@ { value: "none", label: "Return no rows" }, ] - let rawFilters - let matchAny = false - let onEmptyFilter = "all" + const context = getContext("context") - $: parseFilters(filters) - $: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter)) - $: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true }) - $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] - $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] - - // Remove field key prefixes and determine which behaviours to use - const parseFilters = filters => { - matchAny = filters?.find(filter => filter.operator === "allOr") != null - onEmptyFilter = - filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" - rawFilters = (filters || []) - .filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter) - .map(filter => { - const { field } = filter - let newFilter = { ...filter } - delete newFilter.allOr - if (typeof field === "string" && field.match(KeyedFieldRegex) != null) { - const parts = field.split(":") - parts.shift() - newFilter.field = parts.join(":") - } - return newFilter - }) - } - - onMount(() => { - parseFilters(filters) - rawFilters.forEach(filter => { - filter.type = - schemaFields.find(field => field.name === filter.field)?.type || - filter.type - }) - }) - - // Add field key prefixes and a special metadata filter object to indicate - // how to handle filter behaviour - const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => { - let count = 1 - return rawFilters - .filter(filter => filter.field) - .map(filter => ({ - ...filter, - field: `${count++}:${filter.field}`, - })) - .concat(matchAny ? [{ operator: "allOr" }] : []) - .concat([{ onEmptyFilter }]) - } + $: fieldOptions = (schemaFields ?? []) + .filter(field => getValidOperatorsForType(field).length) + .map(field => ({ + label: field.displayName || field.name, + value: field.name, + })) const addFilter = () => { - rawFilters = [ - ...rawFilters, + filters = [ + ...(filters || []), { id: generate(), field: null, @@ -107,22 +66,57 @@ } const removeFilter = id => { - rawFilters = rawFilters.filter(field => field.id !== id) + filters = filters.filter(field => field.id !== id) } const duplicateFilter = id => { - const existingFilter = rawFilters.find(filter => filter.id === id) + const existingFilter = filters.find(filter => filter.id === id) const duplicate = { ...existingFilter, id: generate() } - rawFilters = [...rawFilters, duplicate] + filters = [...filters, duplicate] + } + + const onFieldChange = filter => { + const previousType = filter.type + sanitizeTypes(filter) + sanitizeOperator(filter) + sanitizeValue(filter, previousType) + } + + const onOperatorChange = filter => { + sanitizeOperator(filter) + sanitizeValue(filter, filter.type) + } + + const onValueTypeChange = filter => { + sanitizeValue(filter) + } + + const getFieldOptions = field => { + const schema = schemaFields.find(x => x.name === field) + return schema?.constraints?.inclusion || [] } const getSchema = filter => { - return enrichedSchemaFields.find(field => field.name === filter.field) + return schemaFields.find(field => field.name === filter.field) } + const getValidOperatorsForType = filter => { + if (!filter?.field && !filter?.name) { + return [] + } + + return LuceneUtils.getValidOperatorsForType( + filter, + filter.field || filter.name, + datasource + ) + } + + $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] + const sanitizeTypes = filter => { // Update type based on field - const fieldSchema = enrichedSchemaFields.find(x => x.name === filter.field) + const fieldSchema = schemaFields.find(x => x.name === filter.field) filter.type = fieldSchema?.type filter.subtype = fieldSchema?.subtype @@ -154,88 +148,79 @@ // Ensure array values are properly set and cleared if (Array.isArray(filter.value)) { - if (filter.valueType !== "Value" || filter.type !== "array") { + if (filter.valueType !== "Value" || filter.type !== FieldType.ARRAY) { filter.value = null } - } else if (filter.type === "array" && filter.valueType === "Value") { + } else if ( + filter.type === FieldType.ARRAY && + filter.valueType === "Value" + ) { filter.value = [] } else if ( previousType !== filter.type && (previousType === FieldType.BB_REFERENCE || filter.type === FieldType.BB_REFERENCE) ) { - filter.value = filter.type === "array" ? [] : null + filter.value = filter.type === FieldType.ARRAY ? [] : null } } - const onFieldChange = filter => { - const previousType = filter.type - sanitizeTypes(filter) - sanitizeOperator(filter) - sanitizeValue(filter, previousType) - } - - const onOperatorChange = filter => { - sanitizeOperator(filter) - sanitizeValue(filter, filter.type) - } - - const onValueTypeChange = filter => { - sanitizeValue(filter) - } - - const getFieldOptions = field => { - const schema = enrichedSchemaFields.find(x => x.name === field) - return schema?.constraints?.inclusion || [] - } - - const getValidOperatorsForType = filter => { - if (!filter?.field) { - return [] + function handleAllOr(option) { + filters = filters.filter(f => f.operator !== "allOr") + if (option === "or") { + filters.push({ operator: "allOr" }) } + } - return LuceneUtils.getValidOperatorsForType( - { type: filter.type, subtype: filter.subtype }, - filter.field, - datasource - ) + function handleOnEmptyFilter(value) { + filters = filters?.filter(filter => !filter.onEmptyFilter) + filters.push({ onEmptyFilter: value }) } - -
- - {#if !rawFilters?.length} - Add your first filter expression. - {:else} -
- opt.label} - getOptionValue={opt => opt.value} - on:change={e => (onEmptyFilter = e.detail)} - placeholder={null} - /> +
+ + {#if fieldOptions?.length} + + {#if !fieldFilters?.length} + Add your first filter expression. + {:else} + + {#if behaviourFilters} +
+ opt.label} + getOptionValue={opt => opt.value} + on:change={e => handleOnEmptyFilter(e.detail)} + placeholder={null} + /> + {/if} +
{/if} -
+ {/if} + + {#if fieldFilters?.length}
-
- -
-
- {#each rawFilters as filter} + {#if filtersLabel} +
+ +
+ {/if} +
+ {#each fieldFilters as filter} onValueTypeChange(filter)} - placeholder={null} - /> - {#if filter.field && filter.valueType === "Binding"} - (filter.value = event.detail)} + {#if allowBindings} + - {:else if filter.type === "array" || (filter.type === "options" && filter.operator === "oneOf")} + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchFilterOperator.ONE_OF)} - {:else if filter.type === "options"} + {:else if filter.type === FieldType.OPTIONS} - {:else if filter.type === "boolean"} + {:else if filter.type === FieldType.BOOLEAN} - {:else if filter.type === "datetime"} + {:else if filter.type === FieldType.DATETIME} {:else} - + {/if} - duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> +
+ duplicateFilter(filter.id)} + /> + removeFilter(filter.id)} + /> +
{/each}
{/if} -
+
- -
- + {:else} + None of the table column can be used for filtering. + {/if} + +
diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte b/packages/frontend-core/src/components/FilterUsers.svelte similarity index 88% rename from packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte rename to packages/frontend-core/src/components/FilterUsers.svelte index 88383ba170..1712d7ebdf 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte +++ b/packages/frontend-core/src/components/FilterUsers.svelte @@ -1,9 +1,9 @@