CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-08 15:56:23 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into feature/sql-query-aliasing

Browse source
This commit is contained in:
mike12345567 2023-12-18 13:14:15 +00:00
parent 615bf4a5b6 22c0e81308
commit c6b2366bf0
64 changed files with 977 additions and 1009 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/budibase_ci.yml vendored
View file

@ -204,7 +204,7 @@ jobs:
check-pro-submodule:
runs-on: ubuntu-latest
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3
@ -254,7 +254,7 @@ jobs:
check-accountportal-submodule:
runs-on: ubuntu-latest
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3

7
.github/workflows/close-featurebranch.yml vendored
View file

@ -2,9 +2,7 @@ name: close-featurebranch
on:
pull_request:
types: [closed]
branches:
- master
types: [closed, unlabeled]
workflow_dispatch:
inputs:
BRANCH:
@ -14,6 +12,9 @@ on:
jobs:
release:
if: |
(github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'feature-branch')) ||
github.event.label.name == 'feature-branch'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

13
.github/workflows/deploy-featurebranch.yml vendored
View file

@ -2,12 +2,19 @@ name: deploy-featurebranch
on:
pull_request:
branches:
- master
types: [
labeled,
# default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request)
opened,
synchronize,
reopened,
]
jobs:
release:
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
if: |
(github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') &&
contains(github.event.pull_request.labels.*.name, 'feature-branch')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

2
.vscode/settings.json vendored
View file

@ -1,7 +1,7 @@
{
"editor.formatOnSave": true,
"editor.codeActionsOnSave": {
"source.fixAll": true
"source.fixAll": "explicit"
},
"editor.defaultFormatter": "esbenp.prettier-vscode",
"[json]": {

11
charts/budibase/README.md
View file

@ -157,6 +157,17 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
| services.apps.replicaCount | int | `1` | The number of apps replicas to run. |
| services.apps.resources | object | `{}` | The resources to use for apps pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
| services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
| services.automationWorkers.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. |
| services.automationWorkers.autoscaling.maxReplicas | int | `10` | |
| services.automationWorkers.autoscaling.minReplicas | int | `1` | |
| services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. |
| services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. |
| services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
| services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. |
| services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
| services.automationWorkers.replicaCount | int | `1` | The number of automation worker replicas to run. |
| services.automationWorkers.resources | object | `{}` | The resources to use for automation worker pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
| services.automationWorkers.startupProbe | object | HTTP health checks. | Startup probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
| services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. |
| services.couchdb.backup.interval | string | `""` | Backup interval in seconds |
| services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |

9
charts/budibase/templates/app-service-deployment.yaml
View file

@ -192,7 +192,14 @@ spec:
- name: NODE_TLS_REJECT_UNAUTHORIZED
value: {{ .Values.services.tlsRejectUnauthorized }}
{{ end }}
{{- if .Values.services.automationWorkers.enabled }}
- name: APP_FEATURES
value: "api"
{{- end }}
{{- range .Values.services.apps.extraEnv }}
- name: {{ .name }}
value: {{ .value | quote }}
{{- end }}
image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
imagePullPolicy: Always
{{- if .Values.services.apps.startupProbe }}

248
charts/budibase/templates/automation-worker-service-deployment.yaml Normal file
View file

@ -0,0 +1,248 @@
{{- if .Values.services.automationWorkers.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
{{ if .Values.services.automationWorkers.deploymentAnnotations }}
{{- toYaml .Values.services.automationWorkers.deploymentAnnotations | indent 4 -}}
{{ end }}
labels:
io.kompose.service: automation-worker-service
{{ if .Values.services.automationWorkers.deploymentLabels }}
{{- toYaml .Values.services.automationWorkers.deploymentLabels | indent 4 -}}
{{ end }}
name: automation-worker-service
spec:
replicas: {{ .Values.services.automationWorkers.replicaCount }}
selector:
matchLabels:
io.kompose.service: automation-worker-service
strategy:
type: RollingUpdate
template:
metadata:
annotations:
{{ if .Values.services.automationWorkers.templateAnnotations }}
{{- toYaml .Values.services.automationWorkers.templateAnnotations | indent 8 -}}
{{ end }}
labels:
io.kompose.service: automation-worker-service
{{ if .Values.services.automationWorkers.templateLabels }}
{{- toYaml .Values.services.automationWorkers.templateLabels | indent 8 -}}
{{ end }}
spec:
containers:
- env:
- name: BUDIBASE_ENVIRONMENT
value: {{ .Values.globals.budibaseEnv }}
- name: DEPLOYMENT_ENVIRONMENT
value: "kubernetes"
- name: COUCH_DB_URL
{{ if .Values.services.couchdb.url }}
value: {{ .Values.services.couchdb.url }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
{{ if .Values.services.couchdb.enabled }}
- name: COUCH_DB_USER
valueFrom:
secretKeyRef:
name: {{ template "couchdb.fullname" . }}
key: adminUsername
- name: COUCH_DB_PASSWORD
valueFrom:
secretKeyRef:
name: {{ template "couchdb.fullname" . }}
key: adminPassword
{{ end }}
- name: ENABLE_ANALYTICS
value: {{ .Values.globals.enableAnalytics | quote }}
- name: API_ENCRYPTION_KEY
value: {{ .Values.globals.apiEncryptionKey | quote }}
- name: HTTP_LOGGING
value: {{ .Values.services.automationWorkers.httpLogging | quote }}
- name: INTERNAL_API_KEY
valueFrom:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: internalApiKey
- name: INTERNAL_API_KEY_FALLBACK
value: {{ .Values.globals.internalApiKeyFallback | quote }}
- name: JWT_SECRET
valueFrom:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: jwtSecret
- name: JWT_SECRET_FALLBACK
value: {{ .Values.globals.jwtSecretFallback | quote }}
{{ if .Values.services.objectStore.region }}
- name: AWS_REGION
value: {{ .Values.services.objectStore.region }}
{{ end }}
- name: MINIO_ENABLED
value: {{ .Values.services.objectStore.minio | quote }}
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: objectStoreAccess
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: objectStoreSecret
- name: CLOUDFRONT_CDN
value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
- name: CLOUDFRONT_PUBLIC_KEY_ID
value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
- name: CLOUDFRONT_PRIVATE_KEY_64
value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
- name: MINIO_URL
value: {{ .Values.services.objectStore.url }}
- name: PLUGIN_BUCKET_NAME
value: {{ .Values.services.objectStore.pluginBucketName | quote }}
- name: APPS_BUCKET_NAME
value: {{ .Values.services.objectStore.appsBucketName | quote }}
- name: GLOBAL_BUCKET_NAME
value: {{ .Values.services.objectStore.globalBucketName | quote }}
- name: BACKUPS_BUCKET_NAME
value: {{ .Values.services.objectStore.backupsBucketName | quote }}
- name: PORT
value: {{ .Values.services.automationWorkers.port | quote }}
{{ if .Values.services.worker.publicApiRateLimitPerSecond }}
- name: API_REQ_LIMIT_PER_SEC
value: {{ .Values.globals.automationWorkers.publicApiRateLimitPerSecond | quote }}
{{ end }}
- name: MULTI_TENANCY
value: {{ .Values.globals.multiTenancy | quote }}
- name: OFFLINE_MODE
value: {{ .Values.globals.offlineMode | quote }}
- name: LOG_LEVEL
value: {{ .Values.services.automationWorkers.logLevel | quote }}
- name: REDIS_PASSWORD
value: {{ .Values.services.redis.password }}
- name: REDIS_URL
{{ if .Values.services.redis.url }}
value: {{ .Values.services.redis.url }}
{{ else }}
value: redis-service:{{ .Values.services.redis.port }}
{{ end }}
- name: SELF_HOSTED
value: {{ .Values.globals.selfHosted | quote }}
- name: POSTHOG_TOKEN
value: {{ .Values.globals.posthogToken | quote }}
- name: WORKER_URL
value: http://worker-service:{{ .Values.services.worker.port }}
- name: PLATFORM_URL
value: {{ .Values.globals.platformUrl | quote }}
- name: ACCOUNT_PORTAL_URL
value: {{ .Values.globals.accountPortalUrl | quote }}
- name: ACCOUNT_PORTAL_API_KEY
value: {{ .Values.globals.accountPortalApiKey | quote }}
- name: COOKIE_DOMAIN
value: {{ .Values.globals.cookieDomain | quote }}
- name: HTTP_MIGRATIONS
value: {{ .Values.globals.httpMigrations | quote }}
- name: GOOGLE_CLIENT_ID
value: {{ .Values.globals.google.clientId | quote }}
- name: GOOGLE_CLIENT_SECRET
value: {{ .Values.globals.google.secret | quote }}
- name: AUTOMATION_MAX_ITERATIONS
value: {{ .Values.globals.automationMaxIterations | quote }}
- name: TENANT_FEATURE_FLAGS
value: {{ .Values.globals.tenantFeatureFlags | quote }}
- name: ENCRYPTION_KEY
value: {{ .Values.globals.bbEncryptionKey | quote }}
{{ if .Values.globals.bbAdminUserEmail }}
- name: BB_ADMIN_USER_EMAIL
value: {{ .Values.globals.bbAdminUserEmail | quote }}
{{ end }}
{{ if .Values.globals.bbAdminUserPassword }}
- name: BB_ADMIN_USER_PASSWORD
value: {{ .Values.globals.bbAdminUserPassword | quote }}
{{ end }}
{{ if .Values.globals.pluginsDir }}
- name: PLUGINS_DIR
value: {{ .Values.globals.pluginsDir | quote }}
{{ end }}
{{ if .Values.services.automationWorkers.nodeDebug }}
- name: NODE_DEBUG
value: {{ .Values.services.automationWorkers.nodeDebug | quote }}
{{ end }}
{{ if .Values.globals.datadogApmEnabled }}
- name: DD_LOGS_INJECTION
value: {{ .Values.globals.datadogApmEnabled | quote }}
- name: DD_APM_ENABLED
value: {{ .Values.globals.datadogApmEnabled | quote }}
- name: DD_APM_DD_URL
value: https://trace.agent.datadoghq.eu
{{ end }}
{{ if .Values.globals.globalAgentHttpProxy }}
- name: GLOBAL_AGENT_HTTP_PROXY
value: {{ .Values.globals.globalAgentHttpProxy | quote }}
{{ end }}
{{ if .Values.globals.globalAgentHttpsProxy }}
- name: GLOBAL_AGENT_HTTPS_PROXY
value: {{ .Values.globals.globalAgentHttpsProxy | quote }}
{{ end }}
{{ if .Values.globals.globalAgentNoProxy }}
- name: GLOBAL_AGENT_NO_PROXY
value: {{ .Values.globals.globalAgentNoProxy | quote }}
{{ end }}
{{ if .Values.services.tlsRejectUnauthorized }}
- name: NODE_TLS_REJECT_UNAUTHORIZED
value: {{ .Values.services.tlsRejectUnauthorized }}
{{ end }}
- name: APP_FEATURES
value: "automations"
{{- range .Values.services.automationWorkers.extraEnv }}
- name: {{ .name }}
value: {{ .value | quote }}
{{- end }}
image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
imagePullPolicy: Always
{{- if .Values.services.automationWorkers.startupProbe }}
{{- with .Values.services.automationWorkers.startupProbe }}
startupProbe:
{{- toYaml . | nindent 10 }}
{{- end }}
{{- end }}
{{- if .Values.services.automationWorkers.livenessProbe }}
{{- with .Values.services.automationWorkers.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 10 }}
{{- end }}
{{- end }}
{{- if .Values.services.automationWorkers.readinessProbe }}
{{- with .Values.services.automationWorkers.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 10 }}
{{- end }}
{{- end }}
name: bbautomationworker
ports:
- containerPort: {{ .Values.services.automationWorkers.port }}
{{ with .Values.services.automationWorkers.resources }}
resources:
{{- toYaml . | nindent 10 }}
{{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{ if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName | quote }}
{{ end }}
{{ if .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml .Values.imagePullSecrets | nindent 6 }}
{{ end }}
restartPolicy: Always
serviceAccountName: ""
status: {}
{{- end }}

32
charts/budibase/templates/automation-worker-service-hpa.yaml Normal file
View file

@ -0,0 +1,32 @@
{{- if .Values.services.automationWorkers.autoscaling.enabled }}
apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "budibase.fullname" . }}-apps
labels:
{{- include "budibase.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: automation-worker-service
minReplicas: {{ .Values.services.automationWorkers.autoscaling.minReplicas }}
maxReplicas: {{ .Values.services.automationWorkers.autoscaling.maxReplicas }}
metrics:
{{- if .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

4
charts/budibase/templates/worker-service-deployment.yaml
View file

@ -182,6 +182,10 @@ spec:
- name: NODE_TLS_REJECT_UNAUTHORIZED
value: {{ .Values.services.tlsRejectUnauthorized }}
{{ end }}
{{- range .Values.services.worker.extraEnv }}
- name: {{ .name }}
value: {{ .value | quote }}
{{- end }}
image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
imagePullPolicy: Always
{{- if .Values.services.worker.startupProbe }}

78
charts/budibase/values.yaml
View file

@ -220,6 +220,9 @@ services:
# <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
# for more information on how to set these.
resources: {}
# -- Extra environment variables to set for apps pods. Takes a list of
# name=value pairs.
extraEnv: []
# -- Startup probe configuration for apps pods. You shouldn't need to
# change this, but if you want to you can find more information here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
@ -272,6 +275,78 @@ services:
# and resources set for the apps pods.
targetCPUUtilizationPercentage: 80
automationWorkers:
# -- Whether or not to enable the automation worker service. If you disable this,
# automations will be processed by the apps service.
enabled: true
# @ignore (you shouldn't need to change this)
port: 4002
# -- The number of automation worker replicas to run.
replicaCount: 1
# -- The log level for the automation worker service.
logLevel: info
# -- The resources to use for automation worker pods. See
# <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
# for more information on how to set these.
resources: {}
# -- Extra environment variables to set for automation worker pods. Takes a list of
# name=value pairs.
extraEnv: []
# -- Startup probe configuration for automation worker pods. You shouldn't
# need to change this, but if you want to you can find more information
# here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
# @default -- HTTP health checks.
startupProbe:
# @ignore
httpGet:
path: /health
port: 4002
scheme: HTTP
# @ignore
failureThreshold: 30
# @ignore
periodSeconds: 3
# -- Readiness probe configuration for automation worker pods. You shouldn't
# need to change this, but if you want to you can find more information
# here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
# @default -- HTTP health checks.
readinessProbe:
# @ignore
httpGet:
path: /health
port: 4002
scheme: HTTP
# @ignore
periodSeconds: 3
# @ignore
failureThreshold: 1
# -- Liveness probe configuration for automation worker pods. You shouldn't
# need to change this, but if you want to you can find more information
# here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
# @default -- HTTP health checks.
livenessProbe:
# @ignore
httpGet:
path: /health
port: 4002
scheme: HTTP
# @ignore
failureThreshold: 3
# @ignore
periodSeconds: 30
autoscaling:
# -- Whether to enable horizontal pod autoscaling for the apps service.
enabled: false
minReplicas: 1
maxReplicas: 10
# -- Target CPU utilization percentage for the automation worker service.
# Note that for autoscaling to work, you will need to have metrics-server
# configured, and resources set for the automation worker pods.
targetCPUUtilizationPercentage: 80
worker:
# @ignore (you shouldn't need to change this)
port: 4003
@ -285,6 +360,9 @@ services:
# <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
# for more information on how to set these.
resources: {}
# -- Extra environment variables to set for worker pods. Takes a list of
# name=value pairs.
extraEnv: []
# -- Startup probe configuration for worker pods. You shouldn't need to
# change this, but if you want to you can find more information here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>

1
hosting/proxy/nginx.prod.conf
View file

@ -257,6 +257,7 @@ http {
access_log off;
allow 127.0.0.1;
allow 10.0.0.0/8;
deny all;
location /nginx_status {

4
lerna.json
View file

@ -1,5 +1,5 @@
{
"version": "2.13.37",
"version": "2.13.46",
"npmClient": "yarn",
"packages": [
"packages/*",
@ -22,4 +22,4 @@
"loadEnvFiles": false
}
}
}
}

2
packages/account-portal

@ -1 +1 @@
Subproject commit a0b13270c36dd188e2a953d026b4560a1208008e
Subproject commit 09dae295e3ba6149c4e1d7fe567870c3a38bd277

13
packages/backend-core/src/cache/tests/writethrough.spec.ts vendored
View file

@ -1,15 +1,16 @@
import { DBTestConfiguration } from "../../../tests/extra"
import {
structures,
expectFunctionWasCalledTimesWith,
mocks,
} from "../../../tests"
import { structures } from "../../../tests"
import { Writethrough } from "../writethrough"
import { getDB } from "../../db"
import { Document } from "@budibase/types"
import tk from "timekeeper"
tk.freeze(Date.now())
interface ValueDoc extends Document {
value: any
}
const DELAY = 5000
describe("writethrough", () => {
@ -117,7 +118,7 @@ describe("writethrough", () => {
describe("get", () => {
it("should be able to retrieve", async () => {
await config.doInTenant(async () => {
const response = await writethrough.get(docId)
const response = await writethrough.get<ValueDoc>(docId)
expect(response.value).toBe(4)
})
})

26
packages/backend-core/src/cache/writethrough.ts vendored
View file

@ -7,7 +7,7 @@ import * as locks from "../redis/redlockImpl"
const DEFAULT_WRITE_RATE_MS = 10000
let CACHE: BaseCache | null = null
interface CacheItem {
interface CacheItem<T extends Document> {
doc: any
lastWrite: number
}
@ -24,7 +24,10 @@ function makeCacheKey(db: Database, key: string) {
return db.name + key
}
function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
function makeCacheItem<T extends Document>(
doc: T,
lastWrite: number | null = null
): CacheItem<T> {
return { doc, lastWrite: lastWrite || Date.now() }
}
@ -35,7 +38,7 @@ async function put(
) {
const cache = await getCache()
const key = doc._id
let cacheItem: CacheItem | undefined
let cacheItem: CacheItem<any> | undefined
if (key) {
cacheItem = await cache.get(makeCacheKey(db, key))
}
@ -53,11 +56,8 @@ async function put(
const writeDb = async (toWrite: any) => {
// doc should contain the _id and _rev
const response = await db.put(toWrite, { force: true })
output = {
...doc,
_id: response.id,
_rev: response.rev,
}
output._id = response.id
output._rev = response.rev
}
try {
await writeDb(doc)
@ -84,12 +84,12 @@ async function put(
return { ok: true, id: output._id, rev: output._rev }
}
async function get(db: Database, id: string): Promise<any> {
async function get<T extends Document>(db: Database, id: string): Promise<T> {
const cache = await getCache()
const cacheKey = makeCacheKey(db, id)
let cacheItem: CacheItem = await cache.get(cacheKey)
let cacheItem: CacheItem<T> = await cache.get(cacheKey)
if (!cacheItem) {
const doc = await db.get(id)
const doc = await db.get<T>(id)
cacheItem = makeCacheItem(doc)
await cache.store(cacheKey, cacheItem)
}
@ -123,8 +123,8 @@ export class Writethrough {
return put(this.db, doc, writeRateMs)
}
async get(id: string) {
return get(this.db, id)
async get<T extends Document>(id: string) {
return get<T>(this.db, id)
}
async remove(docOrId: any, rev?: any) {

19
packages/backend-core/src/constants/misc.ts
View file

@ -11,24 +11,7 @@ export enum Cookie {
OIDC_CONFIG = "budibase:oidc:config",
}
export enum Header {
API_KEY = "x-budibase-api-key",
LICENSE_KEY = "x-budibase-license-key",
API_VER = "x-budibase-api-version",
APP_ID = "x-budibase-app-id",
SESSION_ID = "x-budibase-session-id",
TYPE = "x-budibase-type",
PREVIEW_ROLE = "x-budibase-role",
TENANT_ID = "x-budibase-tenant-id",
VERIFICATION_CODE = "x-budibase-verification-code",
RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
RESET_PASSWORD_CODE = "x-budibase-reset-password-code",
RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code",
TOKEN = "x-budibase-token",
CSRF_TOKEN = "x-csrf-token",
CORRELATION_ID = "x-budibase-correlation-id",
AUTHORIZATION = "authorization",
}
export { Header } from "@budibase/shared-core"
export enum GlobalRole {
OWNER = "owner",

1
packages/backend-core/src/environment.ts
View file

@ -107,6 +107,7 @@ const environment = {
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
API_ENCRYPTION_KEY: getAPIEncryptionKey(),
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

4
packages/backend-core/src/queue/inMemoryQueue.ts
View file

@ -68,6 +68,10 @@ class InMemoryQueue {
})
}
async isReady() {
return true
}
// simply puts a message to the queue and emits to the queue for processing
/**
* Simple function to replicate the add message functionality of Bull, putting

1
packages/backend-core/src/redis/redlockImpl.ts
View file

@ -137,7 +137,6 @@ export async function doWithLock<T>(
const result = await task()
return { executed: true, result }
} catch (e: any) {
logWarn(`lock type: ${opts.type} error`, e)
// lock limit exceeded
if (e.name === "LockError") {
if (opts.type === LockType.TRY_ONCE) {

11
packages/backend-core/src/utils/utils.ts
View file

@ -96,7 +96,7 @@ export async function getAppIdFromCtx(ctx: Ctx) {
}
// look in the path
const pathId = parseAppIdFromUrl(ctx.path)
const pathId = parseAppIdFromUrlPath(ctx.path)
if (!appId && pathId) {
appId = confirmAppId(pathId)
}
@ -116,18 +116,21 @@ export async function getAppIdFromCtx(ctx: Ctx) {
// referer header is present from a builder redirect
const referer = ctx.request.headers.referer
if (!appId && referer?.includes(BUILDER_APP_PREFIX)) {
const refererId = parseAppIdFromUrl(ctx.request.headers.referer)
const refererId = parseAppIdFromUrlPath(ctx.request.headers.referer)
appId = confirmAppId(refererId)
}
return appId
}
function parseAppIdFromUrl(url?: string) {
function parseAppIdFromUrlPath(url?: string) {
if (!url) {
return
}
return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX))
return url
.split("?")[0] // Remove any possible query string
.split("/")
.find(subPath => subPath.startsWith(APP_PREFIX))
}
/**

1
packages/builder/src/components/common/bindings/BindingPicker.svelte
View file

@ -79,6 +79,7 @@
bind:this={popover}
anchor={popoverAnchor}
maxWidth={300}
maxHeight={300}
dismissible={false}
>
<Layout gap="S">

2
packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte
View file

@ -257,7 +257,7 @@
<LockedFeature
title={"Audit Logs"}
planType={"Business plan"}
planType={"Enterprise plan"}
description={"View all events that have occurred in your Budibase installation"}
enabled={$licensing.auditLogsEnabled}
upgradeButtonClick={async () => {

10
packages/builder/src/pages/builder/portal/account/usage.svelte
View file

@ -15,6 +15,7 @@
import { DashCard, Usage } from "components/usage"
import { PlanModel } from "constants"
import { sdk } from "@budibase/shared-core"
import { PlanType } from "@budibase/types"
let staticUsage = []
let monthlyUsage = []
@ -106,7 +107,14 @@
}
const planTitle = () => {
return `${capitalise(license?.plan.type)} Plan`
const planType = license?.plan.type
let planName = license?.plan.type
if (planType === PlanType.PREMIUM_PLUS) {
planName = "Premium"
} else if (planType === PlanType.ENTERPRISE_BASIC) {
planName = "Enterprise"
}
return `${capitalise(planName)} Plan`
}
const getDaysRemaining = timestamp => {

2
packages/builder/src/pages/builder/portal/settings/auth/index.svelte
View file

@ -283,7 +283,7 @@
</div>
{#if !$licensing.enforceableSSO}
<Tags>
<Tag icon="LockClosed">Enterprise</Tag>
<Tag icon="LockClosed">Enterprise plan</Tag>
</Tags>
{/if}
</div>

2
packages/builder/src/pages/builder/portal/settings/environment/index.svelte
View file

@ -59,7 +59,7 @@
<LockedFeature
title={"Environment Variables"}
planType={"Business plan"}
planType={"Enterprise plan"}
description={"Add and manage environment variables for development and production"}
enabled={$licensing.environmentVariablesEnabled}
upgradeButtonClick={async () => {

26
packages/frontend-core/src/api/index.js
View file

@ -1,4 +1,5 @@
import { Helpers } from "@budibase/bbui"
import { Header } from "@budibase/shared-core"
import { ApiVersion } from "../constants"
import { buildAnalyticsEndpoints } from "./analytics"
import { buildAppEndpoints } from "./app"
@ -62,6 +63,11 @@ const defaultAPIClientConfig = {
* invoked before the actual JS error is thrown up the stack.
*/
onError: null,
/**
* A function can be passed to be called when an API call returns info about a migration running for a specific app
*/
onMigrationDetected: null,
}
/**
@ -133,9 +139,9 @@ export const createAPIClient = config => {
// Build headers
let headers = { Accept: "application/json" }
headers["x-budibase-session-id"] = APISessionID
headers[Header.SESSION_ID] = APISessionID
if (!external) {
headers["x-budibase-api-version"] = ApiVersion
headers[Header.API_VER] = ApiVersion
}
if (json) {
headers["Content-Type"] = "application/json"
@ -170,6 +176,7 @@ export const createAPIClient = config => {
// Handle response
if (response.status >= 200 && response.status < 400) {
handleMigrations(response)
try {
if (parseResponse) {
return await parseResponse(response)
@ -186,7 +193,18 @@ export const createAPIClient = config => {
}
}
// Performs an API call to the server and caches the response.
const handleMigrations = response => {
if (!config.onMigrationDetected) {
return
}
const migration = response.headers.get(Header.MIGRATING_APP)
if (migration) {
config.onMigrationDetected(migration)
}
}
// Performs an API call to the server and caches the response.
// Future invocation for this URL will return the cached result instead of
// hitting the server again.
const makeCachedApiCall = async params => {
@ -242,7 +260,7 @@ export const createAPIClient = config => {
getAppID: () => {
let headers = {}
config?.attachHeaders(headers)
return headers?.["x-budibase-app-id"]
return headers?.[Header.APP_ID]
},
}

2
packages/pro

@ -1 +1 @@
Subproject commit 056c2093dbc93d9a10ea9f5050c84a84edd8100c
Subproject commit 992486c10044a7495496b97bdf5f454d4020bfba

4
packages/server/Dockerfile
View file

@ -68,9 +68,13 @@ COPY packages/server/builder/ builder/
COPY packages/server/client/ client/
ARG BUDIBASE_VERSION
ARG GIT_COMMIT_SHA
# Ensuring the version argument is sent
RUN test -n "$BUDIBASE_VERSION"
ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
ENV DD_GIT_REPOSITORY_URL=https://github.com/budibase/budibase
ENV DD_GIT_COMMIT_SHA=$GIT_COMMIT_SHA
ENV DD_VERSION=$BUDIBASE_VERSION
EXPOSE 4001

2
packages/server/package.json
View file

@ -65,7 +65,7 @@
"cookies": "0.8.0",
"csvtojson": "2.0.10",
"curlconverter": "3.21.0",
"dd-trace": "3.13.2",
"dd-trace": "4.20.0",
"dotenv": "8.2.0",
"form-data": "4.0.0",
"global-agent": "3.0.0",

75
packages/server/scripts/dev/manage.js
View file

@ -1,7 +1,8 @@
#!/usr/bin/env node
const compose = require("docker-compose")
const path = require("path")
const fs = require("fs")
const { parsed: existingConfig } = require("dotenv").config()
const updateDotEnv = require("update-dotenv")
// This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
const CONFIG = {
@ -17,45 +18,41 @@ const Commands = {
}
async function init() {
const envFilePath = path.join(process.cwd(), ".env")
if (!fs.existsSync(envFilePath)) {
const envFileJson = {
PORT: 4001,
MINIO_URL: "http://localhost:4004",
COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
REDIS_URL: "localhost:6379",
WORKER_URL: "http://localhost:4002",
INTERNAL_API_KEY: "budibase",
ACCOUNT_PORTAL_URL: "http://localhost:10001",
ACCOUNT_PORTAL_API_KEY: "budibase",
PLATFORM_URL: "http://localhost:10000",
JWT_SECRET: "testsecret",
ENCRYPTION_KEY: "testsecret",
REDIS_PASSWORD: "budibase",
MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase",
COUCH_DB_PASSWORD: "budibase",
COUCH_DB_USER: "budibase",
SELF_HOSTED: 1,
DISABLE_ACCOUNT_PORTAL: 1,
MULTI_TENANCY: "",
DISABLE_THREADING: 1,
SERVICE: "app-service",
DEPLOYMENT_ENVIRONMENT: "development",
BB_ADMIN_USER_EMAIL: "",
BB_ADMIN_USER_PASSWORD: "",
PLUGINS_DIR: "",
TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
HTTP_MIGRATIONS: "0",
HTTP_LOGGING: "0",
VERSION: "0.0.0+local",
}
let envFile = ""
Object.keys(envFileJson).forEach(key => {
envFile += `${key}=${envFileJson[key]}\n`
})
fs.writeFileSync(envFilePath, envFile)
let config = {
PORT: "4001",
MINIO_URL: "http://localhost:4004",
COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
REDIS_URL: "localhost:6379",
WORKER_URL: "http://localhost:4002",
INTERNAL_API_KEY: "budibase",
ACCOUNT_PORTAL_URL: "http://localhost:10001",
ACCOUNT_PORTAL_API_KEY: "budibase",
PLATFORM_URL: "http://localhost:10000",
JWT_SECRET: "testsecret",
ENCRYPTION_KEY: "testsecret",
REDIS_PASSWORD: "budibase",
MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase",
COUCH_DB_PASSWORD: "budibase",
COUCH_DB_USER: "budibase",
SELF_HOSTED: "1",
DISABLE_ACCOUNT_PORTAL: "1",
MULTI_TENANCY: "",
DISABLE_THREADING: "1",
SERVICE: "app-service",
DEPLOYMENT_ENVIRONMENT: "development",
BB_ADMIN_USER_EMAIL: "",
BB_ADMIN_USER_PASSWORD: "",
PLUGINS_DIR: "",
TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
HTTP_MIGRATIONS: "0",
HTTP_LOGGING: "0",
VERSION: "0.0.0+local",
}
config = { ...config, ...existingConfig }
await updateDotEnv(config)
}
async function up() {

2
packages/server/src/api/controllers/application.ts
View file

@ -340,7 +340,7 @@ async function performAppCreate(ctx: UserCtx) {
// Initialise the app migration version as the latest one
await appMigrations.updateAppMigrationMetadata({
appId,
version: appMigrations.latestMigration,
version: appMigrations.getLatestMigrationId(),
})
await cache.app.invalidateAppMetadata(appId, newApplication)

26
packages/server/src/api/controllers/migrations.ts
View file

@ -1,14 +1,34 @@
import { context } from "@budibase/backend-core"
import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
import { BBContext } from "@budibase/types"
import { Ctx } from "@budibase/types"
import {
getAppMigrationVersion,
getLatestMigrationId,
} from "../../appMigrations"
export async function migrate(ctx: BBContext) {
export async function migrate(ctx: Ctx) {
const options = ctx.request.body
// don't await as can take a while, just return
migrationImpl(options)
ctx.status = 200
}
export async function fetchDefinitions(ctx: BBContext) {
export async function fetchDefinitions(ctx: Ctx) {
ctx.body = MIGRATIONS
ctx.status = 200
}
export async function getMigrationStatus(ctx: Ctx) {
const appId = context.getAppId()
if (!appId) {
ctx.throw("AppId could not be found")
}
const latestAppliedMigration = await getAppMigrationVersion(appId)
const migrated = latestAppliedMigration === getLatestMigrationId()
ctx.body = { migrated }
ctx.status = 200
}

105
packages/server/src/api/index.ts
View file

@ -4,62 +4,75 @@ import currentApp from "../middleware/currentapp"
import zlib from "zlib"
import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
import { middleware as pro } from "@budibase/pro"
import { apiEnabled, automationsEnabled } from "../features"
import migrations from "../middleware/appMigrations"
import { automationQueue } from "../automations"
export { shutdown } from "./routes/public"
const compress = require("koa-compress")
export const router: Router = new Router()
router.get("/health", ctx => (ctx.status = 200))
router.get("/health", async ctx => {
if (automationsEnabled()) {
if (!(await automationQueue.isReady())) {
ctx.status = 503
return
}
}
ctx.status = 200
})
router.get("/version", ctx => (ctx.body = envCore.VERSION))
router.use(middleware.errorHandling)
router
.use(
compress({
threshold: 2048,
gzip: {
flush: zlib.constants.Z_SYNC_FLUSH,
},
deflate: {
flush: zlib.constants.Z_SYNC_FLUSH,
},
br: false,
})
)
// re-direct before any middlewares occur
.redirect("/", "/builder")
.use(
auth.buildAuthMiddleware([], {
publicAllowed: true,
})
)
// nothing in the server should allow query string tenants
// the server can be public anywhere, so nowhere should throw errors
// if the tenancy has not been set, it'll have to be discovered at application layer
.use(
auth.buildTenancyMiddleware([], [], {
noTenancyRequired: true,
})
)
.use(pro.licensing())
// @ts-ignore
.use(currentApp)
.use(auth.auditLog)
// @ts-ignore
.use(migrations)
// only add the routes if they are enabled
if (apiEnabled()) {
router
.use(
compress({
threshold: 2048,
gzip: {
flush: zlib.constants.Z_SYNC_FLUSH,
},
deflate: {
flush: zlib.constants.Z_SYNC_FLUSH,
},
br: false,
})
)
// re-direct before any middlewares occur
.redirect("/", "/builder")
.use(
auth.buildAuthMiddleware([], {
publicAllowed: true,
})
)
// nothing in the server should allow query string tenants
// the server can be public anywhere, so nowhere should throw errors
// if the tenancy has not been set, it'll have to be discovered at application layer
.use(
auth.buildTenancyMiddleware([], [], {
noTenancyRequired: true,
})
)
.use(pro.licensing())
// @ts-ignore
.use(currentApp)
.use(auth.auditLog)
// @ts-ignore
.use(migrations)
// authenticated routes
for (let route of mainRoutes) {
router.use(route.routes())
router.use(route.allowedMethods())
// authenticated routes
for (let route of mainRoutes) {
router.use(route.routes())
router.use(route.allowedMethods())
}
router.use(publicRoutes.routes())
router.use(publicRoutes.allowedMethods())
// WARNING - static routes will catch everything else after them this must be last
router.use(staticRoutes.routes())
router.use(staticRoutes.allowedMethods())
}
router.use(publicRoutes.routes())
router.use(publicRoutes.allowedMethods())
// WARNING - static routes will catch everything else after them this must be last
router.use(staticRoutes.routes())
router.use(staticRoutes.allowedMethods())

2
packages/server/src/api/routes/migrations.ts
View file

@ -11,4 +11,6 @@ router
auth.internalApi,
migrationsController.fetchDefinitions
)
.get("/api/migrations/status", migrationsController.getMigrationStatus)
export default router

10
packages/server/src/app.ts
View file

@ -9,7 +9,6 @@ import { ServiceType } from "@budibase/types"
import { env as coreEnv } from "@budibase/backend-core"
coreEnv._set("SERVICE_TYPE", ServiceType.APPS)
import { apiEnabled } from "./features"
import createKoaApp from "./koa"
import Koa from "koa"
import { Server } from "http"
@ -18,12 +17,9 @@ import { startup } from "./startup"
let app: Koa, server: Server
async function start() {
// if API disabled, could run automations instead
if (apiEnabled()) {
const koa = createKoaApp()
app = koa.app
server = koa.server
}
const koa = createKoaApp()
app = koa.app
server = koa.server
// startup includes automation runner - if enabled
await startup(app, server)
}

21
packages/server/src/appMigrations/index.ts
View file

@ -1,6 +1,9 @@
import queue from "./queue"
import { Next } from "koa"
import { getAppMigrationVersion } from "./appMigrationMetadata"
import { MIGRATIONS } from "./migrations"
import { UserCtx } from "@budibase/types"
import { Header } from "@budibase/backend-core"
export * from "./appMigrationMetadata"
@ -9,14 +12,20 @@ export type AppMigration = {
func: () => Promise<void>
}
export const latestMigration = MIGRATIONS.map(m => m.id)
.sort()
.reverse()[0]
export const getLatestMigrationId = () =>
MIGRATIONS.map(m => m.id)
.sort()
.reverse()[0]
const getTimestamp = (versionId: string) => versionId?.split("_")[0]
export async function checkMissingMigrations(appId: string) {
export async function checkMissingMigrations(
ctx: UserCtx,
next: Next,
appId: string
) {
const currentVersion = await getAppMigrationVersion(appId)
const latestMigration = getLatestMigrationId()
if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) {
await queue.add(
@ -29,5 +38,9 @@ export async function checkMissingMigrations(appId: string) {
removeOnFail: true,
}
)
ctx.response.set(Header.MIGRATING_APP, appId)
}
return next()
}

25
packages/server/src/appMigrations/tests/migrations.integrity.spec.ts Normal file
View file

@ -0,0 +1,25 @@
import { context } from "@budibase/backend-core"
import * as setup from "../../api/routes/tests/utilities"
import * as migrations from "../migrations"
describe("migration integrity", () => {
// These test is checking that each migration is "idempotent".
// We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
it("each migration can rerun safely", async () => {
const config = setup.getConfig()
await config.init()
await config.doInContext(config.getAppId(), async () => {
const db = context.getAppDB()
for (const migration of migrations.MIGRATIONS) {
await migration.func()
const docs = await db.allDocs({ include_docs: true })
await migration.func()
const latestDocs = await db.allDocs({ include_docs: true })
expect(docs).toEqual(latestDocs)
}
})
})
})

58
packages/server/src/appMigrations/tests/migrations.spec.ts
View file

@ -1,25 +1,53 @@
import { context } from "@budibase/backend-core"
import { Header } from "@budibase/backend-core"
import * as setup from "../../api/routes/tests/utilities"
import { MIGRATIONS } from "../migrations"
import * as migrations from "../migrations"
import { getAppMigrationVersion } from "../appMigrationMetadata"
describe("migration", () => {
// These test is checking that each migration is "idempotent".
// We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
it("each migration can rerun safely", async () => {
jest.mock<typeof migrations>("../migrations", () => ({
MIGRATIONS: [
{
id: "20231211101320_test",
func: async () => {},
},
],
}))
describe("migrations", () => {
it("new apps are created with the latest app migration version set", async () => {
const config = setup.getConfig()
await config.init()
await config.doInContext(config.getAppId(), async () => {
const db = context.getAppDB()
for (const migration of MIGRATIONS) {
await migration.func()
const docs = await db.allDocs({ include_docs: true })
const migrationVersion = await getAppMigrationVersion(config.getAppId())
await migration.func()
const latestDocs = await db.allDocs({ include_docs: true })
expect(docs).toEqual(latestDocs)
}
expect(migrationVersion).toEqual("20231211101320_test")
})
})
it("accessing an app that has no pending migrations will not attach the migrating header", async () => {
const config = setup.getConfig()
await config.init()
const appId = config.getAppId()
const response = await config.api.application.getRaw(appId)
expect(response.headers[Header.MIGRATING_APP]).toBeUndefined()
})
it("accessing an app that has pending migrations will attach the migrating header", async () => {
const config = setup.getConfig()
await config.init()
const appId = config.getAppId()
migrations.MIGRATIONS.push({
id: "20231211105812_new-test",
func: async () => {},
})
const response = await config.api.application.getRaw(appId)
expect(response.headers[Header.MIGRATING_APP]).toEqual(appId)
})
})

12
packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts
View file

@ -4,12 +4,14 @@ import { getAppMigrationVersion } from "../appMigrationMetadata"
import { context } from "@budibase/backend-core"
import { AppMigration } from ".."
const futureTimestamp = `20500101174029`
describe("migrationsProcessor", () => {
it("running migrations will update the latest applied migration", async () => {
const testMigrations: AppMigration[] = [
{ id: "123", func: async () => {} },
{ id: "124", func: async () => {} },
{ id: "125", func: async () => {} },
{ id: `${futureTimestamp}_123`, func: async () => {} },
{ id: `${futureTimestamp}_124`, func: async () => {} },
{ id: `${futureTimestamp}_125`, func: async () => {} },
]
const config = setup.getConfig()
@ -23,13 +25,13 @@ describe("migrationsProcessor", () => {
expect(
await config.doInContext(appId, () => getAppMigrationVersion(appId))
).toBe("125")
).toBe(`${futureTimestamp}_125`)
})
it("no context can be initialised within a migration", async () => {
const testMigrations: AppMigration[] = [
{
id: "123",
id: `${futureTimestamp}_123`,
func: async () => {
await context.doInAppMigrationContext("any", () => {})
},

18
packages/server/src/db/linkedRows/linkUtils.ts
View file

@ -56,7 +56,7 @@ export async function getLinkDocuments(args: {
try {
let linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
// filter to get unique entries
const foundIds: string[] = []
const foundIds = new Set()
linkRows = linkRows.filter(link => {
// make sure anything unique is the correct key
if (
@ -65,9 +65,9 @@ export async function getLinkDocuments(args: {
) {
return false
}
const unique = foundIds.indexOf(link.id) === -1
const unique = !foundIds.has(link.id)
if (unique) {
foundIds.push(link.id)
foundIds.add(link.id)
}
return unique
})
@ -99,9 +99,15 @@ export async function getLinkDocuments(args: {
}
export function getUniqueByProp(array: any[], prop: string) {
return array.filter((obj, pos, arr) => {
return arr.map(mapObj => mapObj[prop]).indexOf(obj[prop]) === pos
})
const seen = new Set()
const filteredArray = []
for (const item of array) {
if (!seen.has(item[prop])) {
seen.add(item[prop])
filteredArray.push(item)
}
}
return filteredArray
}
export function getLinkedTableIDs(table: Table): string[] {

7
packages/server/src/features.ts
View file

@ -22,3 +22,10 @@ export function automationsEnabled() {
export function apiEnabled() {
return featureList.includes(AppFeature.API)
}
export function printFeatures() {
if (!env.APP_FEATURES) {
return
}
console.log(`**** APP FEATURES SET: ${featureList.join(", ")} ****`)
}

4
packages/server/src/middleware/appMigrations.ts
View file

@ -8,7 +8,5 @@ export default async (ctx: UserCtx, next: any) => {
return next()
}
await checkMissingMigrations(appId)
return next()
return checkMissingMigrations(ctx, next, appId)
}

6
packages/server/src/middleware/currentapp.ts
View file

@ -12,6 +12,7 @@ import { getCachedSelf } from "../utilities/global"
import env from "../environment"
import { isWebhookEndpoint } from "./utils"
import { UserCtx, ContextUser } from "@budibase/types"
import tracer from "dd-trace"
export default async (ctx: UserCtx, next: any) => {
// try to get the appID from the request
@ -20,6 +21,11 @@ export default async (ctx: UserCtx, next: any) => {
return next()
}
if (requestAppId) {
const span = tracer.scope().active()
span?.addTags({ app_id: requestAppId })
}
// deny access to application preview
if (!env.isTest()) {
if (

21
packages/server/src/startup.ts
View file

@ -19,11 +19,14 @@ import * as pro from "@budibase/pro"
import * as api from "./api"
import sdk from "./sdk"
import { initialise as initialiseWebsockets } from "./websockets"
import { automationsEnabled } from "./features"
import { automationsEnabled, printFeatures } from "./features"
import Koa from "koa"
import { Server } from "http"
import { AddressInfo } from "net"
let STARTUP_RAN = false
async function initRoutes(app: any) {
async function initRoutes(app: Koa) {
if (!env.isTest()) {
const plugin = await bullboard.init()
app.use(plugin)
@ -48,27 +51,31 @@ async function initPro() {
})
}
function shutdown(server?: any) {
function shutdown(server?: Server) {
if (server) {
server.close()
server.destroy()
}
}
export async function startup(app?: any, server?: any) {
export async function startup(app?: Koa, server?: Server) {
if (STARTUP_RAN) {
return
}
printFeatures()
STARTUP_RAN = true
if (server && !env.CLUSTER_MODE) {
if (app && server && !env.CLUSTER_MODE) {
console.log(`Budibase running on ${JSON.stringify(server.address())}`)
env._set("PORT", server.address().port)
const address = server.address() as AddressInfo
env._set("PORT", address.port)
}
eventEmitter.emitPort(env.PORT)
fileSystem.init()
await redis.init()
eventInit()
initialiseWebsockets(app, server)
if (app && server) {
initialiseWebsockets(app, server)
}
// run migrations on startup if not done via http
// not recommended in a clustered environment

1
packages/server/src/tests/utilities/TestConfiguration.ts
View file

@ -17,7 +17,6 @@ import {
basicWebhook,
} from "./structures"
import {
auth,
cache,
constants,
context,

8
packages/server/src/tests/utilities/api/application.ts
View file

@ -1,3 +1,4 @@
import { Response } from "supertest"
import { App } from "@budibase/types"
import TestConfiguration from "../TestConfiguration"
import { TestAPI } from "./base"
@ -7,12 +8,17 @@ export class ApplicationAPI extends TestAPI {
super(config)
}
get = async (appId: string): Promise<App> => {
getRaw = async (appId: string): Promise<Response> => {
const result = await this.request
.get(`/api/applications/${appId}/appPackage`)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
return result
}
get = async (appId: string): Promise<App> => {
const result = await this.getRaw(appId)
return result.body.application as App
}
}

54
packages/server/src/utilities/rowProcessor/utils.ts
View file

@ -11,6 +11,7 @@ import {
Row,
Table,
} from "@budibase/types"
import tracer from "dd-trace"
interface FormulaOpts {
dynamic?: boolean
@ -50,33 +51,42 @@ export function processFormulas<T extends Row | Row[]>(
inputRows: T,
{ dynamic, contextRows }: FormulaOpts = { dynamic: true }
): T {
const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
if (rows)
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.FORMULA) {
continue
}
return tracer.trace("processFormulas", {}, span => {
const numRows = Array.isArray(inputRows) ? inputRows.length : 1
span?.addTags({ table_id: table._id, dynamic, numRows })
const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
if (rows) {
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.FORMULA) {
continue
}
const isStatic = schema.formulaType === FormulaTypes.STATIC
const isStatic = schema.formulaType === FormulaTypes.STATIC
if (
schema.formula == null ||
(dynamic && isStatic) ||
(!dynamic && !isStatic)
) {
continue
}
// iterate through rows and process formula
for (let i = 0; i < rows.length; i++) {
let row = rows[i]
let context = contextRows ? contextRows[i] : row
rows[i] = {
...row,
[column]: processStringSync(schema.formula, context),
if (
schema.formula == null ||
(dynamic && isStatic) ||
(!dynamic && !isStatic)
) {
continue
}
// iterate through rows and process formula
for (let i = 0; i < rows.length; i++) {
let row = rows[i]
let context = contextRows ? contextRows[i] : row
let formula = schema.formula
rows[i] = {
...row,
[column]: tracer.trace("processStringSync", {}, span => {
span?.addTags({ table_id: table._id, column, static: isStatic })
return processStringSync(formula, context)
}),
}
}
}
}
return Array.isArray(inputRows) ? rows : rows[0]
return Array.isArray(inputRows) ? rows : rows[0]
})
}
/**

19
packages/shared-core/src/constants/api.ts Normal file
View file

@ -0,0 +1,19 @@
export enum Header {
API_KEY = "x-budibase-api-key",
LICENSE_KEY = "x-budibase-license-key",
API_VER = "x-budibase-api-version",
APP_ID = "x-budibase-app-id",
SESSION_ID = "x-budibase-session-id",
TYPE = "x-budibase-type",
PREVIEW_ROLE = "x-budibase-role",
TENANT_ID = "x-budibase-tenant-id",
VERIFICATION_CODE = "x-budibase-verification-code",
RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
RESET_PASSWORD_CODE = "x-budibase-reset-password-code",
RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code",
TOKEN = "x-budibase-token",
CSRF_TOKEN = "x-csrf-token",
CORRELATION_ID = "x-budibase-correlation-id",
AUTHORIZATION = "authorization",
MIGRATING_APP = "x-budibase-migrating-app",
}

2
packages/shared-core/src/constants.ts → packages/shared-core/src/constants/index.ts
View file

@ -1,3 +1,5 @@
export * from "./api"
export const OperatorOptions = {
Equals: {
value: "equal",

8
packages/string-templates/manifest.json
View file

@ -1180,6 +1180,14 @@
"description": "<p>Stringify an object using <code>JSON.stringify</code>.</p>\n"
}
},
"uuid": {
"uuid": {
"args": [],
"numArgs": 0,
"example": "{{ uuid }} -> f34ebc66-93bd-4f7c-b79b-92b5569138bc",
"description": "<p>Generates a UUID, using the V4 method (identical to the browser crypto.randomUUID function).</p>\n"
}
},
"date": {
"date": {
"args": [

2
packages/string-templates/package.json
View file

@ -25,7 +25,7 @@
"manifest": "node ./scripts/gen-collection-info.js"
},
"dependencies": {
"@budibase/handlebars-helpers": "^0.11.9",
"@budibase/handlebars-helpers": "^0.11.11",
"dayjs": "^1.10.8",
"handlebars": "^4.7.6",
"lodash": "4.17.21",

1
packages/string-templates/scripts/gen-collection-info.js
View file

@ -20,6 +20,7 @@ const COLLECTIONS = [
"string",
"comparison",
"object",
"uuid",
]
const FILENAME = join(__dirname, "..", "manifest.json")
const outputJSON = {}

1
packages/string-templates/src/helpers/external.js
View file

@ -16,6 +16,7 @@ const EXTERNAL_FUNCTION_COLLECTIONS = [
"comparison",
"object",
"regex",
"uuid",
]
const ADDED_HELPERS = {

4
packages/string-templates/src/helpers/javascript.js
View file

@ -56,6 +56,10 @@ module.exports.processJS = (handlebars, context) => {
const res = { data: runJS(js, sandboxContext) }
return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}`
} catch (error) {
console.log(`JS error: ${typeof error} ${JSON.stringify(error)}`)
if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") {
return "Timed out while executing JS"
}
return "Error while executing JS"
}
}

2
packages/string-templates/test/constants.js Normal file
View file

@ -0,0 +1,2 @@
module.exports.UUID_REGEX =
/^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i

8
packages/string-templates/test/helpers.spec.js
View file

@ -1,6 +1,7 @@
const { processString, processObject, isValid } = require("../src/index.cjs")
const tableJson = require("./examples/table.json")
const dayjs = require("dayjs")
const { UUID_REGEX } = require("./constants")
describe("test the custom helpers we have applied", () => {
it("should be able to use the object helper", async () => {
@ -477,3 +478,10 @@ describe("Cover a few complex use cases", () => {
expect(output.dataProvider).toBe("%5B%221%22%2C%221%22%5D")
})
})
describe("uuid", () => {
it("should be able to generate a UUID", async () => {
const output = await processString("{{ uuid }}", {})
expect(output).toMatch(UUID_REGEX)
})
})

8
packages/string-templates/test/javascript.spec.js
View file

@ -1,4 +1,5 @@
const { processStringSync, encodeJSBinding } = require("../src/index.cjs")
const { UUID_REGEX } = require("./constants")
const processJS = (js, context) => {
return processStringSync(encodeJSBinding(js), context)
@ -114,7 +115,7 @@ describe("Test the JavaScript helper", () => {
it("should timeout after one second", () => {
const output = processJS(`while (true) {}`)
expect(output).toBe("Error while executing JS")
expect(output).toBe("Timed out while executing JS")
})
it("should prevent access to the process global", () => {
@ -140,4 +141,9 @@ describe("check JS helpers", () => {
const output = processJS(`return helpers.toInt(4.3)`)
expect(output).toBe(4)
})
it("should be able to use uuid", () => {
const output = processJS(`return helpers.uuid()`)
expect(output).toMatch(UUID_REGEX)
})
})

4
packages/worker/Dockerfile
View file

@ -51,8 +51,12 @@ ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
ARG BUDIBASE_VERSION
ARG GIT_COMMIT_SHA
# Ensuring the version argument is sent
RUN test -n "$BUDIBASE_VERSION"
ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
ENV DD_GIT_REPOSITORY_URL=https://github.com/budibase/budibase
ENV DD_GIT_COMMIT_SHA=$GIT_COMMIT_SHA
ENV DD_VERSION=$BUDIBASE_VERSION
CMD ["./docker_run.sh"]

2
packages/worker/package.json
View file

@ -48,7 +48,7 @@
"bcrypt": "5.1.0",
"bcryptjs": "2.4.3",
"bull": "4.10.1",
"dd-trace": "3.13.2",
"dd-trace": "4.20.0",
"dotenv": "8.6.0",
"global-agent": "3.0.0",
"ical-generator": "4.1.0",

70
packages/worker/scripts/dev/manage.js
View file

@ -1,44 +1,40 @@
#!/usr/bin/env node
const path = require("path")
const fs = require("fs")
const { parsed: existingConfig } = require("dotenv").config()
const updateDotEnv = require("update-dotenv")
async function init() {
const envFilePath = path.join(process.cwd(), ".env")
if (!fs.existsSync(envFilePath)) {
const envFileJson = {
SELF_HOSTED: 1,
PORT: 4002,
CLUSTER_PORT: 10000,
JWT_SECRET: "testsecret",
INTERNAL_API_KEY: "budibase",
MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase",
REDIS_URL: "localhost:6379",
REDIS_PASSWORD: "budibase",
MINIO_URL: "http://localhost:4004",
COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
COUCH_DB_USERNAME: "budibase",
COUCH_DB_PASSWORD: "budibase",
// empty string is false
MULTI_TENANCY: "",
DISABLE_ACCOUNT_PORTAL: 1,
ACCOUNT_PORTAL_URL: "http://localhost:10001",
ACCOUNT_PORTAL_API_KEY: "budibase",
PLATFORM_URL: "http://localhost:10000",
APPS_URL: "http://localhost:4001",
SERVICE: "worker-service",
DEPLOYMENT_ENVIRONMENT: "development",
TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
ENABLE_EMAIL_TEST_MODE: 1,
HTTP_LOGGING: 0,
VERSION: "0.0.0+local",
}
let envFile = ""
Object.keys(envFileJson).forEach(key => {
envFile += `${key}=${envFileJson[key]}\n`
})
fs.writeFileSync(envFilePath, envFile)
let config = {
SELF_HOSTED: "1",
PORT: "4002",
CLUSTER_PORT: "10000",
JWT_SECRET: "testsecret",
INTERNAL_API_KEY: "budibase",
MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase",
REDIS_URL: "localhost:6379",
REDIS_PASSWORD: "budibase",
MINIO_URL: "http://localhost:4004",
COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
COUCH_DB_USERNAME: "budibase",
COUCH_DB_PASSWORD: "budibase",
// empty string is false
MULTI_TENANCY: "",
DISABLE_ACCOUNT_PORTAL: "1",
ACCOUNT_PORTAL_URL: "http://localhost:10001",
ACCOUNT_PORTAL_API_KEY: "budibase",
PLATFORM_URL: "http://localhost:10000",
APPS_URL: "http://localhost:4001",
SERVICE: "worker-service",
DEPLOYMENT_ENVIRONMENT: "development",
TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
ENABLE_EMAIL_TEST_MODE: "1",
HTTP_LOGGING: "0",
VERSION: "0.0.0+local",
}
config = { ...config, ...existingConfig }
await updateDotEnv(config)
}
// if more than init required use this to determine the command type

25
packages/worker/src/api/controllers/system/environment.ts
View file

@ -1,6 +1,25 @@
import { Ctx } from "@budibase/types"
import env from "../../../environment"
import { env as coreEnv } from "@budibase/backend-core"
import nodeFetch from "node-fetch"
let sqsAvailable: boolean
async function isSqsAvailable() {
if (sqsAvailable !== undefined) {
return sqsAvailable
}
try {
await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
timeout: 1000,
})
sqsAvailable = true
return true
} catch (e) {
sqsAvailable = false
return false
}
}
export const fetch = async (ctx: Ctx) => {
ctx.body = {
@ -12,4 +31,10 @@ export const fetch = async (ctx: Ctx) => {
baseUrl: env.PLATFORM_URL,
isDev: env.isDev() && !env.isTest(),
}
if (env.SELF_HOSTED) {
ctx.body.infrastructure = {
sqs: await isSqsAvailable(),
}
}
}

19
packages/worker/src/api/routes/system/tests/environment.spec.ts
View file

@ -1,5 +1,7 @@
import { TestConfiguration } from "../../../../tests"
jest.unmock("node-fetch")
describe("/api/system/environment", () => {
const config = new TestConfiguration()
@ -27,5 +29,22 @@ describe("/api/system/environment", () => {
offlineMode: false,
})
})
it("returns the expected environment for self hosters", async () => {
await config.withEnv({ SELF_HOSTED: true }, async () => {
const env = await config.api.environment.getEnvironment()
expect(env.body).toEqual({
cloud: false,
disableAccountPortal: 0,
isDev: false,
multiTenancy: true,
baseUrl: "http://localhost:10000",
offlineMode: false,
infrastructure: {
sqs: false,
},
})
})
})
})
})

29
packages/worker/src/tests/TestConfiguration.ts
View file

@ -36,6 +36,7 @@ import {
} from "@budibase/types"
import API from "./api"
import jwt, { Secret } from "jsonwebtoken"
import cloneDeep from "lodash/fp/cloneDeep"
class TestConfiguration {
server: any
@ -240,6 +241,34 @@ class TestConfiguration {
return { message: "Admin user only endpoint.", status: 403 }
}
async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
let cleanup = this.setEnv(newEnvVars)
try {
await f()
} finally {
cleanup()
}
}
/*
* Sets the environment variables to the given values and returns a function
* that can be called to reset the environment variables to their original values.
*/
setEnv(newEnvVars: Partial<typeof env>): () => void {
const oldEnv = cloneDeep(env)
let key: keyof typeof newEnvVars
for (key in newEnvVars) {
env._set(key, newEnvVars[key])
}
return () => {
for (const [key, value] of Object.entries(oldEnv)) {
env._set(key, value)
}
}
}
// USERS
async createDefaultUser() {

786
yarn.lock
View file

File diff suppressed because it is too large Load diff