Merge branch 'master' into feature/rm-ff-code-per-user-per-creator

2024-10-03 02:27:06 +13:00 · 2023-12-14 09:08:35 +01:00 · 2023-12-14 09:08:35 +01:00 · ed3ad7dea9
commit ed3ad7dea9
parent 54ab1e16fe 99eb6597fb
23 changed files with 632 additions and 156 deletions
--- a/charts/budibase/README.md
+++ b/charts/budibase/README.md
@ -157,6 +157,17 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.apps.replicaCount | int | `1` | The number of apps replicas to run. |
 | services.apps.resources | object | `{}` | The resources to use for apps pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
 | services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.automationWorkers.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. |
+| services.automationWorkers.autoscaling.maxReplicas | int | `10` |  |
+| services.automationWorkers.autoscaling.minReplicas | int | `1` |  |
+| services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. |
+| services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. |
+| services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. |
+| services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.automationWorkers.replicaCount | int | `1` | The number of automation worker replicas to run. |
+| services.automationWorkers.resources | object | `{}` | The resources to use for automation worker pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.automationWorkers.startupProbe | object | HTTP health checks. | Startup probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
 | services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. |
 | services.couchdb.backup.interval | string | `""` | Backup interval in seconds |
 | services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@ -192,7 +192,14 @@ spec:
        - name: NODE_TLS_REJECT_UNAUTHORIZED
          value: {{ .Values.services.tlsRejectUnauthorized }}
        {{ end }}
-
+        {{- if .Values.services.automationWorkers.enabled }}
+        - name: APP_FEATURES
+          value: "api"
+        {{- end }}
+        {{- range .Values.services.apps.extraEnv }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
        image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
        imagePullPolicy: Always
        {{- if .Values.services.apps.startupProbe }}
--- a/charts/budibase/templates/automation-worker-service-deployment.yaml
+++ b/charts/budibase/templates/automation-worker-service-deployment.yaml
@ -0,0 +1,248 @@
+{{- if .Values.services.automationWorkers.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+{{ if .Values.services.automationWorkers.deploymentAnnotations }}
+{{- toYaml .Values.services.automationWorkers.deploymentAnnotations | indent 4 -}}
+{{ end }}
+  labels:
+    io.kompose.service: automation-worker-service
+{{ if .Values.services.automationWorkers.deploymentLabels }}
+{{- toYaml .Values.services.automationWorkers.deploymentLabels | indent 4 -}}
+{{ end }}
+  name: automation-worker-service
+spec:
+  replicas: {{ .Values.services.automationWorkers.replicaCount }}
+  selector:
+    matchLabels:
+      io.kompose.service: automation-worker-service
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+{{ if .Values.services.automationWorkers.templateAnnotations }}
+{{- toYaml .Values.services.automationWorkers.templateAnnotations | indent 8 -}}
+{{ end }}
+      labels:
+        io.kompose.service: automation-worker-service
+{{ if .Values.services.automationWorkers.templateLabels }}
+{{- toYaml .Values.services.automationWorkers.templateLabels | indent 8 -}}
+{{ end }}
+    spec:
+      containers:
+      - env:
+        - name: BUDIBASE_ENVIRONMENT
+          value: {{ .Values.globals.budibaseEnv }}
+        - name: DEPLOYMENT_ENVIRONMENT
+          value: "kubernetes"
+        - name: COUCH_DB_URL
+          {{ if .Values.services.couchdb.url }}
+          value: {{ .Values.services.couchdb.url }}
+          {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
+          {{ end }}
+        {{ if .Values.services.couchdb.enabled }}
+        - name: COUCH_DB_USER
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "couchdb.fullname" . }}
+              key: adminUsername
+        - name: COUCH_DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "couchdb.fullname" . }}
+              key: adminPassword
+        {{ end }}
+        - name: ENABLE_ANALYTICS
+          value: {{ .Values.globals.enableAnalytics | quote }}
+        - name: API_ENCRYPTION_KEY
+          value: {{ .Values.globals.apiEncryptionKey | quote }}
+        - name: HTTP_LOGGING
+          value: {{ .Values.services.automationWorkers.httpLogging | quote }}
+        - name: INTERNAL_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: internalApiKey
+        - name: INTERNAL_API_KEY_FALLBACK
+          value: {{ .Values.globals.internalApiKeyFallback | quote }}
+        - name: JWT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: jwtSecret
+        - name: JWT_SECRET_FALLBACK
+          value: {{ .Values.globals.jwtSecretFallback | quote }}
+        {{ if .Values.services.objectStore.region }}
+        - name: AWS_REGION
+          value: {{ .Values.services.objectStore.region }}
+        {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
+        - name: MINIO_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: objectStoreAccess
+        - name: MINIO_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: objectStoreSecret
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
+        - name: MINIO_URL
+          value: {{ .Values.services.objectStore.url }}
+        - name: PLUGIN_BUCKET_NAME
+          value: {{ .Values.services.objectStore.pluginBucketName | quote }}
+        - name: APPS_BUCKET_NAME
+          value: {{ .Values.services.objectStore.appsBucketName | quote }}
+        - name: GLOBAL_BUCKET_NAME
+          value: {{ .Values.services.objectStore.globalBucketName | quote }}
+        - name: BACKUPS_BUCKET_NAME
+          value: {{ .Values.services.objectStore.backupsBucketName | quote }}
+        - name: PORT
+          value: {{ .Values.services.automationWorkers.port | quote }}
+        {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
+        - name: API_REQ_LIMIT_PER_SEC
+          value: {{ .Values.globals.automationWorkers.publicApiRateLimitPerSecond | quote }}
+        {{ end }}
+        - name: MULTI_TENANCY
+          value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
+        - name: LOG_LEVEL
+          value: {{ .Values.services.automationWorkers.logLevel | quote }}
+        - name: REDIS_PASSWORD
+          value: {{ .Values.services.redis.password }}
+        - name: REDIS_URL
+          {{ if .Values.services.redis.url }}
+          value: {{ .Values.services.redis.url }}
+          {{ else }}
+          value: redis-service:{{ .Values.services.redis.port }}
+          {{ end }}
+        - name: SELF_HOSTED
+          value: {{ .Values.globals.selfHosted | quote }}
+        - name: POSTHOG_TOKEN
+          value: {{ .Values.globals.posthogToken | quote }}
+        - name: WORKER_URL
+          value: http://worker-service:{{ .Values.services.worker.port }}
+        - name: PLATFORM_URL
+          value: {{ .Values.globals.platformUrl | quote }}
+        - name: ACCOUNT_PORTAL_URL
+          value: {{ .Values.globals.accountPortalUrl | quote }}
+        - name: ACCOUNT_PORTAL_API_KEY
+          value: {{ .Values.globals.accountPortalApiKey | quote }}
+        - name: COOKIE_DOMAIN
+          value: {{ .Values.globals.cookieDomain | quote }}
+        - name: HTTP_MIGRATIONS
+          value: {{ .Values.globals.httpMigrations | quote }}
+        - name: GOOGLE_CLIENT_ID
+          value: {{ .Values.globals.google.clientId | quote }}
+        - name: GOOGLE_CLIENT_SECRET
+          value: {{ .Values.globals.google.secret | quote }}
+        - name: AUTOMATION_MAX_ITERATIONS
+          value: {{ .Values.globals.automationMaxIterations | quote }}
+        - name: TENANT_FEATURE_FLAGS
+          value: {{ .Values.globals.tenantFeatureFlags | quote }}
+        - name: ENCRYPTION_KEY
+          value: {{ .Values.globals.bbEncryptionKey | quote }}
+        {{ if .Values.globals.bbAdminUserEmail }}
+        - name: BB_ADMIN_USER_EMAIL
+          value: {{ .Values.globals.bbAdminUserEmail | quote }}
+        {{ end }}
+        {{ if .Values.globals.bbAdminUserPassword }}
+        - name: BB_ADMIN_USER_PASSWORD
+          value: {{ .Values.globals.bbAdminUserPassword | quote }}
+        {{ end }}
+        {{ if .Values.globals.pluginsDir }}
+        - name: PLUGINS_DIR
+          value: {{ .Values.globals.pluginsDir | quote }}
+        {{ end }}
+        {{ if .Values.services.automationWorkers.nodeDebug }}
+        - name: NODE_DEBUG
+          value: {{ .Values.services.automationWorkers.nodeDebug | quote }}
+        {{ end }}
+        {{ if .Values.globals.datadogApmEnabled }}
+        - name: DD_LOGS_INJECTION
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_ENABLED
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_DD_URL
+          value: https://trace.agent.datadoghq.eu
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpProxy }}
+        - name: GLOBAL_AGENT_HTTP_PROXY
+          value: {{ .Values.globals.globalAgentHttpProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpsProxy }}
+        - name: GLOBAL_AGENT_HTTPS_PROXY
+          value: {{ .Values.globals.globalAgentHttpsProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentNoProxy }}
+        - name: GLOBAL_AGENT_NO_PROXY
+          value: {{ .Values.globals.globalAgentNoProxy | quote }}
+        {{ end }}
+        {{ if .Values.services.tlsRejectUnauthorized }}
+        - name: NODE_TLS_REJECT_UNAUTHORIZED
+          value: {{ .Values.services.tlsRejectUnauthorized }}
+        {{ end }}
+        - name: APP_FEATURES
+          value: "automations"
+        {{- range .Values.services.automationWorkers.extraEnv }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
+
+        image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
+        imagePullPolicy: Always
+        {{- if .Values.services.automationWorkers.startupProbe }}
+        {{- with .Values.services.automationWorkers.startupProbe }}
+        startupProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.automationWorkers.livenessProbe }}
+        {{- with .Values.services.automationWorkers.livenessProbe }}
+        livenessProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.automationWorkers.readinessProbe }}
+        {{- with .Values.services.automationWorkers.readinessProbe }}
+        readinessProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        name: bbautomationworker
+        ports:
+        - containerPort: {{ .Values.services.automationWorkers.port }}
+        {{ with .Values.services.automationWorkers.resources }}
+        resources:
+        {{- toYaml . | nindent 10 }}
+        {{ end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      {{ if .Values.schedulerName }}
+      schedulerName: {{ .Values.schedulerName | quote }}
+      {{ end }}    
+      {{ if .Values.imagePullSecrets }}
+      imagePullSecrets:
+      {{- toYaml .Values.imagePullSecrets | nindent 6 }}
+      {{ end }}
+      restartPolicy: Always
+      serviceAccountName: ""
+status: {}
+{{- end }}
--- a/charts/budibase/templates/automation-worker-service-hpa.yaml
+++ b/charts/budibase/templates/automation-worker-service-hpa.yaml
@ -0,0 +1,32 @@
+{{- if .Values.services.automationWorkers.autoscaling.enabled }}
+apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "budibase.fullname" . }}-apps
+  labels:
+    {{- include "budibase.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: automation-worker-service
+  minReplicas: {{ .Values.services.automationWorkers.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.services.automationWorkers.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@ -182,6 +182,10 @@ spec:
        - name: NODE_TLS_REJECT_UNAUTHORIZED
          value: {{ .Values.services.tlsRejectUnauthorized }}
        {{ end }}
+        {{- range .Values.services.worker.extraEnv }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
        image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
        imagePullPolicy: Always
        {{- if .Values.services.worker.startupProbe }}
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@ -220,6 +220,9 @@ services:
    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
    # for more information on how to set these.
    resources: {}
+    # -- Extra environment variables to set for apps pods. Takes a list of
+    # name=value pairs.
+    extraEnv: []
    # -- Startup probe configuration for apps pods. You shouldn't need to
    # change this, but if you want to you can find more information here:
    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
@ -272,6 +275,78 @@ services:
      # and resources set for the apps pods.
      targetCPUUtilizationPercentage: 80

+  automationWorkers:
+    # -- Whether or not to enable the automation worker service. If you disable this,
+    # automations will be processed by the apps service.
+    enabled: true
+    # @ignore (you shouldn't need to change this)
+    port: 4002
+    # -- The number of automation worker replicas to run.
+    replicaCount: 1
+    # -- The log level for the automation worker service.
+    logLevel: info
+    # -- The resources to use for automation worker pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
+    resources: {}
+    # -- Extra environment variables to set for automation worker pods. Takes a list of
+    # name=value pairs.
+    extraEnv: []
+    # -- Startup probe configuration for automation worker pods. You shouldn't
+    # need to change this, but if you want to you can find more information
+    # here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
+    startupProbe:
+      # @ignore
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      # @ignore
+      failureThreshold: 30
+      # @ignore
+      periodSeconds: 3
+    # -- Readiness probe configuration for automation worker pods. You shouldn't
+    # need to change this, but if you want to you can find more information
+    # here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
+    readinessProbe:
+      # @ignore
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      # @ignore
+      periodSeconds: 3
+      # @ignore
+      failureThreshold: 1
+    # -- Liveness probe configuration for automation worker pods. You shouldn't
+    # need to change this, but if you want to you can find more information
+    # here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
+    livenessProbe:
+      # @ignore
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      # @ignore
+      failureThreshold: 3
+      # @ignore
+      periodSeconds: 30
+    autoscaling:
+      # -- Whether to enable horizontal pod autoscaling for the apps service.
+      enabled: false
+      minReplicas: 1
+      maxReplicas: 10
+      # -- Target CPU utilization percentage for the automation worker service.
+      # Note that for autoscaling to work, you will need to have metrics-server
+      # configured, and resources set for the automation worker pods.
+      targetCPUUtilizationPercentage: 80
+
  worker:
    # @ignore (you shouldn't need to change this)
    port: 4003
@ -285,6 +360,9 @@ services:
    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
    # for more information on how to set these.
    resources: {}
+    # -- Extra environment variables to set for worker pods. Takes a list of
+    # name=value pairs.
+    extraEnv: []
    # -- Startup probe configuration for worker pods. You shouldn't need to
    # change this, but if you want to you can find more information here:
    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "2.13.37",
+  "version": "2.13.39",
  "npmClient": "yarn",
  "packages": [
    "packages/*",
@ -22,4 +22,4 @@
      "loadEnvFiles": false
    }
  }
-}
+}
--- a/packages/backend-core/src/cache/tests/writethrough.spec.ts
+++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts
@ -1,15 +1,16 @@
 import { DBTestConfiguration } from "../../../tests/extra"
-import {
-  structures,
-  expectFunctionWasCalledTimesWith,
-  mocks,
-} from "../../../tests"
+import { structures } from "../../../tests"
 import { Writethrough } from "../writethrough"
 import { getDB } from "../../db"
+import { Document } from "@budibase/types"
 import tk from "timekeeper"

 tk.freeze(Date.now())

+interface ValueDoc extends Document {
+  value: any
+}
+
 const DELAY = 5000

 describe("writethrough", () => {
@ -117,7 +118,7 @@ describe("writethrough", () => {
  describe("get", () => {
    it("should be able to retrieve", async () => {
      await config.doInTenant(async () => {
-        const response = await writethrough.get(docId)
+        const response = await writethrough.get<ValueDoc>(docId)
        expect(response.value).toBe(4)
      })
    })
--- a/packages/backend-core/src/cache/writethrough.ts
+++ b/packages/backend-core/src/cache/writethrough.ts
@ -7,7 +7,7 @@ import * as locks from "../redis/redlockImpl"
 const DEFAULT_WRITE_RATE_MS = 10000
 let CACHE: BaseCache | null = null

-interface CacheItem {
+interface CacheItem<T extends Document> {
  doc: any
  lastWrite: number
 }
@ -24,7 +24,10 @@ function makeCacheKey(db: Database, key: string) {
  return db.name + key
 }

-function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
+function makeCacheItem<T extends Document>(
+  doc: T,
+  lastWrite: number | null = null
+): CacheItem<T> {
  return { doc, lastWrite: lastWrite || Date.now() }
 }

@ -35,7 +38,7 @@ async function put(
 ) {
  const cache = await getCache()
  const key = doc._id
-  let cacheItem: CacheItem | undefined
+  let cacheItem: CacheItem<any> | undefined
  if (key) {
    cacheItem = await cache.get(makeCacheKey(db, key))
  }
@ -84,12 +87,12 @@ async function put(
  return { ok: true, id: output._id, rev: output._rev }
 }

-async function get(db: Database, id: string): Promise<any> {
+async function get<T extends Document>(db: Database, id: string): Promise<T> {
  const cache = await getCache()
  const cacheKey = makeCacheKey(db, id)
-  let cacheItem: CacheItem = await cache.get(cacheKey)
+  let cacheItem: CacheItem<T> = await cache.get(cacheKey)
  if (!cacheItem) {
-    const doc = await db.get(id)
+    const doc = await db.get<T>(id)
    cacheItem = makeCacheItem(doc)
    await cache.store(cacheKey, cacheItem)
  }
@ -123,8 +126,8 @@ export class Writethrough {
    return put(this.db, doc, writeRateMs)
  }

-  async get(id: string) {
-    return get(this.db, id)
+  async get<T extends Document>(id: string) {
+    return get<T>(this.db, id)
  }

  async remove(docOrId: any, rev?: any) {
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@ -107,6 +107,7 @@ const environment = {
  ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
  API_ENCRYPTION_KEY: getAPIEncryptionKey(),
  COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
  COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
  COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
  GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
--- a/packages/backend-core/src/queue/inMemoryQueue.ts
+++ b/packages/backend-core/src/queue/inMemoryQueue.ts
@ -68,6 +68,10 @@ class InMemoryQueue {
    })
  }

+  async isReady() {
+    return true
+  }
+
  // simply puts a message to the queue and emits to the queue for processing
  /**
   * Simple function to replicate the add message functionality of Bull, putting
--- a/packages/backend-core/src/redis/redlockImpl.ts
+++ b/packages/backend-core/src/redis/redlockImpl.ts
@ -137,7 +137,6 @@ export async function doWithLock<T>(
    const result = await task()
    return { executed: true, result }
  } catch (e: any) {
-    logWarn(`lock type: ${opts.type} error`, e)
    // lock limit exceeded
    if (e.name === "LockError") {
      if (opts.type === LockType.TRY_ONCE) {
--- a/packages/pro
+++ b/packages/pro
@ -1 +1 @@
-Subproject commit 056c2093dbc93d9a10ea9f5050c84a84edd8100c
+Subproject commit 992486c10044a7495496b97bdf5f454d4020bfba
--- a/packages/server/scripts/dev/manage.js
+++ b/packages/server/scripts/dev/manage.js
@ -1,7 +1,8 @@
 #!/usr/bin/env node
 const compose = require("docker-compose")
 const path = require("path")
-const fs = require("fs")
+const { parsed: existingConfig } = require("dotenv").config()
+const updateDotEnv = require("update-dotenv")

 // This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
 const CONFIG = {
@ -17,45 +18,41 @@ const Commands = {
 }

 async function init() {
-  const envFilePath = path.join(process.cwd(), ".env")
-  if (!fs.existsSync(envFilePath)) {
-    const envFileJson = {
-      PORT: 4001,
-      MINIO_URL: "http://localhost:4004",
-      COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
-      REDIS_URL: "localhost:6379",
-      WORKER_URL: "http://localhost:4002",
-      INTERNAL_API_KEY: "budibase",
-      ACCOUNT_PORTAL_URL: "http://localhost:10001",
-      ACCOUNT_PORTAL_API_KEY: "budibase",
-      PLATFORM_URL: "http://localhost:10000",
-      JWT_SECRET: "testsecret",
-      ENCRYPTION_KEY: "testsecret",
-      REDIS_PASSWORD: "budibase",
-      MINIO_ACCESS_KEY: "budibase",
-      MINIO_SECRET_KEY: "budibase",
-      COUCH_DB_PASSWORD: "budibase",
-      COUCH_DB_USER: "budibase",
-      SELF_HOSTED: 1,
-      DISABLE_ACCOUNT_PORTAL: 1,
-      MULTI_TENANCY: "",
-      DISABLE_THREADING: 1,
-      SERVICE: "app-service",
-      DEPLOYMENT_ENVIRONMENT: "development",
-      BB_ADMIN_USER_EMAIL: "",
-      BB_ADMIN_USER_PASSWORD: "",
-      PLUGINS_DIR: "",
-      TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
-      HTTP_MIGRATIONS: "0",
-      HTTP_LOGGING: "0",
-      VERSION: "0.0.0+local",
-    }
-    let envFile = ""
-    Object.keys(envFileJson).forEach(key => {
-      envFile += `${key}=${envFileJson[key]}\n`
-    })
-    fs.writeFileSync(envFilePath, envFile)
+  let config = {
+    PORT: "4001",
+    MINIO_URL: "http://localhost:4004",
+    COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
+    REDIS_URL: "localhost:6379",
+    WORKER_URL: "http://localhost:4002",
+    INTERNAL_API_KEY: "budibase",
+    ACCOUNT_PORTAL_URL: "http://localhost:10001",
+    ACCOUNT_PORTAL_API_KEY: "budibase",
+    PLATFORM_URL: "http://localhost:10000",
+    JWT_SECRET: "testsecret",
+    ENCRYPTION_KEY: "testsecret",
+    REDIS_PASSWORD: "budibase",
+    MINIO_ACCESS_KEY: "budibase",
+    MINIO_SECRET_KEY: "budibase",
+    COUCH_DB_PASSWORD: "budibase",
+    COUCH_DB_USER: "budibase",
+    SELF_HOSTED: "1",
+    DISABLE_ACCOUNT_PORTAL: "1",
+    MULTI_TENANCY: "",
+    DISABLE_THREADING: "1",
+    SERVICE: "app-service",
+    DEPLOYMENT_ENVIRONMENT: "development",
+    BB_ADMIN_USER_EMAIL: "",
+    BB_ADMIN_USER_PASSWORD: "",
+    PLUGINS_DIR: "",
+    TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
+    HTTP_MIGRATIONS: "0",
+    HTTP_LOGGING: "0",
+    VERSION: "0.0.0+local",
  }
+
+  config = { ...config, ...existingConfig }
+
+  await updateDotEnv(config)
 }

 async function up() {
--- a/packages/server/src/api/index.ts
+++ b/packages/server/src/api/index.ts
@ -4,62 +4,75 @@ import currentApp from "../middleware/currentapp"
 import zlib from "zlib"
 import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
 import { middleware as pro } from "@budibase/pro"
+import { apiEnabled, automationsEnabled } from "../features"
 import migrations from "../middleware/appMigrations"
+import { automationQueue } from "../automations"

 export { shutdown } from "./routes/public"
 const compress = require("koa-compress")

 export const router: Router = new Router()

-router.get("/health", ctx => (ctx.status = 200))
+router.get("/health", async ctx => {
+  if (automationsEnabled()) {
+    if (!(await automationQueue.isReady())) {
+      ctx.status = 503
+      return
+    }
+  }
+  ctx.status = 200
+})
 router.get("/version", ctx => (ctx.body = envCore.VERSION))

 router.use(middleware.errorHandling)

-router
-  .use(
-    compress({
-      threshold: 2048,
-      gzip: {
-        flush: zlib.constants.Z_SYNC_FLUSH,
-      },
-      deflate: {
-        flush: zlib.constants.Z_SYNC_FLUSH,
-      },
-      br: false,
-    })
-  )
-  // re-direct before any middlewares occur
-  .redirect("/", "/builder")
-  .use(
-    auth.buildAuthMiddleware([], {
-      publicAllowed: true,
-    })
-  )
-  // nothing in the server should allow query string tenants
-  // the server can be public anywhere, so nowhere should throw errors
-  // if the tenancy has not been set, it'll have to be discovered at application layer
-  .use(
-    auth.buildTenancyMiddleware([], [], {
-      noTenancyRequired: true,
-    })
-  )
-  .use(pro.licensing())
-  // @ts-ignore
-  .use(currentApp)
-  .use(auth.auditLog)
-  // @ts-ignore
-  .use(migrations)
+// only add the routes if they are enabled
+if (apiEnabled()) {
+  router
+    .use(
+      compress({
+        threshold: 2048,
+        gzip: {
+          flush: zlib.constants.Z_SYNC_FLUSH,
+        },
+        deflate: {
+          flush: zlib.constants.Z_SYNC_FLUSH,
+        },
+        br: false,
+      })
+    )
+    // re-direct before any middlewares occur
+    .redirect("/", "/builder")
+    .use(
+      auth.buildAuthMiddleware([], {
+        publicAllowed: true,
+      })
+    )
+    // nothing in the server should allow query string tenants
+    // the server can be public anywhere, so nowhere should throw errors
+    // if the tenancy has not been set, it'll have to be discovered at application layer
+    .use(
+      auth.buildTenancyMiddleware([], [], {
+        noTenancyRequired: true,
+      })
+    )
+    .use(pro.licensing())
+    // @ts-ignore
+    .use(currentApp)
+    .use(auth.auditLog)
+    // @ts-ignore
+    .use(migrations)

-// authenticated routes
-for (let route of mainRoutes) {
-  router.use(route.routes())
-  router.use(route.allowedMethods())
+  // authenticated routes
+  for (let route of mainRoutes) {
+    router.use(route.routes())
+    router.use(route.allowedMethods())
+  }
+
+  router.use(publicRoutes.routes())
+  router.use(publicRoutes.allowedMethods())
+
+  // WARNING - static routes will catch everything else after them this must be last
+  router.use(staticRoutes.routes())
+  router.use(staticRoutes.allowedMethods())
 }
-
-router.use(publicRoutes.routes())
-router.use(publicRoutes.allowedMethods())
-
-// WARNING - static routes will catch everything else after them this must be last
-router.use(staticRoutes.routes())
-router.use(staticRoutes.allowedMethods())
--- a/packages/server/src/app.ts
+++ b/packages/server/src/app.ts
@ -9,7 +9,6 @@ import { ServiceType } from "@budibase/types"
 import { env as coreEnv } from "@budibase/backend-core"

 coreEnv._set("SERVICE_TYPE", ServiceType.APPS)
-import { apiEnabled } from "./features"
 import createKoaApp from "./koa"
 import Koa from "koa"
 import { Server } from "http"
@ -18,12 +17,9 @@ import { startup } from "./startup"
 let app: Koa, server: Server

 async function start() {
-  // if API disabled, could run automations instead
-  if (apiEnabled()) {
-    const koa = createKoaApp()
-    app = koa.app
-    server = koa.server
-  }
+  const koa = createKoaApp()
+  app = koa.app
+  server = koa.server
  // startup includes automation runner - if enabled
  await startup(app, server)
 }
--- a/packages/server/src/features.ts
+++ b/packages/server/src/features.ts
@ -22,3 +22,10 @@ export function automationsEnabled() {
 export function apiEnabled() {
  return featureList.includes(AppFeature.API)
 }
+
+export function printFeatures() {
+  if (!env.APP_FEATURES) {
+    return
+  }
+  console.log(`**** APP FEATURES SET: ${featureList.join(", ")} ****`)
+}
--- a/packages/server/src/startup.ts
+++ b/packages/server/src/startup.ts
@ -19,11 +19,14 @@ import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
 import { initialise as initialiseWebsockets } from "./websockets"
-import { automationsEnabled } from "./features"
+import { automationsEnabled, printFeatures } from "./features"
+import Koa from "koa"
+import { Server } from "http"
+import { AddressInfo } from "net"

 let STARTUP_RAN = false

-async function initRoutes(app: any) {
+async function initRoutes(app: Koa) {
  if (!env.isTest()) {
    const plugin = await bullboard.init()
    app.use(plugin)
@ -48,27 +51,31 @@ async function initPro() {
  })
 }

-function shutdown(server?: any) {
+function shutdown(server?: Server) {
  if (server) {
    server.close()
    server.destroy()
  }
 }

-export async function startup(app?: any, server?: any) {
+export async function startup(app?: Koa, server?: Server) {
  if (STARTUP_RAN) {
    return
  }
+  printFeatures()
  STARTUP_RAN = true
-  if (server && !env.CLUSTER_MODE) {
+  if (app && server && !env.CLUSTER_MODE) {
    console.log(`Budibase running on ${JSON.stringify(server.address())}`)
-    env._set("PORT", server.address().port)
+    const address = server.address() as AddressInfo
+    env._set("PORT", address.port)
  }
  eventEmitter.emitPort(env.PORT)
  fileSystem.init()
  await redis.init()
  eventInit()
-  initialiseWebsockets(app, server)
+  if (app && server) {
+    initialiseWebsockets(app, server)
+  }

  // run migrations on startup if not done via http
  // not recommended in a clustered environment
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@ -17,7 +17,6 @@ import {
  basicWebhook,
 } from "./structures"
 import {
-  auth,
  cache,
  constants,
  context,
--- a/packages/worker/scripts/dev/manage.js
+++ b/packages/worker/scripts/dev/manage.js
@ -1,44 +1,40 @@
 #!/usr/bin/env node
-const path = require("path")
-const fs = require("fs")
+const { parsed: existingConfig } = require("dotenv").config()
+const updateDotEnv = require("update-dotenv")

 async function init() {
-  const envFilePath = path.join(process.cwd(), ".env")
-  if (!fs.existsSync(envFilePath)) {
-    const envFileJson = {
-      SELF_HOSTED: 1,
-      PORT: 4002,
-      CLUSTER_PORT: 10000,
-      JWT_SECRET: "testsecret",
-      INTERNAL_API_KEY: "budibase",
-      MINIO_ACCESS_KEY: "budibase",
-      MINIO_SECRET_KEY: "budibase",
-      REDIS_URL: "localhost:6379",
-      REDIS_PASSWORD: "budibase",
-      MINIO_URL: "http://localhost:4004",
-      COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
-      COUCH_DB_USERNAME: "budibase",
-      COUCH_DB_PASSWORD: "budibase",
-      // empty string is false
-      MULTI_TENANCY: "",
-      DISABLE_ACCOUNT_PORTAL: 1,
-      ACCOUNT_PORTAL_URL: "http://localhost:10001",
-      ACCOUNT_PORTAL_API_KEY: "budibase",
-      PLATFORM_URL: "http://localhost:10000",
-      APPS_URL: "http://localhost:4001",
-      SERVICE: "worker-service",
-      DEPLOYMENT_ENVIRONMENT: "development",
-      TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
-      ENABLE_EMAIL_TEST_MODE: 1,
-      HTTP_LOGGING: 0,
-      VERSION: "0.0.0+local",
-    }
-    let envFile = ""
-    Object.keys(envFileJson).forEach(key => {
-      envFile += `${key}=${envFileJson[key]}\n`
-    })
-    fs.writeFileSync(envFilePath, envFile)
+  let config = {
+    SELF_HOSTED: "1",
+    PORT: "4002",
+    CLUSTER_PORT: "10000",
+    JWT_SECRET: "testsecret",
+    INTERNAL_API_KEY: "budibase",
+    MINIO_ACCESS_KEY: "budibase",
+    MINIO_SECRET_KEY: "budibase",
+    REDIS_URL: "localhost:6379",
+    REDIS_PASSWORD: "budibase",
+    MINIO_URL: "http://localhost:4004",
+    COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
+    COUCH_DB_USERNAME: "budibase",
+    COUCH_DB_PASSWORD: "budibase",
+    // empty string is false
+    MULTI_TENANCY: "",
+    DISABLE_ACCOUNT_PORTAL: "1",
+    ACCOUNT_PORTAL_URL: "http://localhost:10001",
+    ACCOUNT_PORTAL_API_KEY: "budibase",
+    PLATFORM_URL: "http://localhost:10000",
+    APPS_URL: "http://localhost:4001",
+    SERVICE: "worker-service",
+    DEPLOYMENT_ENVIRONMENT: "development",
+    TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
+    ENABLE_EMAIL_TEST_MODE: "1",
+    HTTP_LOGGING: "0",
+    VERSION: "0.0.0+local",
  }
+
+  config = { ...config, ...existingConfig }
+
+  await updateDotEnv(config)
 }

 // if more than init required use this to determine the command type
--- a/packages/worker/src/api/controllers/system/environment.ts
+++ b/packages/worker/src/api/controllers/system/environment.ts
@ -1,6 +1,25 @@
 import { Ctx } from "@budibase/types"
 import env from "../../../environment"
 import { env as coreEnv } from "@budibase/backend-core"
+import nodeFetch from "node-fetch"
+
+let sqsAvailable: boolean
+async function isSqsAvailable() {
+  if (sqsAvailable !== undefined) {
+    return sqsAvailable
+  }
+
+  try {
+    await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
+      timeout: 1000,
+    })
+    sqsAvailable = true
+    return true
+  } catch (e) {
+    sqsAvailable = false
+    return false
+  }
+}

 export const fetch = async (ctx: Ctx) => {
  ctx.body = {
@ -12,4 +31,10 @@ export const fetch = async (ctx: Ctx) => {
    baseUrl: env.PLATFORM_URL,
    isDev: env.isDev() && !env.isTest(),
  }
+
+  if (env.SELF_HOSTED) {
+    ctx.body.infrastructure = {
+      sqs: await isSqsAvailable(),
+    }
+  }
 }
--- a/packages/worker/src/api/routes/system/tests/environment.spec.ts
+++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts
@ -1,5 +1,7 @@
 import { TestConfiguration } from "../../../../tests"

+jest.unmock("node-fetch")
+
 describe("/api/system/environment", () => {
  const config = new TestConfiguration()

@ -27,5 +29,22 @@ describe("/api/system/environment", () => {
        offlineMode: false,
      })
    })
+
+    it("returns the expected environment for self hosters", async () => {
+      await config.withEnv({ SELF_HOSTED: true }, async () => {
+        const env = await config.api.environment.getEnvironment()
+        expect(env.body).toEqual({
+          cloud: false,
+          disableAccountPortal: 0,
+          isDev: false,
+          multiTenancy: true,
+          baseUrl: "http://localhost:10000",
+          offlineMode: false,
+          infrastructure: {
+            sqs: false,
+          },
+        })
+      })
+    })
  })
 })
--- a/packages/worker/src/tests/TestConfiguration.ts
+++ b/packages/worker/src/tests/TestConfiguration.ts
@ -36,6 +36,7 @@ import {
 } from "@budibase/types"
 import API from "./api"
 import jwt, { Secret } from "jsonwebtoken"
+import cloneDeep from "lodash/fp/cloneDeep"

 class TestConfiguration {
  server: any
@ -240,6 +241,34 @@ class TestConfiguration {
    return { message: "Admin user only endpoint.", status: 403 }
  }

+  async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
+    let cleanup = this.setEnv(newEnvVars)
+    try {
+      await f()
+    } finally {
+      cleanup()
+    }
+  }
+
+  /*
+   * Sets the environment variables to the given values and returns a function
+   * that can be called to reset the environment variables to their original values.
+   */
+  setEnv(newEnvVars: Partial<typeof env>): () => void {
+    const oldEnv = cloneDeep(env)
+
+    let key: keyof typeof newEnvVars
+    for (key in newEnvVars) {
+      env._set(key, newEnvVars[key])
+    }
+
+    return () => {
+      for (const [key, value] of Object.entries(oldEnv)) {
+        env._set(key, value)
+      }
+    }
+  }
+
  // USERS

  async createDefaultUser() {