Merge pull request #9453 from Budibase/feature/environment-variables

Environment Variables

packages/backend-core/src/constants/db.ts

@@ -77,6 +77,7 @@ export const StaticDatabases = {
       apiKeys: "apikeys",
       usageQuota: "usage_quota",
       licenseInfo: "license_info",
+      environmentVariables: "environmentvariables",
     },
   },
   // contains information about tenancy and so on

packages/backend-core/src/context/Context.ts

@@ -1,17 +1,14 @@
 import { AsyncLocalStorage } from "async_hooks"
+import { ContextMap } from "./mainContext"
 
 export default class Context {
-  static storage = new AsyncLocalStorage<Record<string, any>>()
+  static storage = new AsyncLocalStorage<ContextMap>()
 
-  static run(context: Record<string, any>, func: any) {
+  static run(context: ContextMap, func: any) {
     return Context.storage.run(context, () => func())
   }
 
-  static get(): Record<string, any> {
-    return Context.storage.getStore() as Record<string, any>
-  }
-
-  static set(context: Record<string, any>) {
-    Context.storage.enterWith(context)
+  static get(): ContextMap {
+    return Context.storage.getStore() as ContextMap
   }
 }

packages/backend-core/src/context/mainContext.ts

@@ -16,6 +16,7 @@ export type ContextMap = {
   tenantId?: string
   appId?: string
   identity?: IdentityContext
+  environmentVariables?: Record<string, string>
 }
 
 let TEST_APP_ID: string | null = null
@@ -75,7 +76,7 @@ export function getTenantIDFromAppID(appId: string) {
   }
 }
 
-function updateContext(updates: ContextMap) {
+function updateContext(updates: ContextMap): ContextMap {
   let context: ContextMap
   try {
     context = Context.get()
@@ -120,15 +121,23 @@ export async function doInTenant(
   return newContext(updates, task)
 }
 
-export async function doInAppContext(appId: string, task: any): Promise<any> {
-  if (!appId) {
+export async function doInAppContext(
+  appId: string | null,
+  task: any
+): Promise<any> {
+  if (!appId && !env.isTest()) {
     throw new Error("appId is required")
   }
 
-  const tenantId = getTenantIDFromAppID(appId)
-  const updates: ContextMap = { appId }
-  if (tenantId) {
-    updates.tenantId = tenantId
+  let updates: ContextMap
+  if (!appId) {
+    updates = { appId: "" }
+  } else {
+    const tenantId = getTenantIDFromAppID(appId)
+    updates = { appId }
+    if (tenantId) {
+      updates.tenantId = tenantId
+    }
   }
   return newContext(updates, task)
 }
@@ -189,25 +198,25 @@ export const getProdAppId = () => {
   return conversions.getProdAppID(appId)
 }
 
-export function updateTenantId(tenantId?: string) {
-  let context: ContextMap = updateContext({
-    tenantId,
-  })
-  Context.set(context)
+export function doInEnvironmentContext(
+  values: Record<string, string>,
+  task: any
+) {
+  if (!values) {
+    throw new Error("Must supply environment variables.")
+  }
+  const updates = {
+    environmentVariables: values,
+  }
+  return newContext(updates, task)
 }
 
-export function updateAppId(appId: string) {
-  let context: ContextMap = updateContext({
-    appId,
-  })
-  try {
-    Context.set(context)
-  } catch (err) {
-    if (env.isTest()) {
-      TEST_APP_ID = appId
-    } else {
-      throw err
-    }
+export function getEnvironmentVariables() {
+  const context = Context.get()
+  if (!context.environmentVariables) {
+    return null
+  } else {
+    return context.environmentVariables
   }
 }
 

packages/backend-core/src/environment.ts

@@ -37,6 +37,7 @@ const environment = {
   },
   JS_BCRYPT: process.env.JS_BCRYPT,
   JWT_SECRET: process.env.JWT_SECRET,
+  ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,

packages/backend-core/src/events/publishers/environmentVariable.ts

@@ -0,0 +1,38 @@
+import {
+  Event,
+  EnvironmentVariableCreatedEvent,
+  EnvironmentVariableDeletedEvent,
+  EnvironmentVariableUpgradePanelOpenedEvent,
+} from "@budibase/types"
+import { publishEvent } from "../events"
+
+async function created(name: string, environments: string[]) {
+  const properties: EnvironmentVariableCreatedEvent = {
+    name,
+    environments,
+  }
+  await publishEvent(Event.ENVIRONMENT_VARIABLE_CREATED, properties)
+}
+
+async function deleted(name: string) {
+  const properties: EnvironmentVariableDeletedEvent = {
+    name,
+  }
+  await publishEvent(Event.ENVIRONMENT_VARIABLE_DELETED, properties)
+}
+
+async function upgradePanelOpened(userId: string) {
+  const properties: EnvironmentVariableUpgradePanelOpenedEvent = {
+    userId,
+  }
+  await publishEvent(
+    Event.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED,
+    properties
+  )
+}
+
+export default {
+  created,
+  deleted,
+  upgradePanelOpened,
+}

packages/backend-core/src/events/publishers/index.ts

@@ -20,3 +20,4 @@ export { default as backfill } from "./backfill"
 export { default as group } from "./group"
 export { default as plugin } from "./plugin"
 export { default as backup } from "./backup"
+export { default as environmentVariable } from "./environmentVariable"

packages/backend-core/src/security/encryption.ts

@@ -2,19 +2,45 @@ import crypto from "crypto"
 import env from "../environment"
 
 const ALGO = "aes-256-ctr"
-const SECRET = env.JWT_SECRET
 const SEPARATOR = "-"
 const ITERATIONS = 10000
 const RANDOM_BYTES = 16
 const STRETCH_LENGTH = 32
 
+export enum SecretOption {
+  JWT = "jwt",
+  ENCRYPTION = "encryption",
+}
+
+function getSecret(secretOption: SecretOption): string {
+  let secret, secretName
+  switch (secretOption) {
+    case SecretOption.ENCRYPTION:
+      secret = env.ENCRYPTION_KEY
+      secretName = "ENCRYPTION_KEY"
+      break
+    case SecretOption.JWT:
+    default:
+      secret = env.JWT_SECRET
+      secretName = "JWT_SECRET"
+      break
+  }
+  if (!secret) {
+    throw new Error(`Secret "${secretName}" has not been set in environment.`)
+  }
+  return secret
+}
+
 function stretchString(string: string, salt: Buffer) {
   return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512")
 }
 
-export function encrypt(input: string) {
+export function encrypt(
+  input: string,
+  secretOption: SecretOption = SecretOption.JWT
+) {
   const salt = crypto.randomBytes(RANDOM_BYTES)
-  const stretched = stretchString(SECRET!, salt)
+  const stretched = stretchString(getSecret(secretOption), salt)
   const cipher = crypto.createCipheriv(ALGO, stretched, salt)
   const base = cipher.update(input)
   const final = cipher.final()
@@ -22,10 +48,13 @@ export function encrypt(input: string) {
   return `${salt.toString("hex")}${SEPARATOR}${encrypted}`
 }
 
-export function decrypt(input: string) {
+export function decrypt(
+  input: string,
+  secretOption: SecretOption = SecretOption.JWT
+) {
   const [salt, encrypted] = input.split(SEPARATOR)
   const saltBuffer = Buffer.from(salt, "hex")
-  const stretched = stretchString(SECRET!, saltBuffer)
+  const stretched = stretchString(getSecret(secretOption), saltBuffer)
   const decipher = crypto.createDecipheriv(ALGO, stretched, saltBuffer)
   const base = decipher.update(Buffer.from(encrypted, "hex"))
   const final = decipher.final()

packages/bbui/src/Form/Core/EnvDropdown.svelte

@@ -0,0 +1,282 @@
+<script>
+  import "@spectrum-css/textfield/dist/index-vars.css"
+  import { createEventDispatcher, onMount } from "svelte"
+  import clickOutside from "../../Actions/click_outside"
+  import Divider from "../../Divider/Divider.svelte"
+
+  export let value = null
+  export let placeholder = null
+  export let type = "text"
+  export let disabled = false
+  export let id = null
+  export let readonly = false
+  export let updateOnChange = true
+  export let dataCy
+  export let align
+  export let autofocus = false
+  export let variables
+  export let showModal
+  export let environmentVariablesEnabled
+  export let handleUpgradePanel
+  const dispatch = createEventDispatcher()
+
+  let field
+  let focus = false
+  let iconFocused = false
+  let open = false
+
+  //eslint-disable-next-line
+  const STRIP_NAME_REGEX = /(?<=\.)(.*?)(?=\ })/g
+
+  // Strips the name out of the value which is {{ env.Variable }} resulting in an array like ["Variable"]
+  $: hbsValue = String(value)?.match(STRIP_NAME_REGEX) || []
+
+  const updateValue = newValue => {
+    if (readonly) {
+      return
+    }
+    if (type === "number") {
+      const float = parseFloat(newValue)
+      newValue = isNaN(float) ? null : float
+    }
+    dispatch("change", newValue)
+  }
+
+  const onFocus = () => {
+    if (readonly) {
+      return
+    }
+    focus = true
+  }
+
+  const onBlur = event => {
+    if (readonly) {
+      return
+    }
+    focus = false
+    updateValue(event.target.value)
+  }
+
+  const onInput = event => {
+    if (readonly || !updateOnChange) {
+      return
+    }
+    updateValue(event.target.value)
+  }
+
+  const handleOutsideClick = event => {
+    if (open) {
+      event.stopPropagation()
+      open = false
+      focus = false
+      iconFocused = false
+      dispatch("closed")
+    }
+  }
+
+  const handleVarSelect = variable => {
+    open = false
+    focus = false
+    iconFocused = false
+    updateValue(`{{ env.${variable} }}`)
+  }
+
+  onMount(() => {
+    focus = autofocus
+    if (focus) field.focus()
+  })
+
+  function removeVariable() {
+    updateValue("")
+  }
+
+  function openPopover() {
+    open = true
+    focus = true
+    iconFocused = true
+  }
+</script>
+
+<div class="spectrum-InputGroup">
+  <div
+    class:is-disabled={disabled || hbsValue.length}
+    class:is-focused={focus}
+    class="spectrum-Textfield"
+  >
+    <svg
+      class:close-color={hbsValue.length}
+      class:focused={iconFocused}
+      class="hoverable icon-position spectrum-Icon spectrum-Icon--sizeS spectrum-Textfield-validationIcon"
+      focusable="false"
+      aria-hidden="true"
+      on:click={() => {
+        hbsValue.length ? removeVariable() : openPopover()
+      }}
+    >
+      <use
+        xlink:href={`#spectrum-icon-18-${!hbsValue.length ? "Key" : "Close"}`}
+      />
+    </svg>
+
+    <input
+      bind:this={field}
+      disabled={hbsValue.length || disabled}
+      {readonly}
+      {id}
+      data-cy={dataCy}
+      value={hbsValue.length ? `{{ ${hbsValue[0]} }}` : value}
+      placeholder={placeholder || ""}
+      on:click
+      on:blur
+      on:focus
+      on:input
+      on:keyup
+      on:blur={onBlur}
+      on:focus={onFocus}
+      on:input={onInput}
+      {type}
+      style={align ? `text-align: ${align};` : ""}
+      class="spectrum-Textfield-input"
+      inputmode={type === "number" ? "decimal" : "text"}
+    />
+  </div>
+  {#if open}
+    <div
+      use:clickOutside={handleOutsideClick}
+      class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
+    >
+      <ul class="spectrum-Menu" role="listbox">
+        {#if !environmentVariablesEnabled}
+          <div class="no-variables-text primary-text">
+            Upgrade your plan to get environment variables
+          </div>
+        {:else if variables.length}
+          <div style="max-height: 100px">
+            {#each variables as variable, idx}
+              <li
+                class="spectrum-Menu-item"
+                role="option"
+                aria-selected="true"
+                tabindex="0"
+                on:click={() => handleVarSelect(variable.name)}
+              >
+                <span class="spectrum-Menu-itemLabel">
+                  <div class="primary-text">
+                    {variable.name}
+                    <span />
+                  </div>
+                  <svg
+                    class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
+                    focusable="false"
+                    aria-hidden="true"
+                  >
+                    <use xlink:href="#spectrum-css-icon-Checkmark100" />
+                  </svg>
+                </span>
+              </li>
+            {/each}
+          </div>
+        {:else}
+          <div class="no-variables-text primary-text">
+            You don't have any environment variables yet
+          </div>
+        {/if}
+      </ul>
+      <Divider noMargin />
+      {#if environmentVariablesEnabled}
+        <div on:click={() => showModal()} class="add-variable">
+          <svg
+            class="spectrum-Icon spectrum-Icon--sizeS "
+            focusable="false"
+            aria-hidden="true"
+          >
+            <use xlink:href="#spectrum-icon-18-Add" />
+          </svg>
+          <div class="primary-text">Add Variable</div>
+        </div>
+      {:else}
+        <div on:click={() => handleUpgradePanel()} class="add-variable">
+          <svg
+            class="spectrum-Icon spectrum-Icon--sizeS "
+            focusable="false"
+            aria-hidden="true"
+          >
+            <use xlink:href="#spectrum-icon-18-ArrowUp" />
+          </svg>
+          <div class="primary-text">Upgrade plan</div>
+        </div>
+      {/if}
+    </div>
+  {/if}
+</div>
+
+<style>
+  .spectrum-Textfield {
+    width: 100%;
+  }
+
+  .icon-position {
+    position: absolute;
+    top: 25%;
+    right: 2%;
+  }
+
+  .hoverable:hover {
+    cursor: pointer;
+    color: var(--spectrum-global-color-blue-400);
+  }
+
+  .primary-text {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .spectrum-InputGroup {
+    min-width: 0;
+    width: 100%;
+  }
+
+  .spectrum-Popover {
+    max-height: 240px;
+    z-index: 999;
+    top: 100%;
+  }
+
+  .spectrum-Popover.spectrum-Popover--bottom.spectrum-Picker-popover.is-open {
+    width: 100%;
+  }
+
+  .no-variables-height {
+    height: 100px;
+  }
+
+  .no-variables-text {
+    padding: var(--spacing-m);
+    color: var(--spectrum-global-color-gray-600);
+  }
+
+  .add-variable {
+    display: flex;
+    padding: var(--spacing-m) 0 var(--spacing-m) var(--spacing-m);
+    align-items: center;
+    gap: var(--spacing-s);
+    cursor: pointer;
+  }
+
+  .focused {
+    color: var(--spectrum-global-color-blue-400);
+  }
+
+  .add-variable:hover {
+    background: var(--grey-1);
+  }
+
+  .close-color {
+    color: var(--spectrum-global-color-gray-900) !important;
+  }
+
+  .close-color:hover {
+    color: var(--spectrum-global-color-blue-400) !important;
+  }
+</style>

packages/bbui/src/Form/EnvDropdown.svelte

@@ -0,0 +1,52 @@
+<script>
+  import Field from "./Field.svelte"
+  import EnvDropdown from "./Core/EnvDropdown.svelte"
+  import { createEventDispatcher } from "svelte"
+
+  export let value = null
+  export let label = null
+  export let labelPosition = "above"
+  export let placeholder = null
+  export let type = "text"
+  export let disabled = false
+  export let readonly = false
+  export let error = null
+  export let updateOnChange = true
+  export let quiet = false
+  export let dataCy
+  export let autofocus
+  export let variables
+  export let showModal
+  export let environmentVariablesEnabled
+  export let handleUpgradePanel
+  const dispatch = createEventDispatcher()
+  const onChange = e => {
+    value = e.detail
+    dispatch("change", e.detail)
+  }
+</script>
+
+<Field {label} {labelPosition} {error}>
+  <EnvDropdown
+    {dataCy}
+    {updateOnChange}
+    {error}
+    {disabled}
+    {readonly}
+    {value}
+    {placeholder}
+    {type}
+    {quiet}
+    {autofocus}
+    {variables}
+    {showModal}
+    {environmentVariablesEnabled}
+    {handleUpgradePanel}
+    on:change={onChange}
+    on:click
+    on:input
+    on:blur
+    on:focus
+    on:keyup
+  />
+</Field>

packages/bbui/src/index.js

@@ -27,6 +27,7 @@ export { default as RadioGroup } from "./Form/RadioGroup.svelte"
 export { default as Checkbox } from "./Form/Checkbox.svelte"
 export { default as InputDropdown } from "./Form/InputDropdown.svelte"
 export { default as PickerDropdown } from "./Form/PickerDropdown.svelte"
+export { default as EnvDropdown } from "./Form/EnvDropdown.svelte"
 export { default as DetailSummary } from "./DetailSummary/DetailSummary.svelte"
 export { default as Popover } from "./Popover/Popover.svelte"
 export { default as ProgressBar } from "./ProgressBar/ProgressBar.svelte"

packages/builder/src/builderStore/dataBinding.js

@@ -21,6 +21,7 @@ import {
 import { TableNames } from "../constants"
 import { JSONUtils } from "@budibase/frontend-core"
 import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json"
+import { environment, licensing } from "stores/portal"
 
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
@@ -53,8 +54,13 @@ export const getBindableProperties = (asset, componentId) => {
  * Gets all rest bindable data fields
  */
 export const getRestBindings = () => {
+  const environmentVariablesEnabled = get(licensing).environmentVariablesEnabled
   const userBindings = getUserBindings()
-  return [...userBindings, ...getAuthBindings()]
+  return [
+    ...userBindings,
+    ...getAuthBindings(),
+    ...(environmentVariablesEnabled ? getEnvironmentBindings() : []),
+  ]
 }
 
 /**
@@ -89,6 +95,20 @@ export const getAuthBindings = () => {
   return bindings
 }
 
+export const getEnvironmentBindings = () => {
+  let envVars = get(environment).variables
+  return envVars.map(variable => {
+    return {
+      type: "context",
+      runtimeBinding: `env.${makePropSafe(variable.name)}`,
+      readableBinding: `env.${variable.name}`,
+      category: "Environment",
+      icon: "Key",
+      display: { type: "string", name: variable.name },
+    }
+  })
+}
+
 /**
  * Utility - convert a key/value map to an array of custom 'context' bindings
  * @param {object} valueMap Key/value pairings

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -18,6 +18,7 @@
 
   import { automationStore } from "builderStore"
   import { tables } from "stores/backend"
+  import { environment, licensing } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
@@ -33,6 +34,7 @@
   import { Utils } from "@budibase/frontend-core"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
   import { cloneDeep } from "lodash/fp"
+  import { onMount } from "svelte"
 
   export let block
   export let testData
@@ -166,6 +168,24 @@
       )
     }
 
+    // Environment bindings
+    if ($licensing.environmentVariablesEnabled) {
+      bindings = bindings.concat(
+        $environment.variables.map(variable => {
+          return {
+            label: `env.${variable.name}`,
+            path: `env.${variable.name}`,
+            icon: "Key",
+            category: "Environment",
+            display: {
+              type: "string",
+              name: variable.name,
+            },
+          }
+        })
+      )
+    }
+
     return bindings
   }
 
@@ -196,6 +216,14 @@
     onChange({ detail: tempFilters }, defKey)
     drawer.hide()
   }
+
+  onMount(async () => {
+    try {
+      await environment.loadVariables()
+    } catch (error) {
+      console.error(error)
+    }
+  })
 </script>
 
 <div class="fields">

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte

@@ -6,17 +6,26 @@
     Toggle,
     Button,
     TextArea,
+    Modal,
+    EnvDropdown,
     Accordion,
+    notifications,
   } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { capitalise } from "helpers"
   import { IntegrationTypes } from "constants/backend"
   import { createValidationStore } from "helpers/validation/yup"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onMount } from "svelte"
+  import { environment, licensing, auth } from "stores/portal"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
 
   export let datasource
   export let schema
   export let creating
+
+  let createVariableModal
+  let selectedKey
+
   const validation = createValidationStore()
   const dispatch = createEventDispatcher()
 
@@ -70,6 +79,37 @@
       .filter(el => filter(el))
       .map(([key]) => key)
   }
+
+  async function save(data) {
+    try {
+      await environment.createVariable(data)
+      config[selectedKey] = `{{ env.${data.name} }}`
+      createVariableModal.hide()
+    } catch (err) {
+      notifications.error(`Failed to create variable: ${err.message}`)
+    }
+  }
+
+  function showModal(configKey) {
+    selectedKey = configKey
+    createVariableModal.show()
+  }
+
+  async function handleUpgradePanel() {
+    await environment.upgradePanelOpened()
+    $licensing.goToUpgradePage()
+  }
+
+  onMount(async () => {
+    try {
+      await environment.loadVariables()
+      if ($auth.user) {
+        await licensing.init()
+      }
+    } catch (err) {
+      console.error(err)
+    }
+  })
 </script>
 
 <form>
@@ -134,11 +174,15 @@
       {:else}
         <div class="form-row">
           <Label>{getDisplayName(configKey)}</Label>
-          <Input
+          <EnvDropdown
+            showModal={() => showModal(configKey)}
+            variables={$environment.variables}
             type={schema[configKey].type}
             on:change
             bind:value={config[configKey]}
             error={$validation.errors[configKey]}
+            environmentVariablesEnabled={$licensing.environmentVariablesEnabled}
+            {handleUpgradePanel}
           />
         </div>
       {/if}
@@ -146,6 +190,10 @@
   </Layout>
 </form>
 
+<Modal bind:this={createVariableModal}>
+  <CreateEditVariableModal {save} />
+</Modal>
+
 <style>
   .form-row {
     display: grid;

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte

@@ -12,10 +12,12 @@
   import ViewDynamicVariables from "./variables/ViewDynamicVariables.svelte"
   import {
     getRestBindings,
+    getEnvironmentBindings,
     readableToRuntimeBinding,
     runtimeToReadableMap,
   } from "builderStore/dataBinding"
   import { cloneDeep } from "lodash/fp"
+  import { licensing } from "stores/portal"
 
   export let datasource
   export let queries
@@ -93,6 +95,9 @@
     headings
     bind:object={datasource.config.staticVariables}
     on:change
+    bindings={$licensing.environmentVariablesEnabled
+      ? getEnvironmentBindings()
+      : []}
   />
 </Layout>
 <div />

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte

@@ -1,9 +1,23 @@
 <script>
   import { onMount } from "svelte"
-  import { ModalContent, Layout, Select, Body, Input } from "@budibase/bbui"
+  import {
+    ModalContent,
+    Layout,
+    Select,
+    Body,
+    Input,
+    EnvDropdown,
+    Modal,
+    notifications,
+  } from "@budibase/bbui"
   import { AUTH_TYPE_LABELS, AUTH_TYPES } from "./authTypes"
   import BindableCombobox from "components/common/bindings/BindableCombobox.svelte"
-  import { getAuthBindings } from "builderStore/dataBinding"
+  import {
+    getAuthBindings,
+    getEnvironmentBindings,
+  } from "builderStore/dataBinding"
+  import { environment, licensing, auth } from "stores/portal"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
 
   export let configs
   export let currentConfig
@@ -28,7 +42,19 @@
   let hasErrors = false
   let hasChanged = false
 
-  onMount(() => {
+  let createVariableModal
+  let formFieldkey
+
+  onMount(async () => {
+    try {
+      await environment.loadVariables()
+      if ($auth.user) {
+        await licensing.init()
+      }
+    } catch (err) {
+      console.error(err)
+    }
+
     if (currentConfig) {
       deconstructConfig()
     }
@@ -146,6 +172,16 @@
     }
   }
 
+  const save = async data => {
+    try {
+      await environment.createVariable(data)
+      form.basic[formFieldkey] = `{{ env.${data.name} }}`
+      createVariableModal.hide()
+    } catch (err) {
+      notifications.error(`Failed to create variable: ${err.message}`)
+    }
+  }
+
   const onFieldChange = () => {
     checkErrors()
     checkChanged()
@@ -154,6 +190,16 @@
   const onConfirmInternal = () => {
     onConfirm(constructConfig())
   }
+
+  async function handleUpgradePanel() {
+    await environment.upgradePanelOpened()
+    $licensing.goToUpgradePage()
+  }
+
+  function showModal(key) {
+    formFieldkey = key
+    createVariableModal.show()
+  }
 </script>
 
 <ModalContent
@@ -189,26 +235,39 @@
       error={blurred.type ? errors.type : null}
     />
     {#if form.type === AUTH_TYPES.BASIC}
-      <Input
+      <EnvDropdown
         label="Username"
         bind:value={form.basic.username}
         on:change={onFieldChange}
         on:blur={() => (blurred.basic.username = true)}
         error={blurred.basic.username ? errors.basic.username : null}
+        showModal={() => showModal("configKey")}
+        variables={$environment.variables}
+        environmentVariablesEnabled={$licensing.environmentVariablesEnabled}
+        {handleUpgradePanel}
       />
-      <Input
+      <EnvDropdown
         label="Password"
         bind:value={form.basic.password}
         on:change={onFieldChange}
         on:blur={() => (blurred.basic.password = true)}
         error={blurred.basic.password ? errors.basic.password : null}
+        showModal={() => showModal("configKey")}
+        variables={$environment.variables}
+        environmentVariablesEnabled={$licensing.environmentVariablesEnabled}
+        {handleUpgradePanel}
       />
     {/if}
     {#if form.type === AUTH_TYPES.BEARER}
       <BindableCombobox
         label="Token"
         value={form.bearer.token}
-        bindings={getAuthBindings()}
+        bindings={[
+          ...getAuthBindings(),
+          ...($licensing.environmentVariablesEnabled
+            ? getEnvironmentBindings()
+            : []),
+        ]}
         on:change={e => {
           form.bearer.token = e.detail
           onFieldChange()
@@ -226,3 +285,7 @@
     {/if}
   </Layout>
 </ModalContent>
+
+<Modal bind:this={createVariableModal}>
+  <CreateEditVariableModal {save} />
+</Modal>

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -1,6 +1,7 @@
 <script>
   import { goto, params } from "@roxi/routify"
   import { datasources, flags, integrations, queries } from "stores/backend"
+  import { environment } from "stores/portal"
   import {
     Banner,
     Body,
@@ -362,6 +363,13 @@
       notifications.error("Error getting datasources")
     }
 
+    try {
+      // load the environment variables
+      await environment.loadVariables()
+    } catch (error) {
+      notifications.error(`Error getting environment variables - ${error}`)
+    }
+
     datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
     const datasourceUrl = datasource?.config.url
     const qs = query?.fields.queryString

packages/builder/src/components/portal/environment/CreateEditVariableModal.svelte

@@ -0,0 +1,100 @@
+<script>
+  import {
+    ModalContent,
+    Button,
+    Input,
+    Checkbox,
+    Heading,
+    notifications,
+    Context,
+  } from "@budibase/bbui"
+  import { environment } from "stores/portal"
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { getContext } from "svelte"
+  const modalContext = getContext(Context.Modal)
+
+  export let save
+  export let row
+
+  let deleteDialog
+  let name = row?.name || ""
+  let productionValue
+  let developmentValue
+  let useProductionValue = true
+
+  const deleteVariable = async name => {
+    try {
+      await environment.deleteVariable(name)
+      modalContext.hide()
+      notifications.success("Environment variable deleted")
+    } catch (err) {
+      notifications.error(err.message)
+    }
+  }
+
+  const saveVariable = async () => {
+    try {
+      await save({
+        name,
+        production: productionValue,
+        development: developmentValue,
+      })
+      notifications.success("Environment variable saved")
+    } catch (err) {
+      notifications.error(`Error saving environment variable - ${err.message}`)
+    }
+  }
+</script>
+
+<ModalContent
+  onConfirm={() => saveVariable()}
+  title={!row ? "Add new environment variable" : "Edit environment variable"}
+>
+  <Input disabled={row} label="Name" bind:value={name} />
+  <div>
+    <Heading size="XS">Production</Heading>
+    <Input
+      type="password"
+      label="Value"
+      on:change={e => {
+        productionValue = e.detail
+        if (useProductionValue) {
+          developmentValue = e.detail
+        }
+      }}
+      value={productionValue}
+    />
+  </div>
+  <div>
+    <Heading size="XS">Development</Heading>
+    <Input
+      type="password"
+      on:change={e => {
+        developmentValue = e.detail
+      }}
+      disabled={useProductionValue}
+      label="Value"
+      value={useProductionValue ? productionValue : developmentValue}
+    />
+    <Checkbox bind:value={useProductionValue} text="Use production value" />
+  </div>
+
+  <div class="footer" slot="footer">
+    {#if row}
+      <Button on:click={deleteDialog.show} warning>Delete</Button>
+    {/if}
+  </div>
+</ModalContent>
+
+<ConfirmDialog
+  bind:this={deleteDialog}
+  onOk={() => {
+    deleteVariable(row.name)
+  }}
+  okText="Delete Environment Variable"
+  title="Confirm Deletion"
+>
+  Are you sure you wish to delete the environment variable
+  <i>{row.name}?</i>
+  This action cannot be undone.
+</ConfirmDialog>

packages/builder/src/components/start/ExportAppModal.svelte

@@ -1,5 +1,6 @@
 <script>
-  import { ModalContent, Toggle, Body } from "@budibase/bbui"
+  import { ModalContent, Toggle, Body, InlineAlert } from "@budibase/bbui"
+  import { licensing } from "stores/portal"
 
   export let app
   export let published
@@ -16,6 +17,11 @@
 </script>
 
 <ModalContent {title} {confirmText} onConfirm={exportApp}>
+  {#if licensing.environmentVariablesEnabled}
+    <InlineAlert
+      header="Do not share your budibase application exports publicly as they may contain sensitive information such as database credentials or secret keys."
+    />
+  {/if}
   <Body
     >Apps can be exported with or without data that is within internal tables -
     select this below.</Body

packages/builder/src/pages/builder/portal/settings/environment/_components/EditVariableColumn.svelte

@@ -0,0 +1,35 @@
+<script>
+  import { ActionButton, Modal } from "@budibase/bbui"
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { environment } from "stores/portal"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
+
+  export let row
+
+  let editVariableModal
+  let deleteDialog
+
+  const save = async data => {
+    await environment.updateVariable(data)
+    editVariableModal.hide()
+  }
+</script>
+
+<ActionButton size="S" on:click={editVariableModal.show}>Edit</ActionButton>
+
+<Modal bind:this={editVariableModal}>
+  <CreateEditVariableModal {row} {save} />
+</Modal>
+
+<ConfirmDialog
+  bind:this={deleteDialog}
+  onOk={async () => {
+    await environment.deleteVariable(row.name)
+  }}
+  okText="Delete Environment Variable"
+  title="Confirm Deletion"
+>
+  Are you sure you wish to delete the environment variable
+  <i>{row.name}?</i>
+  This action cannot be undone.
+</ConfirmDialog>

packages/builder/src/pages/builder/portal/settings/environment/index.svelte

@@ -0,0 +1,145 @@
+<script>
+  import {
+    Layout,
+    Heading,
+    Body,
+    Button,
+    Divider,
+    Modal,
+    Table,
+    Tags,
+    Tag,
+    InlineAlert,
+    notifications,
+  } from "@budibase/bbui"
+  import { environment, licensing, auth, admin } from "stores/portal"
+  import { onMount } from "svelte"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
+  import EditVariableColumn from "./_components/EditVariableColumn.svelte"
+
+  let modal
+
+  const customRenderers = [{ column: "edit", component: EditVariableColumn }]
+
+  $: noEncryptionKey = $environment.status?.encryptionKeyAvailable === false
+  $: schema = buildSchema(noEncryptionKey)
+
+  onMount(async () => {
+    await environment.checkStatus()
+    await environment.loadVariables()
+  })
+
+  const buildSchema = noEncryptionKey => {
+    const schema = {
+      name: {
+        width: "2fr",
+      },
+    }
+    if (!noEncryptionKey) {
+      schema.edit = {
+        width: "auto",
+        borderLeft: true,
+        displayName: "",
+      }
+    }
+    return schema
+  }
+
+  const save = async data => {
+    try {
+      await environment.createVariable(data)
+      modal.hide()
+    } catch (err) {
+      notifications.error(`Error saving variable: ${err.message}`)
+    }
+  }
+</script>
+
+<Layout noPadding>
+  <Layout gap="XS" noPadding>
+    <div class="title">
+      <Heading size="M">Environment Variables</Heading>
+      {#if !$licensing.environmentVariablesEnabled}
+        <Tags>
+          <Tag icon="LockClosed">Business plan</Tag>
+        </Tags>
+      {/if}
+    </div>
+    <Body
+      >Add and manage environment variables for development and production</Body
+    >
+  </Layout>
+  <Divider size="S" />
+
+  {#if $licensing.environmentVariablesEnabled}
+    {#if noEncryptionKey}
+      <InlineAlert
+        message="Your Budibase installation does not have a key for encryption, please update your app service's environment variables to contain an 'ENCRYPTION_KEY' value."
+        header="No encryption key found"
+        type="error"
+      />
+    {/if}
+    <div>
+      <Button on:click={modal.show} cta disabled={noEncryptionKey}
+        >Add Variable</Button
+      >
+    </div>
+
+    <Layout noPadding>
+      <Table
+        {schema}
+        data={$environment.variables}
+        allowEditColumns={false}
+        allowEditRows={false}
+        allowSelectRows={false}
+        {customRenderers}
+      />
+    </Layout>
+  {:else}
+    <div class="buttons">
+      <Button
+        primary
+        disabled={!$auth.accountPortalAccess && $admin.cloud}
+        on:click={async () => {
+          await environment.upgradePanelOpened()
+          $licensing.goToUpgradePage()
+        }}
+      >
+        Upgrade
+      </Button>
+      <!--Show the view plans button-->
+      <Button
+        secondary
+        on:click={() => {
+          window.open("https://budibase.com/pricing/", "_blank")
+        }}
+      >
+        View Plans
+      </Button>
+    </div>
+  {/if}
+</Layout>
+
+<Modal bind:this={modal}>
+  <CreateEditVariableModal {save} />
+</Modal>
+
+<style>
+  .buttons {
+    display: flex;
+    gap: var(--spacing-l);
+  }
+  .title {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: flex-start;
+    gap: var(--spacing-m);
+  }
+
+  .buttons {
+    display: flex;
+    flex-direction: row;
+    gap: var(--spacing-m);
+  }
+</style>

packages/builder/src/stores/backend/datasources.js

@@ -28,7 +28,7 @@ export function createDatasourcesStore() {
     }))
   }
 
-  const updateDatasource = async response => {
+  const updateDatasource = response => {
     const { datasource, error } = response
     store.update(state => {
       const currentIdx = state.list.findIndex(ds => ds._id === datasource._id)
@@ -52,7 +52,7 @@ export function createDatasourcesStore() {
       datasourceId: datasource?._id,
       tablesFilter,
     })
-    return await updateDatasource(response)
+    return updateDatasource(response)
   }
 
   const save = async (body, fetchSchema = false) => {

packages/builder/src/stores/portal/environment.js

@@ -0,0 +1,68 @@
+import { writable } from "svelte/store"
+import { API } from "api"
+import { Constants } from "@budibase/frontend-core"
+
+export function createEnvironmentStore() {
+  const { subscribe, update } = writable({
+    variables: [],
+    status: {},
+  })
+
+  async function checkStatus() {
+    const status = await API.checkEnvironmentVariableStatus()
+    update(store => {
+      store.status = status
+      return store
+    })
+  }
+
+  async function loadVariables() {
+    const envVars = await API.fetchEnvironmentVariables()
+    const mappedVars = envVars.variables.map(name => ({ name }))
+    update(store => {
+      store.variables = mappedVars
+      return store
+    })
+  }
+
+  async function createVariable(data) {
+    await API.createEnvironmentVariable(data)
+    let mappedVar = { name: data.name }
+    update(store => {
+      store.variables = [mappedVar, ...store.variables]
+      return store
+    })
+  }
+
+  async function deleteVariable(varName) {
+    await API.deleteEnvironmentVariable(varName)
+    update(store => {
+      store.variables = store.variables.filter(
+        envVar => envVar.name !== varName
+      )
+      return store
+    })
+  }
+
+  async function updateVariable(data) {
+    await API.updateEnvironmentVariable(data)
+  }
+
+  async function upgradePanelOpened() {
+    await API.publishEvent(
+      Constants.EventPublishType.ENV_VAR_UPGRADE_PANEL_OPENED
+    )
+  }
+
+  return {
+    subscribe,
+    checkStatus,
+    loadVariables,
+    createVariable,
+    deleteVariable,
+    updateVariable,
+    upgradePanelOpened,
+  }
+}
+
+export const environment = createEnvironmentStore()

packages/builder/src/stores/portal/index.js

@@ -11,4 +11,5 @@ export { groups } from "./groups"
 export { plugins } from "./plugins"
 export { backups } from "./backups"
 export { overview } from "./overview"
+export { environment } from "./environment"
 export { menu } from "./menu"

packages/builder/src/stores/portal/licensing.js

@@ -60,6 +60,9 @@ export const createLicensingStore = () => {
       const backupsEnabled = license.features.includes(
         Constants.Features.BACKUPS
       )
+      const environmentVariablesEnabled = license.features.includes(
+        Constants.Features.ENVIRONMENT_VARIABLES
+      )
 
       store.update(state => {
         return {
@@ -68,6 +71,7 @@ export const createLicensingStore = () => {
           isFreePlan,
           groupsEnabled,
           backupsEnabled,
+          environmentVariablesEnabled,
         }
       })
     },

packages/builder/src/stores/portal/menu.js

@@ -50,6 +50,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         title: "Organisation",
         href: "/builder/portal/settings/organisation",
       },
+      {
+        title: "Environment",
+        href: "/builder/portal/settings/environment",
+      },
     ]
     if (!$admin.cloud) {
       settingsSubPages.push({

packages/frontend-core/src/api/environmentVariables.js

@@ -0,0 +1,36 @@
+export const buildEnvironmentVariableEndpoints = API => ({
+  checkEnvironmentVariableStatus: async () => {
+    return await API.get({
+      url: `/api/env/variables/status`,
+    })
+  },
+
+  /**
+   * Fetches a list of environment variables
+   */
+  fetchEnvironmentVariables: async () => {
+    return await API.get({
+      url: `/api/env/variables`,
+      json: false,
+    })
+  },
+
+  createEnvironmentVariable: async data => {
+    return await API.post({
+      url: `/api/env/variables`,
+      body: data,
+    })
+  },
+  deleteEnvironmentVariable: async varName => {
+    return await API.delete({
+      url: `/api/env/variables/${varName}`,
+    })
+  },
+
+  updateEnvironmentVariable: async data => {
+    return await API.patch({
+      url: `/api/env/variables/${data.name}`,
+      body: data,
+    })
+  },
+})

packages/frontend-core/src/api/events.js

@@ -0,0 +1,13 @@
+export const buildEventEndpoints = API => ({
+  /**
+   * Publish a specific event to the backend.
+   */
+  publishEvent: async eventType => {
+    return await API.post({
+      url: `/api/global/event/publish`,
+      body: {
+        type: eventType,
+      },
+    })
+  },
+})

packages/frontend-core/src/api/index.js

@@ -26,7 +26,8 @@ import { buildLicensingEndpoints } from "./licensing"
 import { buildGroupsEndpoints } from "./groups"
 import { buildPluginEndpoints } from "./plugins"
 import { buildBackupsEndpoints } from "./backups"
-
+import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
+import { buildEventEndpoints } from "./events"
 const defaultAPIClientConfig = {
   /**
    * Certain definitions can't change at runtime for client apps, such as the
@@ -247,5 +248,7 @@ export const createAPIClient = config => {
     ...buildGroupsEndpoints(API),
     ...buildPluginEndpoints(API),
     ...buildBackupsEndpoints(API),
+    ...buildEnvironmentVariableEndpoints(API),
+    ...buildEventEndpoints(API),
   }
 }

packages/frontend-core/src/constants.js

@@ -114,6 +114,7 @@ export const ApiVersion = "1"
 export const Features = {
   USER_GROUPS: "userGroups",
   BACKUPS: "appBackups",
+  ENVIRONMENT_VARIABLES: "environmentVariables",
 }
 
 // Role IDs
@@ -174,3 +175,7 @@ export const Themes = [
     base: "darkest",
   },
 ]
+
+export const EventPublishType = {
+  ENV_VAR_UPGRADE_PANEL_OPENED: "environment_variable_upgrade_panel_opened",
+}

packages/server/__mocks__/node-fetch.ts

@@ -1,5 +1,7 @@
+// @ts-ignore
 import fs from "fs"
 module FetchMock {
+  // @ts-ignore
   const fetch = jest.requireActual("node-fetch")
   let failCount = 0
 

packages/server/scripts/dev/manage.js

@@ -29,6 +29,7 @@ async function init() {
       ACCOUNT_PORTAL_URL: "http://localhost:10001",
       ACCOUNT_PORTAL_API_KEY: "budibase",
       JWT_SECRET: "testsecret",
+      ENCRYPTION_KEY: "testsecret",
       REDIS_PASSWORD: "budibase",
       MINIO_ACCESS_KEY: "budibase",
       MINIO_SECRET_KEY: "budibase",

packages/server/src/api/controllers/application.ts

@@ -112,12 +112,11 @@ function checkAppName(
   }
 }
 
-async function createInstance(template: any, includeSampleData: boolean) {
-  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
-  const baseAppId = generateAppID(tenantId)
-  const appId = generateDevAppID(baseAppId)
-  await context.updateAppId(appId)
-
+async function createInstance(
+  appId: string,
+  template: any,
+  includeSampleData: boolean
+) {
   const db = context.getAppDB()
   await db.put({
     _id: "_design/database",
@@ -250,82 +249,90 @@ async function performAppCreate(ctx: BBContext) {
     instanceConfig.file = ctx.request.files.templateFile
   }
   const includeSampleData = isQsTrue(ctx.request.body.sampleData)
-  const instance = await createInstance(instanceConfig, includeSampleData)
-  const appId = instance._id
-  const db = context.getAppDB()
+  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
+  const appId = generateDevAppID(generateAppID(tenantId))
 
-  let newApplication: App = {
-    _id: DocumentType.APP_METADATA,
-    _rev: undefined,
-    appId,
-    type: "app",
-    version: packageJson.version,
-    componentLibraries: ["@budibase/standard-components"],
-    name: name,
-    url: url,
-    template: templateKey,
-    instance,
-    tenantId: tenancy.getTenantId(),
-    updatedAt: new Date().toISOString(),
-    createdAt: new Date().toISOString(),
-    status: AppStatus.DEV,
-    navigation: {
-      navigation: "Top",
-      title: name,
-      navWidth: "Large",
-      navBackground: "var(--spectrum-global-color-gray-100)",
-      links: [
-        {
-          url: "/home",
-          text: "Home",
-        },
-      ],
-    },
-    theme: "spectrum--light",
-    customTheme: {
-      buttonBorderRadius: "16px",
-    },
-  }
+  return await context.doInAppContext(appId, async () => {
+    const instance = await createInstance(
+      appId,
+      instanceConfig,
+      includeSampleData
+    )
+    const db = context.getAppDB()
 
-  // If we used a template or imported an app there will be an existing doc.
-  // Fetch and migrate some metadata from the existing app.
-  try {
-    const existing: App = await db.get(DocumentType.APP_METADATA)
-    const keys: (keyof App)[] = [
-      "_rev",
-      "navigation",
-      "theme",
-      "customTheme",
-      "icon",
-    ]
-    keys.forEach(key => {
-      if (existing[key]) {
-        // @ts-ignore
-        newApplication[key] = existing[key]
-      }
-    })
-
-    // Migrate navigation settings and screens if required
-    if (existing) {
-      const navigation = await migrateAppNavigation()
-      if (navigation) {
-        newApplication.navigation = navigation
-      }
+    let newApplication: App = {
+      _id: DocumentType.APP_METADATA,
+      _rev: undefined,
+      appId,
+      type: "app",
+      version: packageJson.version,
+      componentLibraries: ["@budibase/standard-components"],
+      name: name,
+      url: url,
+      template: templateKey,
+      instance,
+      tenantId: tenancy.getTenantId(),
+      updatedAt: new Date().toISOString(),
+      createdAt: new Date().toISOString(),
+      status: AppStatus.DEV,
+      navigation: {
+        navigation: "Top",
+        title: name,
+        navWidth: "Large",
+        navBackground: "var(--spectrum-global-color-gray-100)",
+        links: [
+          {
+            url: "/home",
+            text: "Home",
+          },
+        ],
+      },
+      theme: "spectrum--light",
+      customTheme: {
+        buttonBorderRadius: "16px",
+      },
     }
-  } catch (err) {
-    // Nothing to do
-  }
 
-  const response = await db.put(newApplication, { force: true })
-  newApplication._rev = response.rev
+    // If we used a template or imported an app there will be an existing doc.
+    // Fetch and migrate some metadata from the existing app.
+    try {
+      const existing: App = await db.get(DocumentType.APP_METADATA)
+      const keys: (keyof App)[] = [
+        "_rev",
+        "navigation",
+        "theme",
+        "customTheme",
+        "icon",
+      ]
+      keys.forEach(key => {
+        if (existing[key]) {
+          // @ts-ignore
+          newApplication[key] = existing[key]
+        }
+      })
 
-  /* istanbul ignore next */
-  if (!env.isTest()) {
-    await createApp(appId)
-  }
+      // Migrate navigation settings and screens if required
+      if (existing) {
+        const navigation = await migrateAppNavigation()
+        if (navigation) {
+          newApplication.navigation = navigation
+        }
+      }
+    } catch (err) {
+      // Nothing to do
+    }
 
-  await cache.app.invalidateAppMetadata(appId, newApplication)
-  return newApplication
+    const response = await db.put(newApplication, { force: true })
+    newApplication._rev = response.rev
+
+    /* istanbul ignore next */
+    if (!env.isTest()) {
+      await createApp(appId)
+    }
+
+    await cache.app.invalidateAppMetadata(appId, newApplication)
+    return newApplication
+  })
 }
 
 async function creationEvents(request: any, app: App) {

packages/server/src/api/controllers/datasource.ts

@@ -12,9 +12,11 @@ import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
-import { BBContext, Datasource, Row } from "@budibase/types"
+import { UserCtx, Datasource, Row } from "@budibase/types"
+import sdk from "../../sdk"
+import { mergeConfigs } from "../../sdk/app/datasources/datasources"
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
   const internalTables = await db.allDocs(
@@ -43,25 +45,23 @@ export async function fetch(ctx: BBContext) {
     )
   ).rows.map(row => row.doc)
 
-  const allDatasources = [bbInternalDb, ...datasources]
+  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
+    bbInternalDb,
+    ...datasources,
+  ])
 
   for (let datasource of allDatasources) {
-    if (datasource.config && datasource.config.auth) {
-      // strip secrets from response so they don't show in the network request
-      delete datasource.config.auth
-    }
-
     if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
-      datasource.entities = internal[datasource._id]
+      datasource.entities = internal[datasource._id!]
     }
   }
 
   ctx.body = [bbInternalDb, ...datasources]
 }
 
-export async function buildSchemaFromDb(ctx: BBContext) {
+export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const datasource = await db.get(ctx.params.datasourceId)
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
 
   let { tables, error } = await buildSchemaHelper(datasource)
@@ -146,11 +146,11 @@ async function invalidateVariables(
   await invalidateDynamicVariables(toInvalidate)
 }
 
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
-  let datasource = await db.get(datasourceId)
-  const auth = datasource.config.auth
+  let datasource = await sdk.datasources.get(datasourceId)
+  const auth = datasource.config?.auth
   await invalidateVariables(datasource, ctx.request.body)
 
   const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
@@ -159,10 +159,13 @@ export async function update(ctx: BBContext) {
     ? { name: ctx.request.body?.name }
     : ctx.request.body
 
-  datasource = { ...datasource, ...dataSourceBody }
+  datasource = {
+    ...datasource,
+    ...sdk.datasources.mergeConfigs(dataSourceBody, datasource),
+  }
   if (auth && !ctx.request.body.auth) {
     // don't strip auth config from DB
-    datasource.config.auth = auth
+    datasource.config!.auth = auth
   }
 
   const response = await db.put(datasource)
@@ -179,10 +182,12 @@ export async function update(ctx: BBContext) {
 
   ctx.status = 200
   ctx.message = "Datasource saved successfully."
-  ctx.body = { datasource }
+  ctx.body = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
@@ -213,7 +218,9 @@ export async function save(ctx: BBContext) {
     }
   }
 
-  const response: any = { datasource }
+  const response: any = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
   if (schemaError) {
     response.error = schemaError
   }
@@ -251,11 +258,11 @@ async function destroyInternalTablesBySourceId(datasourceId: string) {
   }
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
 
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   // Delete all queries for the datasource
 
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
@@ -279,13 +286,14 @@ export async function destroy(ctx: BBContext) {
   ctx.status = 200
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const database = context.getAppDB()
-  ctx.body = await database.get(ctx.params.datasourceId)
+  const datasource = await database.get(ctx.params.datasourceId)
+  ctx.body = await sdk.datasources.removeSecretSingle(datasource)
 }
 
 // dynamic query functionality
-export async function query(ctx: BBContext) {
+export async function query(ctx: UserCtx) {
   const queryJson = ctx.request.body
   try {
     ctx.body = await getDatasourceAndQuery(queryJson)
@@ -313,7 +321,7 @@ function updateError(error: any, newError: any, tables: string[]) {
 
 async function buildSchemaHelper(datasource: Datasource) {
   const Connector = await getIntegration(datasource.source)
-
+  datasource = await sdk.datasources.enrich(datasource)
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
   await connector.buildSchema(datasource._id, datasource.entities)

packages/server/src/api/controllers/query/index.ts

@@ -7,6 +7,8 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
 import env from "../../../environment"
 import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
+import sdk from "../../../sdk"
+import { QueryEvent } from "../../../threads/definitions"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
@@ -81,7 +83,7 @@ export async function save(ctx: any) {
   const db = context.getAppDB()
   const query = ctx.request.body
 
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
 
   let eventFn
   if (!query._id) {
@@ -126,9 +128,9 @@ function getAuthConfig(ctx: any) {
 }
 
 export async function preview(ctx: any) {
-  const db = context.getAppDB()
-
-  const datasource = await db.get(ctx.request.body.datasourceId)
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
+    ctx.request.body.datasourceId
+  )
   const query = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
@@ -137,20 +139,22 @@ export async function preview(ctx: any) {
   const authConfigCtx: any = getAuthConfig(ctx)
 
   try {
-    const runFn = () =>
-      Runner.run({
-        appId: ctx.appId,
-        datasource,
-        queryVerb,
-        fields,
-        parameters,
-        transformer,
-        queryId,
-        ctx: {
-          user: ctx.user,
-          auth: { ...authConfigCtx },
-        },
-      })
+    const inputs: QueryEvent = {
+      appId: ctx.appId,
+      datasource,
+      queryVerb,
+      fields,
+      parameters,
+      transformer,
+      queryId,
+      // have to pass down to the thread runner - can't put into context now
+      environmentVariables: envVars,
+      ctx: {
+        user: ctx.user,
+        auth: { ...authConfigCtx },
+      },
+    }
+    const runFn = () => Runner.run(inputs)
 
     const { rows, keys, info, extra } = await quotas.addQuery(runFn, {
       datasourceId: datasource._id,
@@ -201,7 +205,9 @@ async function execute(
   const db = context.getAppDB()
 
   const query = await db.get(ctx.params.queryId)
-  const datasource = await db.get(query.datasourceId)
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
+    query.datasourceId
+  )
 
   let authConfigCtx: any = {}
   if (!opts.isAutomation) {
@@ -219,21 +225,23 @@ async function execute(
 
   // call the relevant CRUD method on the integration class
   try {
-    const runFn = () =>
-      Runner.run({
-        appId: ctx.appId,
-        datasource,
-        queryVerb: query.queryVerb,
-        fields: query.fields,
-        pagination: ctx.request.body.pagination,
-        parameters: enrichedParameters,
-        transformer: query.transformer,
-        queryId: ctx.params.queryId,
-        ctx: {
-          user: ctx.user,
-          auth: { ...authConfigCtx },
-        },
-      })
+    const inputs: QueryEvent = {
+      appId: ctx.appId,
+      datasource,
+      queryVerb: query.queryVerb,
+      fields: query.fields,
+      pagination: ctx.request.body.pagination,
+      parameters: enrichedParameters,
+      transformer: query.transformer,
+      queryId: ctx.params.queryId,
+      // have to pass down to the thread runner - can't put into context now
+      environmentVariables: envVars,
+      ctx: {
+        user: ctx.user,
+        auth: { ...authConfigCtx },
+      },
+    }
+    const runFn = () => Runner.run(inputs)
 
     const { rows, pagination, extra } = await quotas.addQuery(runFn, {
       datasourceId: datasource._id,
@@ -266,18 +274,18 @@ export async function executeV2(
 const removeDynamicVariables = async (queryId: any) => {
   const db = context.getAppDB()
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
-  const dynamicVariables = datasource.config.dynamicVariables
+  const datasource = await sdk.datasources.get(query.datasourceId)
+  const dynamicVariables = datasource.config?.dynamicVariables as any[]
 
   if (dynamicVariables) {
     // delete dynamic variables from the datasource
-    datasource.config.dynamicVariables = dynamicVariables.filter(
+    datasource.config!.dynamicVariables = dynamicVariables!.filter(
       (dv: any) => dv.queryId !== queryId
     )
     await db.put(datasource)
 
     // invalidate the deleted variables
-    const variablesToDelete = dynamicVariables.filter(
+    const variablesToDelete = dynamicVariables!.filter(
       (dv: any) => dv.queryId === queryId
     )
     await invalidateDynamicVariables(variablesToDelete)
@@ -289,7 +297,7 @@ export async function destroy(ctx: any) {
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
   ctx.message = `Query deleted.`
   ctx.status = 200

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -25,6 +25,7 @@ import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
 import { removeKeyNumbering } from "./utils"
+import sdk from "../../../sdk"
 
 export interface ManyRelationship {
   tableId?: string
@@ -664,8 +665,7 @@ export class ExternalRequest {
       throw "Unable to run without a table name"
     }
     if (!this.datasource) {
-      const db = context.getAppDB()
-      this.datasource = await db.get(datasourceId)
+      this.datasource = await sdk.datasources.get(datasourceId!)
       if (!this.datasource || !this.datasource.entities) {
         throw "No tables found, fetch tables before query."
       }

packages/server/src/api/controllers/row/external.ts

@@ -19,6 +19,7 @@ import {
   Table,
   Datasource,
 } from "@budibase/types"
+import sdk from "../../../sdk"
 
 export async function handleRequest(
   operation: Operation,
@@ -179,10 +180,9 @@ export async function validate(ctx: BBContext) {
 
 export async function exportRows(ctx: BBContext) {
   const { datasourceId } = breakExternalTableId(ctx.params.tableId)
-  const db = context.getAppDB()
   const format = ctx.query.format
   const { columns } = ctx.request.body
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId!)
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
@@ -225,8 +225,7 @@ export async function fetchEnrichedRow(ctx: BBContext) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const db = context.getAppDB()
-  const datasource: Datasource = await db.get(datasourceId)
+  const datasource: Datasource = await sdk.datasources.get(datasourceId!)
   if (!tableName) {
     ctx.throw(400, "Unable to find table.")
   }

packages/server/src/api/controllers/row/utils.ts

@@ -8,6 +8,7 @@ export { removeKeyNumbering } from "../../../integrations/base/utils"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
 import { Ctx } from "@budibase/types"
+import sdk from "../../../sdk"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -21,8 +22,7 @@ validateJs.extend(validateJs.validators.datetime, {
 
 export async function getDatasourceAndQuery(json: any) {
   const datasourceId = json.endpoint.datasourceId
-  const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   return makeExternalQuery(datasource, json)
 }
 

packages/server/src/api/controllers/static/index.ts

@@ -1,20 +1,21 @@
 require("svelte/register")
 
-const send = require("koa-send")
-const { resolve, join } = require("../../../utilities/centralPath")
+import { resolve, join } from "../../../utilities/centralPath"
 const uuid = require("uuid")
 import { ObjectStoreBuckets } from "../../../constants"
-const { processString } = require("@budibase/string-templates")
-const {
+import { processString } from "@budibase/string-templates"
+import {
   loadHandlebarsFile,
   NODE_MODULES_PATH,
   TOP_LEVEL_PATH,
-} = require("../../../utilities/fileSystem")
+} from "../../../utilities/fileSystem"
 import env from "../../../environment"
-const { DocumentType } = require("../../../db/utils")
-const { context, objectStore, utils } = require("@budibase/backend-core")
-const AWS = require("aws-sdk")
-const fs = require("fs")
+import { DocumentType } from "../../../db/utils"
+import { context, objectStore, utils } from "@budibase/backend-core"
+import AWS from "aws-sdk"
+import fs from "fs"
+import sdk from "../../../sdk"
+const send = require("koa-send")
 
 async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   const response = await objectStore.upload({
@@ -110,7 +111,7 @@ export const serveApp = async function (ctx: any) {
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
       usedPlugins: plugins,
     })
 
@@ -135,7 +136,7 @@ export const serveBuilderPreview = async function (ctx: any) {
     let appId = context.getAppId()
     const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
     ctx.body = await processString(previewHbs, {
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
     })
   } else {
     // just return the app info for jest to assert on
@@ -150,13 +151,11 @@ export const serveClientLibrary = async function (ctx: any) {
 }
 
 export const getSignedUploadURL = async function (ctx: any) {
-  const database = context.getAppDB()
-
   // Ensure datasource is valid
   let datasource
   try {
     const { datasourceId } = ctx.params
-    datasource = await database.get(datasourceId)
+    datasource = await sdk.datasources.get(datasourceId, { enriched: true })
     if (!datasource) {
       ctx.throw(400, "The specified datasource could not be found")
     }
@@ -172,8 +171,8 @@ export const getSignedUploadURL = async function (ctx: any) {
   // Determine type of datasource and generate signed URL
   let signedUrl
   let publicUrl
-  const awsRegion = datasource?.config?.region || "eu-west-1"
-  if (datasource.source === "S3") {
+  const awsRegion = (datasource?.config?.region || "eu-west-1") as string
+  if (datasource?.source === "S3") {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
@@ -182,8 +181,8 @@ export const getSignedUploadURL = async function (ctx: any) {
     try {
       const s3 = new AWS.S3({
         region: awsRegion,
-        accessKeyId: datasource?.config?.accessKeyId,
-        secretAccessKey: datasource?.config?.secretAccessKey,
+        accessKeyId: datasource?.config?.accessKeyId as string,
+        secretAccessKey: datasource?.config?.secretAccessKey as string,
         apiVersion: "2006-03-01",
         signatureVersion: "v4",
       })

packages/server/src/api/controllers/table/external.ts

@@ -219,7 +219,7 @@ export async function save(ctx: BBContext) {
   }
 
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   if (!datasource.entities) {
     datasource.entities = {}
   }
@@ -322,15 +322,17 @@ export async function destroy(ctx: BBContext) {
   const datasourceId = getDatasourceId(tableToDelete)
 
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId!)
   const tables = datasource.entities
 
   const operation = Operation.DELETE_TABLE
-  await makeTableRequest(datasource, operation, tableToDelete, tables)
+  if (tables) {
+    await makeTableRequest(datasource, operation, tableToDelete, tables)
+    cleanupRelationships(tableToDelete, tables)
+    delete tables[tableToDelete.name]
+    datasource.entities = tables
+  }
 
-  cleanupRelationships(tableToDelete, tables)
-
-  delete datasource.entities[tableToDelete.name]
   await db.put(datasource)
 
   return tableToDelete

packages/server/src/api/controllers/webhook.ts

@@ -39,60 +39,62 @@ export async function destroy(ctx: BBContext) {
 }
 
 export async function buildSchema(ctx: BBContext) {
-  await context.updateAppId(ctx.params.instance)
-  const db = context.getAppDB()
-  const webhook = (await db.get(ctx.params.id)) as Webhook
-  webhook.bodySchema = toJsonSchema(ctx.request.body)
-  // update the automation outputs
-  if (webhook.action.type === WebhookActionType.AUTOMATION) {
-    let automation = (await db.get(webhook.action.target)) as Automation
-    const autoOutputs = automation.definition.trigger.schema.outputs
-    let properties = webhook.bodySchema.properties
-    // reset webhook outputs
-    autoOutputs.properties = {
-      body: autoOutputs.properties.body,
-    }
-    for (let prop of Object.keys(properties)) {
-      autoOutputs.properties[prop] = {
-        type: properties[prop].type,
-        description: AUTOMATION_DESCRIPTION,
+  await context.doInAppContext(ctx.params.instance, async () => {
+    const db = context.getAppDB()
+    const webhook = (await db.get(ctx.params.id)) as Webhook
+    webhook.bodySchema = toJsonSchema(ctx.request.body)
+    // update the automation outputs
+    if (webhook.action.type === WebhookActionType.AUTOMATION) {
+      let automation = (await db.get(webhook.action.target)) as Automation
+      const autoOutputs = automation.definition.trigger.schema.outputs
+      let properties = webhook.bodySchema.properties
+      // reset webhook outputs
+      autoOutputs.properties = {
+        body: autoOutputs.properties.body,
       }
+      for (let prop of Object.keys(properties)) {
+        autoOutputs.properties[prop] = {
+          type: properties[prop].type,
+          description: AUTOMATION_DESCRIPTION,
+        }
+      }
+      await db.put(automation)
     }
-    await db.put(automation)
-  }
-  ctx.body = await db.put(webhook)
+    ctx.body = await db.put(webhook)
+  })
 }
 
 export async function trigger(ctx: BBContext) {
   const prodAppId = dbCore.getProdAppID(ctx.params.instance)
-  await context.updateAppId(prodAppId)
-  try {
-    const db = context.getAppDB()
-    const webhook = (await db.get(ctx.params.id)) as Webhook
-    // validate against the schema
-    if (webhook.bodySchema) {
-      validate(ctx.request.body, webhook.bodySchema)
-    }
-    const target = await db.get(webhook.action.target)
-    if (webhook.action.type === WebhookActionType.AUTOMATION) {
-      // trigger with both the pure request and then expand it
-      // incase the user has produced a schema to bind to
-      await triggers.externalTrigger(target, {
-        body: ctx.request.body,
-        ...ctx.request.body,
-        appId: prodAppId,
-      })
-    }
-    ctx.status = 200
-    ctx.body = {
-      message: "Webhook trigger fired successfully",
-    }
-  } catch (err: any) {
-    if (err.status === 404) {
+  await context.doInAppContext(prodAppId, async () => {
+    try {
+      const db = context.getAppDB()
+      const webhook = (await db.get(ctx.params.id)) as Webhook
+      // validate against the schema
+      if (webhook.bodySchema) {
+        validate(ctx.request.body, webhook.bodySchema)
+      }
+      const target = await db.get(webhook.action.target)
+      if (webhook.action.type === WebhookActionType.AUTOMATION) {
+        // trigger with both the pure request and then expand it
+        // incase the user has produced a schema to bind to
+        await triggers.externalTrigger(target, {
+          body: ctx.request.body,
+          ...ctx.request.body,
+          appId: prodAppId,
+        })
+      }
       ctx.status = 200
       ctx.body = {
-        message: "Application not deployed yet.",
+        message: "Webhook trigger fired successfully",
+      }
+    } catch (err: any) {
+      if (err.status === 404) {
+        ctx.status = 200
+        ctx.body = {
+          message: "Application not deployed yet.",
+        }
       }
     }
-  }
+  })
 }

packages/server/src/api/routes/index.ts

@@ -33,6 +33,7 @@ export { default as publicRoutes } from "./public"
 
 const appBackupRoutes = pro.appBackups
 const scheduleRoutes = pro.schedules
+const environmentVariableRoutes = pro.environmentVariables
 
 export const mainRoutes: Router[] = [
   appBackupRoutes,
@@ -63,6 +64,7 @@ export const mainRoutes: Router[] = [
   migrationRoutes,
   pluginRoutes,
   scheduleRoutes,
+  environmentVariableRoutes,
   // these need to be handled last as they still use /api/:tableId
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,

packages/server/src/api/routes/row.ts

@@ -3,7 +3,7 @@ import * as rowController from "../controllers/row"
 import authorized from "../../middleware/authorized"
 import { paramResource, paramSubResource } from "../../middleware/resourceId"
 import { permissions } from "@budibase/backend-core"
-const { internalSearchValidator } = require("./utils/validators")
+import { internalSearchValidator } from "./utils/validators"
 const { PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -2,7 +2,8 @@ jest.mock("pg")
 import * as setup from "./utilities"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import { checkCacheForDynamicVariable } from "../../../threads/utils"
-import { events } from "@budibase/backend-core"
+import { context, events } from "@budibase/backend-core"
+import sdk from "../../../sdk"
 
 let { basicDatasource } = setup.structures
 const pg = require("pg")
@@ -184,4 +185,37 @@ describe("/datasources", () => {
       })
     })
   })
+
+  describe("check secret replacement", () => {
+    async function makeDatasource() {
+      datasource = basicDatasource()
+      datasource.datasource.config.password = "testing"
+      const res = await request
+        .post(`/api/datasources`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      return res.body.datasource
+    }
+
+    it("should save a datasource with password", async () => {
+      const datasource = await makeDatasource()
+      expect(datasource.config.password).toBe("--secret-value--")
+    })
+
+    it("should not the password on update with the --secret-value--", async () => {
+      const datasource = await makeDatasource()
+      await request
+        .put(`/api/datasources/${datasource._id}`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      await context.doInAppContext(config.getAppId(), async () => {
+        const dbDatasource: any = await sdk.datasources.get(datasource._id)
+        expect(dbDatasource.config.password).toBe("testing")
+      })
+    })
+  })
 })

packages/server/src/definitions/automations.ts

@@ -24,6 +24,7 @@ export interface TriggerOutput {
 
 export interface AutomationContext extends AutomationResults {
   steps: any[]
+  env?: Record<string, string>
   trigger: any
 }
 

packages/server/src/index.ts

@@ -1,6 +1,6 @@
 import { bootstrap } from "global-agent"
 const fixPath = require("fix-path")
-const { checkDevelopmentEnvironment } = require("./utilities/fileSystem")
+import { checkDevelopmentEnvironment } from "./utilities/fileSystem"
 
 function runServer() {
   // this will shutdown the system if development environment not ready

packages/server/src/integrations/base/query.ts

@@ -1,10 +1,12 @@
 import { QueryJson, Datasource } from "@budibase/types"
-const { getIntegration } = require("../index")
+import { getIntegration } from "../index"
+import sdk from "../../sdk"
 
 export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
 ) {
+  datasource = await sdk.datasources.enrich(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function
   if (Integration.prototype.query) {

packages/server/src/integrations/base/sqlTable.ts

@@ -3,7 +3,7 @@ import { Operation, QueryJson, RenameColumn, Table } from "@budibase/types"
 import { breakExternalTableId } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
-const { FieldTypes, RelationshipTypes } = require("../../constants")
+import { FieldTypes, RelationshipTypes } from "../../constants"
 
 function generateSchema(
   schema: CreateTableBuilder,

packages/server/src/integrations/dynamodb.ts

@@ -5,8 +5,8 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-const AWS = require("aws-sdk")
-const { AWS_REGION } = require("../db/dynamoClient")
+import AWS from "aws-sdk"
+import { AWS_REGION } from "../db/dynamoClient"
 
 interface DynamoDBConfig {
   region: string
@@ -182,7 +182,7 @@ class DynamoDBIntegration implements IntegrationBase {
     return response
   }
 
-  async describe(query: { table: string }) {
+  async describe(query: { table: string }): Promise<any> {
     const params = {
       TableName: query.table,
     }

packages/server/src/integrations/mysql.ts

@@ -16,7 +16,7 @@ import {
   finaliseExternalTables,
 } from "./utils"
 import dayjs from "dayjs"
-const { NUMBER_REGEX } = require("../utilities")
+import { NUMBER_REGEX } from "../utilities"
 import Sql from "./base/sql"
 import { MySQLColumn } from "./base/types"
 

packages/server/src/integrations/queries/sql.ts

@@ -1,55 +1,10 @@
-import { findHBSBlocks, processStringSync } from "@budibase/string-templates"
+import { findHBSBlocks } from "@budibase/string-templates"
 import { DatasourcePlus } from "@budibase/types"
+import sdk from "../../sdk"
 
 const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
 
-export function enrichQueryFields(
-  fields: { [key: string]: any },
-  parameters = {}
-) {
-  const enrichedQuery: { [key: string]: any } = Array.isArray(fields) ? [] : {}
-  if (!fields || !parameters) {
-    return enrichedQuery
-  }
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      enrichedQuery[key] = processStringSync(fields[key], parameters, {
-        noEscaping: true,
-        noHelpers: true,
-        escapeNewlines: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-  return enrichedQuery
-}
-
-export function interpolateSQL(
+export async function interpolateSQL(
   fields: { [key: string]: any },
   parameters: { [key: string]: any },
   integration: DatasourcePlus
@@ -90,7 +45,7 @@ export function interpolateSQL(
     else if (listRegexMatch) {
       arrays.push(binding)
       // determine the length of the array
-      const value = enrichQueryFields([binding], parameters)[0]
+      const value = (await sdk.queries.enrichContext([binding], parameters))[0]
         .split(",")
         .map((val: string) => val.trim())
       // build a string like ($1, $2, $3)
@@ -109,7 +64,7 @@ export function interpolateSQL(
   }
   // replicate the knex structure
   fields.sql = sql
-  fields.bindings = enrichQueryFields(variables, parameters)
+  fields.bindings = await sdk.queries.enrichContext(variables, parameters)
   // check for arrays in the data
   let updated: string[] = []
   for (let i = 0; i < variables.length; i++) {

packages/server/src/integrations/rest.ts

@@ -16,11 +16,11 @@ import {
 import { get } from "lodash"
 import * as https from "https"
 import qs from "querystring"
-const fetch = require("node-fetch")
-const { formatBytes } = require("../utilities")
-const { performance } = require("perf_hooks")
-const FormData = require("form-data")
-const { URLSearchParams } = require("url")
+import fetch from "node-fetch"
+import { formatBytes } from "../utilities"
+import { performance } from "perf_hooks"
+import FormData from "form-data"
+import { URLSearchParams } from "url"
 
 const BodyTypes = {
   NONE: "none",
@@ -204,12 +204,12 @@ class RestIntegration implements IntegrationBase {
 
       // Append page number or cursor param if configured
       if (pageParam && paginationValues.page != null) {
-        params.append(pageParam, paginationValues.page)
+        params.append(pageParam, paginationValues.page as string)
       }
 
       // Append page size param if configured
       if (sizeParam && paginationValues.limit != null) {
-        params.append(sizeParam, paginationValues.limit)
+        params.append(sizeParam, String(paginationValues.limit))
       }
 
       // Prepend query string with pagination params
@@ -280,7 +280,7 @@ class RestIntegration implements IntegrationBase {
       case BodyTypes.ENCODED:
         const params = new URLSearchParams()
         for (let [key, value] of Object.entries(object)) {
-          params.append(key, value)
+          params.append(key, value as string)
         }
         addPaginationToBody((key: string, value: any) => {
           params.append(key, value)

packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -2,7 +2,7 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration"
 import * as syncRows from "../syncRows"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-const { db: dbCore } = require("@budibase/backend-core")
+import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
@@ -24,13 +24,17 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await config.createTable()
-      await config.createRow()
+      await context.doInAppContext(app1.appId, async () => {
+        await config.createTable()
+        await config.createRow()
+      })
       // app 2
       const app2 = await config.createApp("second-app")
-      await config.createTable()
-      await config.createRow()
-      await config.createRow()
+      await context.doInAppContext(app2.appId, async () => {
+        await config.createTable()
+        await config.createRow()
+        await config.createRow()
+      })
 
       // migrate
       await syncRows.run()

packages/server/src/sdk/app/datasources/datasources.ts

@@ -0,0 +1,98 @@
+import { context } from "@budibase/backend-core"
+import { processObjectSync, findHBSBlocks } from "@budibase/string-templates"
+import {
+  Datasource,
+  DatasourceFieldType,
+  Integration,
+  PASSWORD_REPLACEMENT,
+} from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
+import { getEnvironmentVariables } from "../../utils"
+import { getDefinitions } from "../../../integrations"
+
+const ENV_VAR_PREFIX = "env."
+const USER_PREFIX = "user"
+
+async function enrichDatasourceWithValues(datasource: Datasource) {
+  const cloned = cloneDeep(datasource)
+  const env = await getEnvironmentVariables()
+  const processed = processObjectSync(cloned, { env }, { onlyFound: true })
+  return {
+    datasource: processed as Datasource,
+    envVars: env as Record<string, string>,
+  }
+}
+
+export async function enrich(datasource: Datasource) {
+  const { datasource: response } = await enrichDatasourceWithValues(datasource)
+  return response
+}
+
+export async function get(
+  datasourceId: string,
+  opts?: { enriched: boolean }
+): Promise<Datasource> {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  if (opts?.enriched) {
+    return (await enrichDatasourceWithValues(datasource)).datasource
+  } else {
+    return datasource
+  }
+}
+
+export async function getWithEnvVars(datasourceId: string) {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  return enrichDatasourceWithValues(datasource)
+}
+
+export async function removeSecrets(datasources: Datasource[]) {
+  const definitions = await getDefinitions()
+  for (let datasource of datasources) {
+    const schema = definitions[datasource.source]
+    if (datasource.config) {
+      // strip secrets from response, so they don't show in the network request
+      if (datasource.config.auth) {
+        delete datasource.config.auth
+      }
+      // remove passwords
+      for (let key of Object.keys(datasource.config)) {
+        if (typeof datasource.config[key] !== "string") {
+          continue
+        }
+        const blocks = findHBSBlocks(datasource.config[key] as string)
+        const usesEnvVars =
+          blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+        if (
+          !usesEnvVars &&
+          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD
+        ) {
+          datasource.config[key] = PASSWORD_REPLACEMENT
+        }
+      }
+    }
+  }
+  return datasources
+}
+
+export async function removeSecretSingle(datasource: Datasource) {
+  return (await removeSecrets([datasource]))[0]
+}
+
+export function mergeConfigs(update: Datasource, old: Datasource) {
+  if (!update.config) {
+    return update
+  }
+  for (let [key, value] of Object.entries(update.config)) {
+    if (value !== PASSWORD_REPLACEMENT) {
+      continue
+    }
+    if (old.config?.[key]) {
+      update.config[key] = old.config?.[key]
+    } else {
+      delete update.config[key]
+    }
+  }
+  return update
+}

packages/server/src/sdk/app/datasources/index.ts

@@ -0,0 +1,5 @@
+import * as datasources from "./datasources"
+
+export default {
+  ...datasources,
+}

packages/server/src/sdk/app/queries/index.ts

@@ -0,0 +1,5 @@
+import * as queries from "./queries"
+
+export default {
+  ...queries,
+}

packages/server/src/sdk/app/queries/queries.ts

@@ -0,0 +1,50 @@
+import { getEnvironmentVariables } from "../../utils"
+import { processStringSync } from "@budibase/string-templates"
+
+export async function enrichContext(
+  fields: Record<string, any>,
+  inputs = {}
+): Promise<Record<string, any>> {
+  const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
+  if (!fields || !inputs) {
+    return enrichedQuery
+  }
+  const env = await getEnvironmentVariables()
+  const parameters = { ...inputs, env }
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = await enrichContext(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: true,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+  return enrichedQuery
+}

packages/server/src/sdk/app/tables/index.ts

@@ -6,6 +6,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { Table, Database } from "@budibase/types"
+import datasources from "../datasources"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
@@ -23,9 +24,11 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
   }))
 }
 
-async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+async function getAllExternalTables(
+  datasourceId: any
+): Promise<Record<string, Table>> {
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await datasources.get(datasourceId, { enriched: true })
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
   }
@@ -44,7 +47,7 @@ async function getTable(tableId: any): Promise<Table> {
   const db = context.getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
+    const datasource = await datasources.get(datasourceId!)
     const table = await getExternalTable(datasourceId, tableName)
     return { ...table, sql: isSQL(datasource) }
   } else {

packages/server/src/sdk/index.ts

@@ -2,6 +2,8 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as datasources } from "./app/datasources"
+import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
@@ -12,6 +14,8 @@ const sdk = {
   applications,
   rows,
   users,
+  datasources,
+  queries,
 }
 
 // default export for TS

packages/server/src/sdk/utils/index.ts

@@ -0,0 +1,16 @@
+import { environmentVariables } from "@budibase/pro"
+import { context, db as dbCore } from "@budibase/backend-core"
+import { AppEnvironment } from "@budibase/types"
+
+export async function getEnvironmentVariables() {
+  let envVars = context.getEnvironmentVariables()
+  if (!envVars) {
+    const appId = context.getAppId()
+    const appEnv = dbCore.isDevAppID(appId)
+      ? AppEnvironment.DEVELOPMENT
+      : AppEnvironment.PRODUCTION
+
+    envVars = await environmentVariables.fetchValues(appEnv)
+  }
+  return envVars
+}

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -364,20 +364,23 @@ class TestConfiguration {
     // create dev app
     // clear any old app
     this.appId = null
-    // @ts-ignore
-    await context.updateAppId(null)
-    this.app = await this._req({ name: appName }, null, controllers.app.create)
-    this.appId = this.app.appId
-    // @ts-ignore
-    await context.updateAppId(this.appId)
+    await context.doInAppContext(null, async () => {
+      this.app = await this._req(
+        { name: appName },
+        null,
+        controllers.app.create
+      )
+      this.appId = this.app.appId
+    })
+    return await context.doInAppContext(this.appId, async () => {
+      // create production app
+      this.prodApp = await this.publish()
 
-    // create production app
-    this.prodApp = await this.publish()
+      this.allApps.push(this.prodApp)
+      this.allApps.push(this.app)
 
-    this.allApps.push(this.prodApp)
-    this.allApps.push(this.app)
-
-    return this.app
+      return this.app
+    })
   }
 
   async publish() {

packages/server/src/threads/automation.ts

@@ -16,7 +16,6 @@ import { storeLog } from "../automations/logging"
 import { Automation, AutomationStep, AutomationStatus } from "@budibase/types"
 import {
   LoopStep,
-  LoopStepType,
   LoopInput,
   TriggerOutput,
   AutomationContext,
@@ -26,6 +25,7 @@ import { WorkerCallback } from "./definitions"
 import { context, logging } from "@budibase/backend-core"
 import { processObject } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
+import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
@@ -225,6 +225,8 @@ class Orchestrator {
   }
 
   async execute() {
+    // this will retrieve from context created at start of thread
+    this._context.env = await sdkUtils.getEnvironmentVariables()
     let automation = this._automation
     let stopped = false
     let loopStep: AutomationStep | undefined = undefined
@@ -478,7 +480,11 @@ export const removeStalled = async (job: Job) => {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   await context.doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(job)
-    await automationOrchestrator.stopCron("stalled")
+    const envVars = await sdkUtils.getEnvironmentVariables()
+    // put into automation thread for whole context
+    await context.doInEnvironmentContext(envVars, async () => {
+      const automationOrchestrator = new Orchestrator(job)
+      await automationOrchestrator.stopCron("stalled")
+    })
   })
 }

packages/server/src/threads/definitions.ts

@@ -1,3 +1,5 @@
+import { EnvironmentVariablesDecrypted } from "@budibase/types"
+
 export type WorkerCallback = (error: any, response?: any) => void
 
 export interface QueryEvent {
@@ -9,6 +11,7 @@ export interface QueryEvent {
   pagination?: any
   transformer: any
   queryId: string
+  environmentVariables?: Record<string, string>
   ctx?: any
 }
 

packages/server/src/threads/query.ts

@@ -6,13 +6,11 @@ import { getIntegration } from "../integrations"
 import { processStringSync } from "@budibase/string-templates"
 import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
+import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
 
-const { isSQL } = require("../integrations/utils")
-const {
-  enrichQueryFields,
-  interpolateSQL,
-} = require("../integrations/queries/sql")
+import { isSQL } from "../integrations/utils"
+import { interpolateSQL } from "../integrations/queries/sql"
 
 class QueryRunner {
   datasource: any
@@ -62,10 +60,11 @@ class QueryRunner {
     }
 
     if (datasourceClone.config.authConfigs) {
-      datasourceClone.config.authConfigs =
-        datasourceClone.config.authConfigs.map((config: any) => {
-          return enrichQueryFields(config, this.ctx)
-        })
+      const updatedConfigs = []
+      for (let config of datasourceClone.config.authConfigs) {
+        updatedConfigs.push(await sdk.queries.enrichContext(config, this.ctx))
+      }
+      datasourceClone.config.authConfigs = updatedConfigs
     }
 
     const integration = new Integration(datasourceClone.config)
@@ -75,12 +74,15 @@ class QueryRunner {
 
     // Enrich the parameters with the addition context items.
     // 'user' is now a reserved variable key in mapping parameters
-    const enrichedParameters = enrichQueryFields(parameters, this.ctx)
+    const enrichedParameters = await sdk.queries.enrichContext(
+      parameters,
+      this.ctx
+    )
     const enrichedContext = { ...enrichedParameters, ...this.ctx }
 
     // Parse global headers
     if (datasourceClone.config.defaultHeaders) {
-      datasourceClone.config.defaultHeaders = enrichQueryFields(
+      datasourceClone.config.defaultHeaders = await sdk.queries.enrichContext(
         datasourceClone.config.defaultHeaders,
         enrichedContext
       )
@@ -89,9 +91,9 @@ class QueryRunner {
     let query
     // handle SQL injections by interpolating the variables
     if (isSQL(datasourceClone)) {
-      query = interpolateSQL(fieldsClone, enrichedParameters, integration)
+      query = await interpolateSQL(fieldsClone, enrichedParameters, integration)
     } else {
-      query = enrichQueryFields(fieldsClone, enrichedContext)
+      query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
     }
 
     // Add pagination values for REST queries
@@ -166,7 +168,9 @@ class QueryRunner {
   async runAnotherQuery(queryId: string, parameters: any) {
     const db = context.getAppDB()
     const query = await db.get(queryId)
-    const datasource = await db.get(query.datasourceId)
+    const datasource = await sdk.datasources.get(query.datasourceId, {
+      enriched: true,
+    })
     return new QueryRunner(
       {
         datasource,
@@ -280,7 +284,7 @@ class QueryRunner {
 }
 
 export function execute(input: QueryEvent, callback: WorkerCallback) {
-  context.doInAppContext(input.appId!, async () => {
+  const run = async () => {
     const Runner = new QueryRunner(input)
     try {
       const response = await Runner.execute()
@@ -288,5 +292,14 @@ export function execute(input: QueryEvent, callback: WorkerCallback) {
     } catch (err) {
       callback(err)
     }
+  }
+  context.doInAppContext(input.appId!, async () => {
+    if (input.environmentVariables) {
+      return context.doInEnvironmentContext(input.environmentVariables, () => {
+        return run()
+      })
+    } else {
+      return run()
+    }
   })
 }

packages/server/src/utilities/fileSystem/filesystem.ts

@@ -1,10 +1,10 @@
 import { PathLike } from "fs"
-const { budibaseTempDir } = require("../budibaseDir")
-const fs = require("fs")
-const { join } = require("path")
-const uuid = require("uuid/v4")
+import fs from "fs"
+import { budibaseTempDir } from "../budibaseDir"
+import { join } from "path"
 import env from "../../environment"
 import tar from "tar"
+const uuid = require("uuid/v4")
 
 export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 
@@ -112,6 +112,7 @@ export const sendTempFile = (fileContents: any) => {
  * allows a centralised location to check logic is all good.
  */
 export const readFileSync = (filepath: PathLike, options = "utf8") => {
+  // @ts-ignore
   return fs.readFileSync(filepath, options)
 }
 
@@ -147,6 +148,7 @@ export const findFileRec = (startPath: PathLike, filter: string): any => {
 
   const files = fs.readdirSync(startPath)
   for (let i = 0, len = files.length; i < len; i++) {
+    // @ts-ignore
     const filename = join(startPath, files[i])
     const stat = fs.lstatSync(filename)
 

packages/server/src/utilities/fileSystem/plugin.ts

@@ -1,8 +1,7 @@
 import { Plugin } from "@budibase/types"
-
-const { budibaseTempDir } = require("../budibaseDir")
-const fs = require("fs")
-const { join } = require("path")
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
 import { objectStore } from "@budibase/backend-core"
 
 const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")

packages/server/src/utilities/fileSystem/template.ts

@@ -1,5 +1,5 @@
-const fs = require("fs")
-const { join } = require("path")
+import fs from "fs"
+import { join } from "path"
 import { ObjectStoreBuckets } from "../../constants"
 import { objectStore } from "@budibase/backend-core"
 

packages/string-templates/src/helpers/index.js

@@ -32,11 +32,15 @@ const HELPERS = [
   // javascript helper
   new Helper(HelperFunctionNames.JS, processJS, false),
   // this help is applied to all statements
-  new Helper(HelperFunctionNames.ALL, (value, { __opts }) => {
+  new Helper(HelperFunctionNames.ALL, (value, inputs) => {
+    const { __opts } = inputs
     if (isObject(value)) {
       return new SafeString(JSON.stringify(value))
     }
     // null/undefined values produce bad results
+    if (__opts && __opts.onlyFound && value == null) {
+      return __opts.input
+    }
     if (value == null || typeof value !== "string") {
       return value == null ? "" : value
     }

packages/string-templates/src/index.js

@@ -146,16 +146,31 @@ module.exports.processStringSync = (string, context, opts) => {
   if (typeof string !== "string") {
     throw "Cannot process non-string types."
   }
-  try {
-    const template = createTemplate(string, opts)
+  function process(stringPart) {
+    const template = createTemplate(stringPart, opts)
     const now = Math.floor(Date.now() / 1000) * 1000
     return processors.postprocess(
       template({
         now: new Date(now).toISOString(),
-        __opts: opts,
+        __opts: {
+          ...opts,
+          input: stringPart,
+        },
         ...context,
       })
     )
+  }
+  try {
+    if (opts && opts.onlyFound) {
+      const blocks = exports.findHBSBlocks(string)
+      for (let block of blocks) {
+        const outcome = process(block)
+        string = string.replace(block, outcome)
+      }
+      return string
+    } else {
+      return process(string)
+    }
   } catch (err) {
     return input
   }

packages/string-templates/test/basic.spec.js

@@ -221,3 +221,9 @@ describe("check find hbs blocks function", () => {
   })
 })
 
+describe("should leave HBS blocks if not found using option", () => {
+  it("should replace one, leave one", async () => {
+    const output = await processString("{{ a }}, {{ b }}", { b: 1 }, { onlyFound: true })
+    expect(output).toBe("{{ a }}, 1")
+  })
+})

packages/types/src/api/web/app/backup.ts

@@ -12,6 +12,11 @@ export interface CreateAppBackupRequest {
   name: string
 }
 
+export interface CreateAppBackupResponse {
+  backupId: string
+  message: string
+}
+
 export interface UpdateAppBackupRequest {
   name: string
 }

packages/types/src/api/web/global/environmentVariables.ts

@@ -0,0 +1,18 @@
+export interface StatusEnvironmentVariableResponse {
+  encryptionKeyAvailable: boolean
+}
+
+export interface CreateEnvironmentVariableRequest {
+  name: string
+  production: string
+  development: string
+}
+
+export interface UpdateEnvironmentVariableRequest {
+  production: string
+  development: string
+}
+
+export interface GetEnvironmentVariablesResponse {
+  variables: string[]
+}

packages/types/src/api/web/global/events.ts

@@ -0,0 +1,7 @@
+export enum EventPublishType {
+  ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable_upgrade_panel_opened",
+}
+
+export interface PostEventPublishRequest {
+  type: EventPublishType
+}

packages/types/src/api/web/global/index.ts

@@ -0,0 +1,2 @@
+export * from "./environmentVariables"
+export * from "./events"

packages/types/src/api/web/index.ts

@@ -3,3 +3,4 @@ export * from "./user"
 export * from "./errors"
 export * from "./schedule"
 export * from "./app"
+export * from "./global"

packages/types/src/documents/app/datasource.ts

@@ -8,7 +8,7 @@ export interface Datasource extends Document {
   source: SourceName
   // the config is defined by the schema
   config?: {
-    [key: string]: string | number | boolean
+    [key: string]: string | number | boolean | any[]
   }
   plus?: boolean
   entities?: {

packages/types/src/documents/global/environmentVariables.ts

@@ -0,0 +1,20 @@
+import { Document } from "../document"
+
+export interface EnvironmentVariablesDoc extends Document {
+  variables: string
+}
+
+export type EnvironmentVariableValue = {
+  production: string
+  development: string
+}
+
+// what comes out of the "variables" when it is decrypted
+export type EnvironmentVariablesDecrypted = Record<
+  string,
+  EnvironmentVariableValue
+>
+
+export interface EnvironmentVariablesDocDecrypted extends Document {
+  variables: EnvironmentVariablesDecrypted
+}

packages/types/src/documents/global/index.ts

@@ -5,3 +5,4 @@ export * from "./plugin"
 export * from "./quotas"
 export * from "./schedule"
 export * from "./templates"
+export * from "./environmentVariables"

packages/types/src/sdk/datasources.ts

@@ -1,5 +1,7 @@
 import { Table } from "../documents"
 
+export const PASSWORD_REPLACEMENT = "--secret-value--"
+
 export enum Operation {
   CREATE = "CREATE",
   READ = "READ",

packages/types/src/sdk/environmentVariables.ts

@@ -0,0 +1,4 @@
+export enum AppEnvironment {
+  PRODUCTION = "production",
+  DEVELOPMENT = "development",
+}

packages/types/src/sdk/events/environmentVariable.ts

@@ -0,0 +1,14 @@
+import { BaseEvent } from "./event"
+
+export interface EnvironmentVariableCreatedEvent extends BaseEvent {
+  name: string
+  environments: string[]
+}
+
+export interface EnvironmentVariableDeletedEvent extends BaseEvent {
+  name: string
+}
+
+export interface EnvironmentVariableUpgradePanelOpenedEvent extends BaseEvent {
+  userId: string
+}

packages/types/src/sdk/events/event.ts

@@ -172,6 +172,11 @@ export enum Event {
   // BACKUP
   APP_BACKUP_RESTORED = "app:backup:restored",
   APP_BACKUP_TRIGGERED = "app:backup:triggered",
+
+  // ENVIRONMENT VARIABLE
+  ENVIRONMENT_VARIABLE_CREATED = "environment_variable:created",
+  ENVIRONMENT_VARIABLE_DELETED = "environment_variable:deleted",
+  ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable:upgrade_panel_opened",
 }
 
 // properties added at the final stage of the event pipeline

packages/types/src/sdk/events/index.ts

@@ -21,3 +21,4 @@ export * from "./identification"
 export * from "./userGroup"
 export * from "./plugin"
 export * from "./backup"
+export * from "./environmentVariable"

packages/types/src/sdk/index.ts

@@ -11,3 +11,4 @@ export * from "./locks"
 export * from "./db"
 export * from "./middleware"
 export * from "./featureFlag"
+export * from "./environmentVariables"

packages/worker/src/api/controllers/global/events.ts

@@ -0,0 +1,17 @@
+import {
+  UserCtx,
+  PostEventPublishRequest,
+  EventPublishType,
+} from "@budibase/types"
+import { events } from "@budibase/backend-core"
+
+export async function publish(ctx: UserCtx<PostEventPublishRequest, void>) {
+  switch (ctx.request.body.type) {
+    case EventPublishType.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED:
+      await events.environmentVariable.upgradePanelOpened(ctx.user._id!)
+      break
+    default:
+      ctx.throw(400, "Invalid publish event type.")
+  }
+  ctx.status = 200
+}

packages/worker/src/api/routes/global/events.ts

@@ -0,0 +1,8 @@
+import Router from "@koa/router"
+import * as controller from "../../controllers/global/events"
+
+const router: Router = new Router()
+
+router.post("/api/global/event/publish", controller.publish)
+
+export default router

packages/worker/src/api/routes/index.ts

@@ -7,6 +7,7 @@ import templateRoutes from "./global/templates"
 import emailRoutes from "./global/email"
 import authRoutes from "./global/auth"
 import roleRoutes from "./global/roles"
+import eventRoutes from "./global/events"
 import environmentRoutes from "./system/environment"
 import tenantsRoutes from "./system/tenants"
 import statusRoutes from "./system/status"
@@ -34,4 +35,5 @@ export const routes: Router[] = [
   migrationRoutes,
   accountRoutes,
   restoreRoutes,
+  eventRoutes,
 ]