Merge branch 'develop' into api-tests-generate-tenants

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.2.12-alpha.44",
+  "version": "2.2.12-alpha.50",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.2.12-alpha.44",
+  "version": "2.2.12-alpha.50",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
-    "@budibase/types": "2.2.12-alpha.44",
+    "@budibase/types": "2.2.12-alpha.50",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/constants/db.ts

@@ -77,6 +77,7 @@ export const StaticDatabases = {
       apiKeys: "apikeys",
       usageQuota: "usage_quota",
       licenseInfo: "license_info",
+      environmentVariables: "environmentvariables",
     },
   },
   // contains information about tenancy and so on

packages/backend-core/src/context/Context.ts

@@ -1,17 +1,14 @@
 import { AsyncLocalStorage } from "async_hooks"
+import { ContextMap } from "./mainContext"
 
 export default class Context {
-  static storage = new AsyncLocalStorage<Record<string, any>>()
+  static storage = new AsyncLocalStorage<ContextMap>()
 
-  static run(context: Record<string, any>, func: any) {
+  static run(context: ContextMap, func: any) {
     return Context.storage.run(context, () => func())
   }
 
-  static get(): Record<string, any> {
+  static get(): ContextMap {
-    return Context.storage.getStore() as Record<string, any>
+    return Context.storage.getStore() as ContextMap
-  }
-
-  static set(context: Record<string, any>) {
-    Context.storage.enterWith(context)
   }
 }

packages/backend-core/src/context/mainContext.ts

@@ -16,6 +16,7 @@ export type ContextMap = {
   tenantId?: string
   appId?: string
   identity?: IdentityContext
+  environmentVariables?: Record<string, string>
 }
 
 let TEST_APP_ID: string | null = null
@@ -75,7 +76,7 @@ export function getTenantIDFromAppID(appId: string) {
   }
 }
 
-function updateContext(updates: ContextMap) {
+function updateContext(updates: ContextMap): ContextMap {
   let context: ContextMap
   try {
     context = Context.get()
@@ -120,15 +121,23 @@ export async function doInTenant(
   return newContext(updates, task)
 }
 
-export async function doInAppContext(appId: string, task: any): Promise<any> {
+export async function doInAppContext(
-  if (!appId) {
+  appId: string | null,
+  task: any
+): Promise<any> {
+  if (!appId && !env.isTest()) {
     throw new Error("appId is required")
   }
 
-  const tenantId = getTenantIDFromAppID(appId)
+  let updates: ContextMap
-  const updates: ContextMap = { appId }
+  if (!appId) {
-  if (tenantId) {
+    updates = { appId: "" }
-    updates.tenantId = tenantId
+  } else {
+    const tenantId = getTenantIDFromAppID(appId)
+    updates = { appId }
+    if (tenantId) {
+      updates.tenantId = tenantId
+    }
   }
   return newContext(updates, task)
 }
@@ -189,25 +198,25 @@ export const getProdAppId = () => {
   return conversions.getProdAppID(appId)
 }
 
-export function updateTenantId(tenantId?: string) {
+export function doInEnvironmentContext(
-  let context: ContextMap = updateContext({
+  values: Record<string, string>,
-    tenantId,
+  task: any
-  })
+) {
-  Context.set(context)
+  if (!values) {
+    throw new Error("Must supply environment variables.")
+  }
+  const updates = {
+    environmentVariables: values,
+  }
+  return newContext(updates, task)
 }
 
-export function updateAppId(appId: string) {
+export function getEnvironmentVariables() {
-  let context: ContextMap = updateContext({
+  const context = Context.get()
-    appId,
+  if (!context.environmentVariables) {
-  })
+    return null
-  try {
+  } else {
-    Context.set(context)
+    return context.environmentVariables
-  } catch (err) {
-    if (env.isTest()) {
-      TEST_APP_ID = appId
-    } else {
-      throw err
-    }
   }
 }
 

packages/backend-core/src/environment.ts

@@ -37,6 +37,7 @@ const environment = {
   },
   JS_BCRYPT: process.env.JS_BCRYPT,
   JWT_SECRET: process.env.JWT_SECRET,
+  ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,

packages/backend-core/src/events/publishers/environmentVariable.ts

@@ -0,0 +1,38 @@
+import {
+  Event,
+  EnvironmentVariableCreatedEvent,
+  EnvironmentVariableDeletedEvent,
+  EnvironmentVariableUpgradePanelOpenedEvent,
+} from "@budibase/types"
+import { publishEvent } from "../events"
+
+async function created(name: string, environments: string[]) {
+  const properties: EnvironmentVariableCreatedEvent = {
+    name,
+    environments,
+  }
+  await publishEvent(Event.ENVIRONMENT_VARIABLE_CREATED, properties)
+}
+
+async function deleted(name: string) {
+  const properties: EnvironmentVariableDeletedEvent = {
+    name,
+  }
+  await publishEvent(Event.ENVIRONMENT_VARIABLE_DELETED, properties)
+}
+
+async function upgradePanelOpened(userId: string) {
+  const properties: EnvironmentVariableUpgradePanelOpenedEvent = {
+    userId,
+  }
+  await publishEvent(
+    Event.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED,
+    properties
+  )
+}
+
+export default {
+  created,
+  deleted,
+  upgradePanelOpened,
+}

packages/backend-core/src/events/publishers/index.ts

@@ -20,3 +20,4 @@ export { default as backfill } from "./backfill"
 export { default as group } from "./group"
 export { default as plugin } from "./plugin"
 export { default as backup } from "./backup"
+export { default as environmentVariable } from "./environmentVariable"

packages/backend-core/src/events/publishers/user.ts

@@ -13,6 +13,7 @@ import {
   UserPermissionAssignedEvent,
   UserPermissionRemovedEvent,
   UserUpdatedEvent,
+  UserOnboardingEvent,
 } from "@budibase/types"
 
 async function created(user: User, timestamp?: number) {
@@ -36,6 +37,13 @@ async function deleted(user: User) {
   await publishEvent(Event.USER_DELETED, properties)
 }
 
+export async function onboardingComplete(user: User) {
+  const properties: UserOnboardingEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_ONBOARDING_COMPLETE, properties)
+}
+
 // PERMISSIONS
 
 async function permissionAdminAssigned(user: User, timestamp?: number) {
@@ -126,6 +134,7 @@ export default {
   permissionAdminRemoved,
   permissionBuilderAssigned,
   permissionBuilderRemoved,
+  onboardingComplete,
   invited,
   inviteAccepted,
   passwordForceReset,

packages/backend-core/src/security/encryption.ts

@@ -2,19 +2,45 @@ import crypto from "crypto"
 import env from "../environment"
 
 const ALGO = "aes-256-ctr"
-const SECRET = env.JWT_SECRET
 const SEPARATOR = "-"
 const ITERATIONS = 10000
 const RANDOM_BYTES = 16
 const STRETCH_LENGTH = 32
 
+export enum SecretOption {
+  JWT = "jwt",
+  ENCRYPTION = "encryption",
+}
+
+function getSecret(secretOption: SecretOption): string {
+  let secret, secretName
+  switch (secretOption) {
+    case SecretOption.ENCRYPTION:
+      secret = env.ENCRYPTION_KEY
+      secretName = "ENCRYPTION_KEY"
+      break
+    case SecretOption.JWT:
+    default:
+      secret = env.JWT_SECRET
+      secretName = "JWT_SECRET"
+      break
+  }
+  if (!secret) {
+    throw new Error(`Secret "${secretName}" has not been set in environment.`)
+  }
+  return secret
+}
+
 function stretchString(string: string, salt: Buffer) {
   return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512")
 }
 
-export function encrypt(input: string) {
+export function encrypt(
+  input: string,
+  secretOption: SecretOption = SecretOption.JWT
+) {
   const salt = crypto.randomBytes(RANDOM_BYTES)
-  const stretched = stretchString(SECRET!, salt)
+  const stretched = stretchString(getSecret(secretOption), salt)
   const cipher = crypto.createCipheriv(ALGO, stretched, salt)
   const base = cipher.update(input)
   const final = cipher.final()
@@ -22,10 +48,13 @@ export function encrypt(input: string) {
   return `${salt.toString("hex")}${SEPARATOR}${encrypted}`
 }
 
-export function decrypt(input: string) {
+export function decrypt(
+  input: string,
+  secretOption: SecretOption = SecretOption.JWT
+) {
   const [salt, encrypted] = input.split(SEPARATOR)
   const saltBuffer = Buffer.from(salt, "hex")
-  const stretched = stretchString(SECRET!, saltBuffer)
+  const stretched = stretchString(getSecret(secretOption), saltBuffer)
   const decipher = crypto.createDecipheriv(ALGO, stretched, saltBuffer)
   const base = decipher.update(Buffer.from(encrypted, "hex"))
   const final = decipher.final()

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.2.12-alpha.44",
+  "version": "2.2.12-alpha.50",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.2.12-alpha.44",
+    "@budibase/string-templates": "2.2.12-alpha.50",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/bbui/src/Actions/position_dropdown.js

@@ -3,6 +3,9 @@ export default function positionDropdown(
   { anchor, align, maxWidth, useAnchorWidth }
 ) {
   const update = () => {
+    if (!anchor) {
+      return
+    }
     const anchorBounds = anchor.getBoundingClientRect()
     const elementBounds = element.getBoundingClientRect()
     let styles = {
@@ -13,6 +16,8 @@ export default function positionDropdown(
       top: null,
     }
 
+    let popoverLeftPad = 20
+
     // Determine vertical styles
     if (window.innerHeight - anchorBounds.bottom < 100) {
       styles.top = anchorBounds.top - elementBounds.height - 5
@@ -29,7 +34,13 @@ export default function positionDropdown(
       styles.minWidth = anchorBounds.width
     }
     if (align === "right") {
-      styles.left = anchorBounds.left + anchorBounds.width - elementBounds.width
+      let left =
+        anchorBounds.left + anchorBounds.width / 2 - elementBounds.width
+      // Accommodate margin on popover: 1.25rem; ~20px
+      if (left + elementBounds.width + popoverLeftPad > window.innerWidth) {
+        left -= 20
+      }
+      styles.left = left
     } else if (align === "right-side") {
       styles.left = anchorBounds.left + anchorBounds.width
     } else {
@@ -54,8 +65,11 @@ export default function positionDropdown(
   const resizeObserver = new ResizeObserver(entries => {
     entries.forEach(update)
   })
-  resizeObserver.observe(anchor)
+  if (anchor) {
+    resizeObserver.observe(anchor)
+  }
   resizeObserver.observe(element)
+  resizeObserver.observe(document.body)
 
   document.addEventListener("scroll", update, true)
 

packages/bbui/src/Button/Button.svelte

@@ -15,11 +15,13 @@
   export let tooltip = undefined
   export let dataCy
   export let newStyles = true
+  export let id
 
   let showTooltip = false
 </script>
 
 <button
+  {id}
   class:spectrum-Button--cta={cta}
   class:spectrum-Button--primary={primary}
   class:spectrum-Button--secondary={secondary}

packages/bbui/src/Form/Core/EnvDropdown.svelte

@@ -0,0 +1,282 @@
+<script>
+  import "@spectrum-css/textfield/dist/index-vars.css"
+  import { createEventDispatcher, onMount } from "svelte"
+  import clickOutside from "../../Actions/click_outside"
+  import Divider from "../../Divider/Divider.svelte"
+
+  export let value = null
+  export let placeholder = null
+  export let type = "text"
+  export let disabled = false
+  export let id = null
+  export let readonly = false
+  export let updateOnChange = true
+  export let dataCy
+  export let align
+  export let autofocus = false
+  export let variables
+  export let showModal
+  export let environmentVariablesEnabled
+  export let handleUpgradePanel
+  const dispatch = createEventDispatcher()
+
+  let field
+  let focus = false
+  let iconFocused = false
+  let open = false
+
+  //eslint-disable-next-line
+  const STRIP_NAME_REGEX = /(?<=\.)(.*?)(?=\ })/g
+
+  // Strips the name out of the value which is {{ env.Variable }} resulting in an array like ["Variable"]
+  $: hbsValue = String(value)?.match(STRIP_NAME_REGEX) || []
+
+  const updateValue = newValue => {
+    if (readonly) {
+      return
+    }
+    if (type === "number") {
+      const float = parseFloat(newValue)
+      newValue = isNaN(float) ? null : float
+    }
+    dispatch("change", newValue)
+  }
+
+  const onFocus = () => {
+    if (readonly) {
+      return
+    }
+    focus = true
+  }
+
+  const onBlur = event => {
+    if (readonly) {
+      return
+    }
+    focus = false
+    updateValue(event.target.value)
+  }
+
+  const onInput = event => {
+    if (readonly || !updateOnChange) {
+      return
+    }
+    updateValue(event.target.value)
+  }
+
+  const handleOutsideClick = event => {
+    if (open) {
+      event.stopPropagation()
+      open = false
+      focus = false
+      iconFocused = false
+      dispatch("closed")
+    }
+  }
+
+  const handleVarSelect = variable => {
+    open = false
+    focus = false
+    iconFocused = false
+    updateValue(`{{ env.${variable} }}`)
+  }
+
+  onMount(() => {
+    focus = autofocus
+    if (focus) field.focus()
+  })
+
+  function removeVariable() {
+    updateValue("")
+  }
+
+  function openPopover() {
+    open = true
+    focus = true
+    iconFocused = true
+  }
+</script>
+
+<div class="spectrum-InputGroup">
+  <div
+    class:is-disabled={disabled || hbsValue.length}
+    class:is-focused={focus}
+    class="spectrum-Textfield"
+  >
+    <svg
+      class:close-color={hbsValue.length}
+      class:focused={iconFocused}
+      class="hoverable icon-position spectrum-Icon spectrum-Icon--sizeS spectrum-Textfield-validationIcon"
+      focusable="false"
+      aria-hidden="true"
+      on:click={() => {
+        hbsValue.length ? removeVariable() : openPopover()
+      }}
+    >
+      <use
+        xlink:href={`#spectrum-icon-18-${!hbsValue.length ? "Key" : "Close"}`}
+      />
+    </svg>
+
+    <input
+      bind:this={field}
+      disabled={hbsValue.length || disabled}
+      {readonly}
+      {id}
+      data-cy={dataCy}
+      value={hbsValue.length ? `{{ ${hbsValue[0]} }}` : value}
+      placeholder={placeholder || ""}
+      on:click
+      on:blur
+      on:focus
+      on:input
+      on:keyup
+      on:blur={onBlur}
+      on:focus={onFocus}
+      on:input={onInput}
+      type={hbsValue.length ? "text" : type}
+      style={align ? `text-align: ${align};` : ""}
+      class="spectrum-Textfield-input"
+      inputmode={type === "number" ? "decimal" : "text"}
+    />
+  </div>
+  {#if open}
+    <div
+      use:clickOutside={handleOutsideClick}
+      class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
+    >
+      <ul class="spectrum-Menu" role="listbox">
+        {#if !environmentVariablesEnabled}
+          <div class="no-variables-text primary-text">
+            Upgrade your plan to get environment variables
+          </div>
+        {:else if variables.length}
+          <div style="max-height: 100px">
+            {#each variables as variable, idx}
+              <li
+                class="spectrum-Menu-item"
+                role="option"
+                aria-selected="true"
+                tabindex="0"
+                on:click={() => handleVarSelect(variable.name)}
+              >
+                <span class="spectrum-Menu-itemLabel">
+                  <div class="primary-text">
+                    {variable.name}
+                    <span />
+                  </div>
+                  <svg
+                    class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
+                    focusable="false"
+                    aria-hidden="true"
+                  >
+                    <use xlink:href="#spectrum-css-icon-Checkmark100" />
+                  </svg>
+                </span>
+              </li>
+            {/each}
+          </div>
+        {:else}
+          <div class="no-variables-text primary-text">
+            You don't have any environment variables yet
+          </div>
+        {/if}
+      </ul>
+      <Divider noMargin />
+      {#if environmentVariablesEnabled}
+        <div on:click={() => showModal()} class="add-variable">
+          <svg
+            class="spectrum-Icon spectrum-Icon--sizeS "
+            focusable="false"
+            aria-hidden="true"
+          >
+            <use xlink:href="#spectrum-icon-18-Add" />
+          </svg>
+          <div class="primary-text">Add Variable</div>
+        </div>
+      {:else}
+        <div on:click={() => handleUpgradePanel()} class="add-variable">
+          <svg
+            class="spectrum-Icon spectrum-Icon--sizeS "
+            focusable="false"
+            aria-hidden="true"
+          >
+            <use xlink:href="#spectrum-icon-18-ArrowUp" />
+          </svg>
+          <div class="primary-text">Upgrade plan</div>
+        </div>
+      {/if}
+    </div>
+  {/if}
+</div>
+
+<style>
+  .spectrum-Textfield {
+    width: 100%;
+  }
+
+  .icon-position {
+    position: absolute;
+    top: 25%;
+    right: 2%;
+  }
+
+  .hoverable:hover {
+    cursor: pointer;
+    color: var(--spectrum-global-color-blue-400);
+  }
+
+  .primary-text {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .spectrum-InputGroup {
+    min-width: 0;
+    width: 100%;
+  }
+
+  .spectrum-Popover {
+    max-height: 240px;
+    z-index: 999;
+    top: 100%;
+  }
+
+  .spectrum-Popover.spectrum-Popover--bottom.spectrum-Picker-popover.is-open {
+    width: 100%;
+  }
+
+  .no-variables-height {
+    height: 100px;
+  }
+
+  .no-variables-text {
+    padding: var(--spacing-m);
+    color: var(--spectrum-global-color-gray-600);
+  }
+
+  .add-variable {
+    display: flex;
+    padding: var(--spacing-m) 0 var(--spacing-m) var(--spacing-m);
+    align-items: center;
+    gap: var(--spacing-s);
+    cursor: pointer;
+  }
+
+  .focused {
+    color: var(--spectrum-global-color-blue-400);
+  }
+
+  .add-variable:hover {
+    background: var(--grey-1);
+  }
+
+  .close-color {
+    color: var(--spectrum-global-color-gray-900) !important;
+  }
+
+  .close-color:hover {
+    color: var(--spectrum-global-color-blue-400) !important;
+  }
+</style>

packages/bbui/src/Form/EnvDropdown.svelte

@@ -0,0 +1,52 @@
+<script>
+  import Field from "./Field.svelte"
+  import EnvDropdown from "./Core/EnvDropdown.svelte"
+  import { createEventDispatcher } from "svelte"
+
+  export let value = null
+  export let label = null
+  export let labelPosition = "above"
+  export let placeholder = null
+  export let type = "text"
+  export let disabled = false
+  export let readonly = false
+  export let error = null
+  export let updateOnChange = true
+  export let quiet = false
+  export let dataCy
+  export let autofocus
+  export let variables
+  export let showModal
+  export let environmentVariablesEnabled
+  export let handleUpgradePanel
+  const dispatch = createEventDispatcher()
+  const onChange = e => {
+    value = e.detail
+    dispatch("change", e.detail)
+  }
+</script>
+
+<Field {label} {labelPosition} {error}>
+  <EnvDropdown
+    {dataCy}
+    {updateOnChange}
+    {error}
+    {disabled}
+    {readonly}
+    {value}
+    {placeholder}
+    {type}
+    {quiet}
+    {autofocus}
+    {variables}
+    {showModal}
+    {environmentVariablesEnabled}
+    {handleUpgradePanel}
+    on:change={onChange}
+    on:click
+    on:input
+    on:blur
+    on:focus
+    on:keyup
+  />
+</Field>

packages/bbui/src/Popover/Popover.svelte

@@ -19,9 +19,7 @@
   export let showTip = false
   export let open = false
   export let useAnchorWidth = false
-
+  export let dismissible = true
-  let tipSvg =
-    '<svg xmlns="http://www.w3.org/svg/2000" width="23" height="12" class="spectrum-Popover-tip" > <path class="spectrum-Popover-tip-triangle" d="M 0.7071067811865476 0 L 11.414213562373096 10.707106781186548 L 22.121320343559645 0" /> </svg>'
 
   $: tooltipClasses = showTip
     ? `spectrum-Popover--withTip spectrum-Popover--${direction}`
@@ -67,9 +65,15 @@
   <Portal {target}>
     <div
       tabindex="0"
-      use:positionDropdown={{ anchor, align, maxWidth, useAnchorWidth }}
+      use:positionDropdown={{
+        anchor,
+        align,
+        maxWidth,
+        useAnchorWidth,
+        showTip: false,
+      }}
       use:clickOutside={{
-        callback: handleOutsideClick,
+        callback: dismissible ? handleOutsideClick : () => {},
         anchor,
       }}
       on:keydown={handleEscape}
@@ -78,10 +82,6 @@
       data-cy={dataCy}
       transition:fly|local={{ y: -20, duration: 200 }}
     >
-      {#if showTip}
-        {@html tipSvg}
-      {/if}
-
       <slot />
     </div>
   </Portal>
@@ -91,6 +91,7 @@
   .spectrum-Popover {
     min-width: var(--spectrum-global-dimension-size-2000);
     border-color: var(--spectrum-global-color-gray-300);
+    overflow: auto;
   }
   .spectrum-Popover.is-open.spectrum-Popover--withTip {
     margin-top: var(--spacing-xs);

packages/bbui/src/Tabs/Tab.svelte

@@ -3,6 +3,7 @@
   import Portal from "svelte-portal"
   export let title
   export let icon = ""
+  export let id
 
   const dispatch = createEventDispatcher()
   let selected = getContext("tab")
@@ -31,10 +32,7 @@
   $: {
     if ($selected.title === title && tab_internal) {
       if ($selected.info?.left !== tab_internal.getBoundingClientRect().left) {
-        $selected = {
+        setTabInfo()
-          ...$selected,
-          info: tab_internal.getBoundingClientRect(),
-        }
       }
     }
   }
@@ -50,6 +48,7 @@
 </script>
 
 <div
+  {id}
   bind:this={tab_internal}
   on:click={onClick}
   class:is-selected={$selected.title === title}

packages/bbui/src/index.js

@@ -27,6 +27,7 @@ export { default as RadioGroup } from "./Form/RadioGroup.svelte"
 export { default as Checkbox } from "./Form/Checkbox.svelte"
 export { default as InputDropdown } from "./Form/InputDropdown.svelte"
 export { default as PickerDropdown } from "./Form/PickerDropdown.svelte"
+export { default as EnvDropdown } from "./Form/EnvDropdown.svelte"
 export { default as DetailSummary } from "./DetailSummary/DetailSummary.svelte"
 export { default as Popover } from "./Popover/Popover.svelte"
 export { default as ProgressBar } from "./ProgressBar/ProgressBar.svelte"

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.2.12-alpha.44",
+  "version": "2.2.12-alpha.50",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.44",
+    "@budibase/bbui": "2.2.12-alpha.50",
-    "@budibase/client": "2.2.12-alpha.44",
+    "@budibase/client": "2.2.12-alpha.50",
-    "@budibase/frontend-core": "2.2.12-alpha.44",
+    "@budibase/frontend-core": "2.2.12-alpha.50",
-    "@budibase/string-templates": "2.2.12-alpha.44",
+    "@budibase/string-templates": "2.2.12-alpha.50",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/accordion": "^3.0.24",
     "@spectrum-css/page": "^3.0.1",

packages/builder/src/builderStore/dataBinding.js

@@ -21,6 +21,7 @@ import {
 import { TableNames } from "../constants"
 import { JSONUtils } from "@budibase/frontend-core"
 import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json"
+import { environment, licensing } from "stores/portal"
 
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
@@ -53,8 +54,13 @@ export const getBindableProperties = (asset, componentId) => {
  * Gets all rest bindable data fields
  */
 export const getRestBindings = () => {
+  const environmentVariablesEnabled = get(licensing).environmentVariablesEnabled
   const userBindings = getUserBindings()
-  return [...userBindings, ...getAuthBindings()]
+  return [
+    ...userBindings,
+    ...getAuthBindings(),
+    ...(environmentVariablesEnabled ? getEnvironmentBindings() : []),
+  ]
 }
 
 /**
@@ -89,6 +95,20 @@ export const getAuthBindings = () => {
   return bindings
 }
 
+export const getEnvironmentBindings = () => {
+  let envVars = get(environment).variables
+  return envVars.map(variable => {
+    return {
+      type: "context",
+      runtimeBinding: `env.${makePropSafe(variable.name)}`,
+      readableBinding: `env.${variable.name}`,
+      category: "Environment",
+      icon: "Key",
+      display: { type: "string", name: variable.name },
+    }
+  })
+}
+
 /**
  * Utility - convert a key/value map to an array of custom 'context' bindings
  * @param {object} valueMap Key/value pairings

packages/builder/src/builderStore/store/frontend.js

@@ -63,6 +63,10 @@ const INITIAL_FRONTEND_STATE = {
   selectedScreenId: null,
   selectedComponentId: null,
   selectedLayoutId: null,
+
+  // onboarding
+  onboarding: false,
+  tourNodes: null,
 }
 
 export const getFrontendStore = () => {

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -18,6 +18,7 @@
 
   import { automationStore } from "builderStore"
   import { tables } from "stores/backend"
+  import { environment, licensing } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
@@ -33,6 +34,7 @@
   import { Utils } from "@budibase/frontend-core"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
   import { cloneDeep } from "lodash/fp"
+  import { onMount } from "svelte"
 
   export let block
   export let testData
@@ -166,6 +168,24 @@
       )
     }
 
+    // Environment bindings
+    if ($licensing.environmentVariablesEnabled) {
+      bindings = bindings.concat(
+        $environment.variables.map(variable => {
+          return {
+            label: `env.${variable.name}`,
+            path: `env.${variable.name}`,
+            icon: "Key",
+            category: "Environment",
+            display: {
+              type: "string",
+              name: variable.name,
+            },
+          }
+        })
+      )
+    }
+
     return bindings
   }
 
@@ -196,6 +216,14 @@
     onChange({ detail: tempFilters }, defKey)
     drawer.hide()
   }
+
+  onMount(async () => {
+    try {
+      await environment.loadVariables()
+    } catch (error) {
+      console.error(error)
+    }
+  })
 </script>
 
 <div class="fields">

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -39,6 +39,23 @@
   $: showError($fetch.error)
   $: id, (filters = null)
 
+  let appliedFilter
+  let rawFilter
+  let appliedSort
+  let selectedRows = []
+
+  $: enrichedSchema,
+    () => {
+      appliedFilter = null
+      rawFilter = null
+      appliedSort = null
+      selectedRows = []
+    }
+
+  $: if (Number.isInteger($fetch.pageNumber)) {
+    selectedRows = []
+  }
+
   const showError = error => {
     if (error) {
       notifications.error(error?.message || "Unable to fetch data.")
@@ -95,11 +112,15 @@
   }
 
   // Fetch data whenever sorting option changes
-  const onSort = e => {
+  const onSort = async e => {
-    fetch.update({
+    const sort = {
       sortColumn: e.detail.column,
       sortOrder: e.detail.order,
-    })
+    }
+    await fetch.update(sort)
+    appliedSort = { ...sort }
+    appliedSort.sortOrder = appliedSort.sortOrder.toLowerCase()
+    selectedRows = []
   }
 
   // Fetch data whenever filters change
@@ -108,16 +129,19 @@
     fetch.update({
       filter: filters,
     })
+    appliedFilter = e.detail
   }
 
   // Fetch data whenever schema changes
   const onUpdateColumns = () => {
+    selectedRows = []
     fetch.refresh()
   }
 
   // Fetch data whenever rows are modified. Unfortunately we have to lose
   // our pagination place, as our bookmarks will have shifted.
   const onUpdateRows = () => {
+    selectedRows = []
     fetch.refresh()
   }
 
@@ -142,6 +166,9 @@
     disableSorting
     on:updatecolumns={onUpdateColumns}
     on:updaterows={onUpdateRows}
+    on:selectionUpdated={e => {
+      selectedRows = e.detail
+    }}
     customPlaceholder
   >
     <div class="buttons">
@@ -183,6 +210,9 @@
         <ExportButton
           disabled={!hasRows || !hasCols}
           view={$tables.selected?._id}
+          filters={appliedFilter}
+          sorting={appliedSort}
+          {selectedRows}
         />
         {#key id}
           <TableFilterButton

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -16,6 +16,7 @@
     UNSORTABLE_TYPES,
   } from "constants"
   import RoleCell from "./cells/RoleCell.svelte"
+  import { createEventDispatcher } from "svelte"
 
   export let schema = {}
   export let data = []
@@ -28,6 +29,8 @@
   export let disableSorting = false
   export let customPlaceholder = false
 
+  const dispatch = createEventDispatcher()
+
   let selectedRows = []
   let editableColumn
   let editableRow
@@ -36,6 +39,7 @@
   let customRenderers = []
   let confirmDelete
 
+  $: selectedRows, dispatch("selectionUpdated", selectedRows)
   $: isUsersTable = tableId === TableNames.USERS
   $: data && resetSelectedRows()
   $: editRowComponent = isUsersTable ? CreateEditUser : CreateEditRow

packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte

@@ -3,7 +3,10 @@
   import ExportModal from "../modals/ExportModal.svelte"
 
   export let view
+  export let filters
+  export let sorting
   export let disabled = false
+  export let selectedRows
 
   let modal
 </script>
@@ -18,5 +21,5 @@
   Export
 </ActionButton>
 <Modal bind:this={modal}>
-  <ExportModal {view} />
+  <ExportModal {view} {filters} {sorting} {selectedRows} />
 </Modal>

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -1,7 +1,14 @@
 <script>
-  import { Select, ModalContent, notifications } from "@budibase/bbui"
+  import {
+    Select,
+    ModalContent,
+    notifications,
+    Body,
+    Table,
+  } from "@budibase/bbui"
   import download from "downloadjs"
   import { API } from "api"
+  import { Constants, LuceneUtils } from "@budibase/frontend-core"
 
   const FORMATS = [
     {
@@ -19,8 +26,71 @@
   ]
 
   export let view
+  export let filters
+  export let sorting
+  export let selectedRows = []
 
   let exportFormat = FORMATS[0].key
+  let filterLookup
+
+  $: luceneFilter = LuceneUtils.buildLuceneQuery(filters)
+  $: exportOpDisplay = buildExportOpDisplay(sorting, filterDisplay, filters)
+
+  const buildFilterLookup = () => {
+    return Object.keys(Constants.OperatorOptions).reduce((acc, key) => {
+      const op = Constants.OperatorOptions[key]
+      acc[op.value] = op.label
+      return acc
+    }, {})
+  }
+  filterLookup = buildFilterLookup()
+
+  const filterDisplay = () => {
+    if (!filters) {
+      return []
+    }
+    return filters.map(filter => {
+      let newFieldName = filter.field + ""
+      const parts = newFieldName.split(":")
+      parts.shift()
+      newFieldName = parts.join(":")
+      return {
+        Field: newFieldName,
+        Operation: filterLookup[filter.operator],
+        "Field Value": filter.value || "",
+      }
+    })
+  }
+
+  const buildExportOpDisplay = (sorting, filterDisplay) => {
+    let filterDisplayConfig = filterDisplay()
+    if (sorting) {
+      filterDisplayConfig = [
+        ...filterDisplayConfig,
+        {
+          Field: sorting.sortColumn,
+          Operation: "Order By",
+          "Field Value": sorting.sortOrder,
+        },
+      ]
+    }
+    return filterDisplayConfig
+  }
+
+  const displaySchema = {
+    Field: {
+      type: "string",
+      fieldName: "Field",
+    },
+    Operation: {
+      type: "string",
+      fieldName: "Operation",
+    },
+    "Field Value": {
+      type: "string",
+      fieldName: "Value",
+    },
+  }
 
   async function exportView() {
     try {
@@ -33,9 +103,74 @@
       notifications.error(`Unable to export ${exportFormat.toUpperCase()} data`)
     }
   }
+
+  async function exportRows() {
+    if (selectedRows?.length) {
+      const data = await API.exportRows({
+        tableId: view,
+        rows: selectedRows.map(row => row._id),
+        format: exportFormat,
+      })
+      download(data, `export.${exportFormat}`)
+    } else if (filters || sorting) {
+      const data = await API.exportRows({
+        tableId: view,
+        format: exportFormat,
+        search: {
+          query: luceneFilter,
+          sort: sorting?.sortColumn,
+          sortOrder: sorting?.sortOrder,
+          paginate: false,
+        },
+      })
+      download(data, `export.${exportFormat}`)
+    } else {
+      await exportView()
+    }
+  }
 </script>
 
-<ModalContent title="Export Data" confirmText="Export" onConfirm={exportView}>
+<ModalContent
+  title="Export Data"
+  confirmText="Export"
+  onConfirm={exportRows}
+  size={filters?.length || sorting ? "M" : "S"}
+>
+  {#if selectedRows?.length}
+    <Body size="S">
+      <strong>{selectedRows?.length}</strong>
+      {`row${selectedRows?.length > 1 ? "s" : ""} will be exported`}
+    </Body>
+  {:else if filters || (sorting?.sortOrder && sorting?.sortColumn)}
+    <Body size="S">
+      {#if !filters}
+        Exporting <strong>all</strong> rows
+      {:else}
+        Filters applied
+      {/if}
+    </Body>
+
+    <div class="table-wrap">
+      <Table
+        schema={displaySchema}
+        data={exportOpDisplay}
+        {filters}
+        loading={false}
+        rowCount={filters?.length + 1}
+        disableSorting={true}
+        allowSelectRows={false}
+        allowEditRows={false}
+        allowEditColumns={false}
+        quiet={true}
+        compact={true}
+      />
+    </div>
+  {:else}
+    <Body size="S">
+      Exporting <strong>all</strong> rows
+    </Body>
+  {/if}
+
   <Select
     label="Format"
     bind:value={exportFormat}
@@ -45,3 +180,9 @@
     getOptionValue={x => x.key}
   />
 </ModalContent>
+
+<style>
+  .table-wrap :global(.wrapper) {
+    max-width: 400px;
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte

@@ -6,17 +6,26 @@
     Toggle,
     Button,
     TextArea,
+    Modal,
+    EnvDropdown,
     Accordion,
+    notifications,
   } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { capitalise } from "helpers"
   import { IntegrationTypes } from "constants/backend"
   import { createValidationStore } from "helpers/validation/yup"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onMount } from "svelte"
+  import { environment, licensing, auth } from "stores/portal"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
 
   export let datasource
   export let schema
   export let creating
+
+  let createVariableModal
+  let selectedKey
+
   const validation = createValidationStore()
   const dispatch = createEventDispatcher()
 
@@ -70,6 +79,37 @@
       .filter(el => filter(el))
       .map(([key]) => key)
   }
+
+  async function save(data) {
+    try {
+      await environment.createVariable(data)
+      config[selectedKey] = `{{ env.${data.name} }}`
+      createVariableModal.hide()
+    } catch (err) {
+      notifications.error(`Failed to create variable: ${err.message}`)
+    }
+  }
+
+  function showModal(configKey) {
+    selectedKey = configKey
+    createVariableModal.show()
+  }
+
+  async function handleUpgradePanel() {
+    await environment.upgradePanelOpened()
+    $licensing.goToUpgradePage()
+  }
+
+  onMount(async () => {
+    try {
+      await environment.loadVariables()
+      if ($auth.user) {
+        await licensing.init()
+      }
+    } catch (err) {
+      console.error(err)
+    }
+  })
 </script>
 
 <form>
@@ -134,11 +174,15 @@
       {:else}
         <div class="form-row">
           <Label>{getDisplayName(configKey)}</Label>
-          <Input
+          <EnvDropdown
+            showModal={() => showModal(configKey)}
+            variables={$environment.variables}
             type={schema[configKey].type}
             on:change
             bind:value={config[configKey]}
             error={$validation.errors[configKey]}
+            environmentVariablesEnabled={$licensing.environmentVariablesEnabled}
+            {handleUpgradePanel}
           />
         </div>
       {/if}
@@ -146,6 +190,10 @@
   </Layout>
 </form>
 
+<Modal bind:this={createVariableModal}>
+  <CreateEditVariableModal {save} />
+</Modal>
+
 <style>
   .form-row {
     display: grid;

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte

@@ -12,10 +12,12 @@
   import ViewDynamicVariables from "./variables/ViewDynamicVariables.svelte"
   import {
     getRestBindings,
+    getEnvironmentBindings,
     readableToRuntimeBinding,
     runtimeToReadableMap,
   } from "builderStore/dataBinding"
   import { cloneDeep } from "lodash/fp"
+  import { licensing } from "stores/portal"
 
   export let datasource
   export let queries
@@ -93,6 +95,9 @@
     headings
     bind:object={datasource.config.staticVariables}
     on:change
+    bindings={$licensing.environmentVariablesEnabled
+      ? getEnvironmentBindings()
+      : []}
   />
 </Layout>
 <div />

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte

@@ -1,9 +1,23 @@
 <script>
   import { onMount } from "svelte"
-  import { ModalContent, Layout, Select, Body, Input } from "@budibase/bbui"
+  import {
+    ModalContent,
+    Layout,
+    Select,
+    Body,
+    Input,
+    EnvDropdown,
+    Modal,
+    notifications,
+  } from "@budibase/bbui"
   import { AUTH_TYPE_LABELS, AUTH_TYPES } from "./authTypes"
   import BindableCombobox from "components/common/bindings/BindableCombobox.svelte"
-  import { getAuthBindings } from "builderStore/dataBinding"
+  import {
+    getAuthBindings,
+    getEnvironmentBindings,
+  } from "builderStore/dataBinding"
+  import { environment, licensing, auth } from "stores/portal"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
 
   export let configs
   export let currentConfig
@@ -28,7 +42,19 @@
   let hasErrors = false
   let hasChanged = false
 
-  onMount(() => {
+  let createVariableModal
+  let formFieldkey
+
+  onMount(async () => {
+    try {
+      await environment.loadVariables()
+      if ($auth.user) {
+        await licensing.init()
+      }
+    } catch (err) {
+      console.error(err)
+    }
+
     if (currentConfig) {
       deconstructConfig()
     }
@@ -146,6 +172,16 @@
     }
   }
 
+  const save = async data => {
+    try {
+      await environment.createVariable(data)
+      form.basic[formFieldkey] = `{{ env.${data.name} }}`
+      createVariableModal.hide()
+    } catch (err) {
+      notifications.error(`Failed to create variable: ${err.message}`)
+    }
+  }
+
   const onFieldChange = () => {
     checkErrors()
     checkChanged()
@@ -154,6 +190,16 @@
   const onConfirmInternal = () => {
     onConfirm(constructConfig())
   }
+
+  async function handleUpgradePanel() {
+    await environment.upgradePanelOpened()
+    $licensing.goToUpgradePage()
+  }
+
+  function showModal(key) {
+    formFieldkey = key
+    createVariableModal.show()
+  }
 </script>
 
 <ModalContent
@@ -189,26 +235,40 @@
       error={blurred.type ? errors.type : null}
     />
     {#if form.type === AUTH_TYPES.BASIC}
-      <Input
+      <EnvDropdown
         label="Username"
         bind:value={form.basic.username}
         on:change={onFieldChange}
         on:blur={() => (blurred.basic.username = true)}
         error={blurred.basic.username ? errors.basic.username : null}
+        showModal={() => showModal("configKey")}
+        variables={$environment.variables}
+        environmentVariablesEnabled={$licensing.environmentVariablesEnabled}
+        {handleUpgradePanel}
       />
-      <Input
+      <EnvDropdown
         label="Password"
+        type="password"
         bind:value={form.basic.password}
         on:change={onFieldChange}
         on:blur={() => (blurred.basic.password = true)}
         error={blurred.basic.password ? errors.basic.password : null}
+        showModal={() => showModal("configKey")}
+        variables={$environment.variables}
+        environmentVariablesEnabled={$licensing.environmentVariablesEnabled}
+        {handleUpgradePanel}
       />
     {/if}
     {#if form.type === AUTH_TYPES.BEARER}
       <BindableCombobox
         label="Token"
         value={form.bearer.token}
-        bindings={getAuthBindings()}
+        bindings={[
+          ...getAuthBindings(),
+          ...($licensing.environmentVariablesEnabled
+            ? getEnvironmentBindings()
+            : []),
+        ]}
         on:change={e => {
           form.bearer.token = e.detail
           onFieldChange()
@@ -226,3 +286,7 @@
     {/if}
   </Layout>
 </ModalContent>
+
+<Modal bind:this={createVariableModal}>
+  <CreateEditVariableModal {save} />
+</Modal>

packages/builder/src/components/deploy/DeployModal.svelte

@@ -11,6 +11,8 @@
   import { store } from "builderStore"
   import { ProgressCircle } from "@budibase/bbui"
   import CopyInput from "components/common/inputs/CopyInput.svelte"
+  import TourWrap from "../portal/onboarding/TourWrap.svelte"
+  import { TOUR_STEP_KEYS } from "../portal/onboarding/tours.js"
 
   let publishModal
   let asyncModal
@@ -54,7 +56,11 @@
   }
 </script>
 
-<Button cta on:click={publishModal.show}>Publish</Button>
+<TourWrap tourStepKey={TOUR_STEP_KEYS.BUILDER_APP_PUBLISH}>
+  <Button cta on:click={publishModal.show} id={"builder-app-publish-button"}>
+    Publish
+  </Button>
+</TourWrap>
 <Modal bind:this={publishModal}>
   <ModalContent
     title="Publish to production"

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -1,6 +1,7 @@
 <script>
   import { goto, params } from "@roxi/routify"
   import { datasources, flags, integrations, queries } from "stores/backend"
+  import { environment } from "stores/portal"
   import {
     Banner,
     Body,
@@ -362,6 +363,13 @@
       notifications.error("Error getting datasources")
     }
 
+    try {
+      // load the environment variables
+      await environment.loadVariables()
+    } catch (error) {
+      notifications.error(`Error getting environment variables - ${error}`)
+    }
+
     datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
     const datasourceUrl = datasource?.config.url
     const qs = query?.fields.queryString

packages/builder/src/components/portal/environment/CreateEditVariableModal.svelte

@@ -0,0 +1,100 @@
+<script>
+  import {
+    ModalContent,
+    Button,
+    Input,
+    Checkbox,
+    Heading,
+    notifications,
+    Context,
+  } from "@budibase/bbui"
+  import { environment } from "stores/portal"
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { getContext } from "svelte"
+  const modalContext = getContext(Context.Modal)
+
+  export let save
+  export let row
+
+  let deleteDialog
+  let name = row?.name || ""
+  let productionValue
+  let developmentValue
+  let useProductionValue = true
+
+  const deleteVariable = async name => {
+    try {
+      await environment.deleteVariable(name)
+      modalContext.hide()
+      notifications.success("Environment variable deleted")
+    } catch (err) {
+      notifications.error(err.message)
+    }
+  }
+
+  const saveVariable = async () => {
+    try {
+      await save({
+        name,
+        production: productionValue,
+        development: developmentValue,
+      })
+      notifications.success("Environment variable saved")
+    } catch (err) {
+      notifications.error(`Error saving environment variable - ${err.message}`)
+    }
+  }
+</script>
+
+<ModalContent
+  onConfirm={() => saveVariable()}
+  title={!row ? "Add new environment variable" : "Edit environment variable"}
+>
+  <Input disabled={row} label="Name" bind:value={name} />
+  <div>
+    <Heading size="XS">Production</Heading>
+    <Input
+      type="password"
+      label="Value"
+      on:change={e => {
+        productionValue = e.detail
+        if (useProductionValue) {
+          developmentValue = e.detail
+        }
+      }}
+      value={productionValue}
+    />
+  </div>
+  <div>
+    <Heading size="XS">Development</Heading>
+    <Input
+      type="password"
+      on:change={e => {
+        developmentValue = e.detail
+      }}
+      disabled={useProductionValue}
+      label="Value"
+      value={useProductionValue ? productionValue : developmentValue}
+    />
+    <Checkbox bind:value={useProductionValue} text="Use production value" />
+  </div>
+
+  <div class="footer" slot="footer">
+    {#if row}
+      <Button on:click={deleteDialog.show} warning>Delete</Button>
+    {/if}
+  </div>
+</ModalContent>
+
+<ConfirmDialog
+  bind:this={deleteDialog}
+  onOk={() => {
+    deleteVariable(row.name)
+  }}
+  okText="Delete Environment Variable"
+  title="Confirm Deletion"
+>
+  Are you sure you wish to delete the environment variable
+  <i>{row.name}?</i>
+  This action cannot be undone.
+</ConfirmDialog>

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -0,0 +1,173 @@
+<script>
+  import { Popover, Layout, Heading, Body, Button } from "@budibase/bbui"
+  import { store } from "builderStore"
+  import { TOURS } from "./tours.js"
+  import { goto, layout, isActive } from "@roxi/routify"
+
+  let popoverAnchor
+  let popover
+  let tourSteps = null
+  let tourStep
+  let tourStepIdx
+  let lastStep
+
+  $: tourNodes = { ...$store.tourNodes }
+  $: tourKey = $store.tourKey
+  $: tourStepKey = $store.tourStepKey
+
+  const initTour = targetKey => {
+    if (!targetKey) {
+      return
+    }
+    tourSteps = [...TOURS[targetKey]]
+    tourStepIdx = 0
+    tourStep = { ...tourSteps[tourStepIdx] }
+  }
+
+  $: initTour(tourKey)
+
+  const updateTourStep = targetStepKey => {
+    if (!tourSteps?.length) {
+      return
+    }
+    tourStepIdx = getCurrentStepIdx(tourSteps, targetStepKey)
+    lastStep = tourStepIdx + 1 == tourSteps.length
+    tourStep = { ...tourSteps[tourStepIdx] }
+    tourStep.onLoad()
+  }
+
+  $: updateTourStep(tourStepKey)
+
+  const showPopover = (tourStep, tourNodes, popover) => {
+    if (!tourStep) {
+      return
+    }
+    popoverAnchor = tourNodes[tourStep.id]
+    popover?.show()
+  }
+
+  $: showPopover(tourStep, tourNodes, popover)
+
+  const navigateStep = step => {
+    if (step.route) {
+      const activeNav = $layout.children.find(c => $isActive(c.path))
+      if (activeNav) {
+        store.update(state => {
+          if (!state.previousTopNavPath) state.previousTopNavPath = {}
+          state.previousTopNavPath[activeNav.path] = window.location.pathname
+          $goto(state.previousTopNavPath[step.route] || step.route)
+          return state
+        })
+      }
+    }
+  }
+
+  const nextStep = async () => {
+    if (!lastStep === true) {
+      let target = tourSteps[tourStepIdx + 1]
+      if (target) {
+        store.update(state => ({
+          ...state,
+          tourStepKey: target.id,
+        }))
+        navigateStep(target)
+      } else {
+        console.log("Could not retrieve step")
+      }
+    } else {
+      if (typeof tourStep.onComplete === "function") {
+        tourStep.onComplete()
+      }
+      popover.hide()
+    }
+  }
+
+  const previousStep = async () => {
+    if (tourStepIdx > 0) {
+      let target = tourSteps[tourStepIdx - 1]
+      if (target) {
+        store.update(state => ({
+          ...state,
+          tourStepKey: target.id,
+        }))
+        navigateStep(target)
+      } else {
+        console.log("Could not retrieve step")
+      }
+    }
+  }
+
+  const getCurrentStepIdx = (steps, tourStepKey) => {
+    if (!steps?.length) {
+      return
+    }
+    if (steps?.length && !tourStepKey) {
+      return 0
+    }
+    return steps.findIndex(step => step.id === tourStepKey)
+  }
+</script>
+
+{#key tourStepKey}
+  <Popover
+    align={tourStep?.align}
+    bind:this={popover}
+    anchor={popoverAnchor}
+    dataCy="tour-popover-menu"
+    maxWidth={300}
+    dismissible={false}
+  >
+    <Layout gap="M">
+      <div class="tour-header">
+        <Heading size="XS">{tourStep?.title || "-"}</Heading>
+        <div>{`${tourStepIdx + 1}/${tourSteps?.length}`}</div>
+      </div>
+      <Body size="S">
+        <span class="tour-body">
+          {#if tourStep.layout}
+            <svelte:component this={tourStep.layout} />
+          {:else}
+            {tourStep?.body || ""}
+          {/if}
+        </span>
+      </Body>
+      <div class="tour-footer">
+        <div class="tour-navigation">
+          {#if tourStepIdx > 0}
+            <Button
+              secondary
+              on:click={previousStep}
+              disabled={tourStepIdx == 0}
+            >
+              <div>Back</div>
+            </Button>
+          {/if}
+          <Button cta on:click={nextStep}>
+            <div>{lastStep ? "Finish" : "Next"}</div>
+          </Button>
+        </div>
+      </div>
+    </Layout>
+  </Popover>
+{/key}
+
+<style>
+  .tour-navigation {
+    grid-gap: var(--spectrum-alias-grid-baseline);
+    display: flex;
+    justify-content: end;
+  }
+  :global([data-cy="tour-popover-menu"]) {
+    padding: 10px;
+    margin-top: var(--spacing-l);
+  }
+  .tour-body :global(.feature-list) {
+    margin-bottom: 0px;
+    padding-left: var(--spacing-xl);
+  }
+  .tour-header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+  }
+</style>

packages/builder/src/components/portal/onboarding/TourWrap.svelte

@@ -0,0 +1,29 @@
+<script>
+  import { tourHandler } from "./tourHandler"
+  import { TOURS } from "./tours"
+  import { onMount, onDestroy } from "svelte"
+  import { store } from "builderStore"
+
+  export let tourStepKey
+
+  let currentTour
+  let ready = false
+  let handler
+
+  onMount(() => {
+    if (!$store.tourKey) return
+
+    currentTour = TOURS[$store.tourKey].find(step => step.id === tourStepKey)
+
+    const elem = document.querySelector(currentTour.query)
+    handler = tourHandler(elem, tourStepKey)
+    ready = true
+  })
+  onDestroy(() => {
+    if (handler) {
+      handler.destroy()
+    }
+  })
+</script>
+
+<slot />

packages/builder/src/components/portal/onboarding/steps/OnboardingData.svelte

@@ -0,0 +1,10 @@
+<div>
+  In this section you can mange the data for your app:
+  <ul class="feature-list">
+    <li>Connect data sources</li>
+    <li>Edit data</li>
+    <li>Manage read & write access</li>
+    <li>Create views</li>
+    <li>Add bindings</li>
+  </ul>
+</div>

packages/builder/src/components/portal/onboarding/steps/OnboardingDesign.svelte

@@ -0,0 +1,10 @@
+<div>
+  After setting up your data, Design is where you build the screens for your
+  app:
+  <ul class="feature-list">
+    <li>Add screens</li>
+    <li>Add components</li>
+    <li>Choose your theme</li>
+    <li>Edit navigation</li>
+  </ul>
+</div>

packages/builder/src/components/portal/onboarding/steps/OnboardingPublish.svelte

@@ -0,0 +1,7 @@
+<div>
+  Once you’re happy with your app you can publish it to production!
+  <p>
+    After publishing, any changes you make will not take affect until you next
+    publish.
+  </p>
+</div>

packages/builder/src/components/portal/onboarding/steps/index.js

@@ -0,0 +1,3 @@
+export { default as OnboardingData } from "./OnboardingData.svelte"
+export { default as OnboardingDesign } from "./OnboardingDesign.svelte"
+export { default as OnboardingPublish } from "./OnboardingPublish.svelte"

packages/builder/src/components/portal/onboarding/tourHandler.js

@@ -0,0 +1,47 @@
+import { store } from "builderStore/index"
+import { get } from "svelte/store"
+
+const registerNode = async (node, tourStepKey) => {
+  if (!node) {
+    console.log("Tour Handler - an anchor node is required")
+  }
+
+  if (!get(store).tourKey) {
+    console.log("Tour Handler - No active tour ", tourStepKey, node)
+    return
+  }
+
+  store.update(state => {
+    const update = {
+      ...state,
+      tourNodes: {
+        ...state.tourNodes,
+        [tourStepKey]: node,
+      },
+    }
+    return update
+  })
+}
+
+export function tourHandler(node, tourStepKey) {
+  if (node && tourStepKey) {
+    registerNode(node, tourStepKey)
+  }
+  return {
+    destroy: () => {
+      const updatedTourNodes = get(store).tourNodes
+      if (updatedTourNodes && updatedTourNodes[tourStepKey]) {
+        delete updatedTourNodes[tourStepKey]
+        store.update(state => {
+          const update = {
+            ...state,
+            tourNodes: {
+              ...updatedTourNodes,
+            },
+          }
+          return update
+        })
+      }
+    },
+  }
+}

packages/builder/src/components/portal/onboarding/tours.js

@@ -0,0 +1,95 @@
+import { get } from "svelte/store"
+import { store } from "builderStore"
+import { users, auth } from "stores/portal"
+import analytics from "analytics"
+import { OnboardingData, OnboardingDesign, OnboardingPublish } from "./steps"
+const ONBOARDING_EVENT_PREFIX = "onboarding"
+
+export const TOUR_STEP_KEYS = {
+  BUILDER_APP_PUBLISH: "builder-app-publish",
+  BUILDER_DATA_SECTION: "builder-data-section",
+  BUILDER_DESIGN_SECTION: "builder-design-section",
+  BUILDER_AUTOMATE_SECTION: "builder-automate-section",
+}
+
+export const TOUR_KEYS = {
+  TOUR_BUILDER_ONBOARDING: "builder-onboarding",
+}
+
+const tourEvent = eventKey => {
+  analytics.captureEvent(`${ONBOARDING_EVENT_PREFIX}:${eventKey}`, {
+    eventSource: EventSource.PORTAL,
+  })
+}
+
+const getTours = () => {
+  return {
+    [TOUR_KEYS.TOUR_BUILDER_ONBOARDING]: [
+      {
+        id: TOUR_STEP_KEYS.BUILDER_DATA_SECTION,
+        title: "Data",
+        route: "/builder/app/:application/data",
+        layout: OnboardingData,
+        query: ".topcenternav .spectrum-Tabs-item#builder-data-tab",
+        onLoad: async () => {
+          tourEvent(TOUR_STEP_KEYS.BUILDER_DATA_SECTION)
+        },
+        align: "left",
+      },
+      {
+        id: TOUR_STEP_KEYS.BUILDER_DESIGN_SECTION,
+        title: "Design",
+        route: "/builder/app/:application/design",
+        layout: OnboardingDesign,
+        query: ".topcenternav .spectrum-Tabs-item#builder-design-tab",
+        onLoad: () => {
+          tourEvent(TOUR_STEP_KEYS.BUILDER_DESIGN_SECTION)
+        },
+        align: "left",
+      },
+      {
+        id: TOUR_STEP_KEYS.BUILDER_AUTOMATE_SECTION,
+        title: "Automations",
+        route: "/builder/app/:application/automate",
+        query: ".topcenternav .spectrum-Tabs-item#builder-automate-tab",
+        body: "Once you have your app screens made, you can set up automations to fit in with your current workflow",
+        onLoad: () => {
+          tourEvent(TOUR_STEP_KEYS.BUILDER_AUTOMATE_SECTION)
+        },
+        align: "left",
+      },
+      {
+        id: TOUR_STEP_KEYS.BUILDER_APP_PUBLISH,
+        title: "Publish",
+        layout: OnboardingPublish,
+        query: ".toprightnav #builder-app-publish-button",
+        onLoad: () => {
+          tourEvent(TOUR_STEP_KEYS.BUILDER_APP_PUBLISH)
+        },
+        onComplete: async () => {
+          // Mark the users onboarding as complete
+          // Clear all tour related state
+          if (get(auth).user) {
+            await users.save({
+              ...get(auth).user,
+              onboardedAt: new Date().toISOString(),
+            })
+
+            // Update the cached user
+            await auth.getSelf()
+
+            store.update(state => ({
+              ...state,
+              tourNodes: undefined,
+              tourKey: undefined,
+              tourKeyStep: undefined,
+              onboarding: false,
+            }))
+          }
+        },
+      },
+    ],
+  }
+}
+
+export const TOURS = getTours()

packages/builder/src/components/start/ExportAppModal.svelte

@@ -1,5 +1,6 @@
 <script>
-  import { ModalContent, Toggle, Body } from "@budibase/bbui"
+  import { ModalContent, Toggle, Body, InlineAlert } from "@budibase/bbui"
+  import { licensing } from "stores/portal"
 
   export let app
   export let published
@@ -16,6 +17,11 @@
 </script>
 
 <ModalContent {title} {confirmText} onConfirm={exportApp}>
+  {#if licensing.environmentVariablesEnabled}
+    <InlineAlert
+      header="Do not share your budibase application exports publicly as they may contain sensitive information such as database credentials or secret keys."
+    />
+  {/if}
   <Body
     >Apps can be exported with or without data that is within internal tables -
     select this below.</Body

packages/builder/src/pages/builder/admin/index.svelte

@@ -4,37 +4,45 @@
     Heading,
     notifications,
     Layout,
-    Input,
     Body,
-    ActionButton,
     Modal,
   } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import { API } from "api"
   import { admin, auth } from "stores/portal"
-  import PasswordRepeatInput from "components/common/users/PasswordRepeatInput.svelte"
   import ImportAppsModal from "./_components/ImportAppsModal.svelte"
   import Logo from "assets/bb-emblem.svg"
   import { onMount } from "svelte"
+  import { FancyForm, FancyInput, ActionButton } from "@budibase/bbui"
+  import { TestimonialPage } from "@budibase/frontend-core/src/components"
+  import { passwordsMatch, handleError } from "../auth/_components/utils"
 
-  let adminUser = {}
-  let error
   let modal
+  let form
+  let errors = {}
+  let formData = {}
+  let submitted = false
 
   $: tenantId = $auth.tenantId
-  $: multiTenancyEnabled = $admin.multiTenancy
   $: cloud = $admin.cloud
   $: imported = $admin.importComplete
 
   async function save() {
+    form.validate()
+    if (Object.keys(errors).length > 0) {
+      return
+    }
+    submitted = true
     try {
-      adminUser.tenantId = tenantId
+      let adminUser = { ...formData, tenantId }
+      delete adminUser.confirmationPassword
       // Save the admin user
       await API.createAdminUser(adminUser)
       notifications.success("Admin user created")
       await admin.init()
       $goto("../portal")
     } catch (error) {
+      submitted = false
       notifications.error("Failed to create admin user")
     }
   }
@@ -53,35 +61,103 @@
 <Modal bind:this={modal} padding={false} width="600px">
   <ImportAppsModal />
 </Modal>
-<section>
+
-  <div class="container">
+<TestimonialPage>
-    <Layout>
+  <Layout gap="M" noPadding>
+    <Layout justifyItems="center" noPadding>
       <img alt="logo" src={Logo} />
-      <Layout gap="XS" justifyItems="center" noPadding>
+      <Heading size="M">Create an admin user</Heading>
-        <Heading size="M">Create an admin user</Heading>
+      <Body>The admin user has access to everything in Budibase.</Body>
-        <Body size="M" textAlign="center">
+    </Layout>
-          The admin user has access to everything in Budibase.
+    <Layout gap="S" noPadding>
-        </Body>
+      <FancyForm bind:this={form}>
-      </Layout>
+        <FancyInput
-      <Layout gap="XS" noPadding>
+          label="Email"
-        <Input label="Email" bind:value={adminUser.email} />
+          value={formData.email}
-        <PasswordRepeatInput bind:password={adminUser.password} bind:error />
+          on:change={e => {
-      </Layout>
+            formData = {
-      <Layout gap="XS" noPadding>
+              ...formData,
-        <Button cta disabled={error} on:click={save}>
+              email: e.detail,
-          Create super admin user
+            }
-        </Button>
+          }}
-        {#if multiTenancyEnabled}
+          validate={() => {
-          <ActionButton
+            let fieldError = {
-            quiet
+              email: !formData.email ? "Please enter a valid email" : undefined,
-            on:click={() => {
+            }
-              admin.unload()
+            errors = handleError({ ...errors, ...fieldError })
-              $goto("../auth/org")
+          }}
-            }}
+          disabled={submitted}
-          >
+          error={errors.email}
-            Change organisation
+        />
-          </ActionButton>
+        <FancyInput
-        {:else if !cloud && !imported}
+          label="Password"
+          value={formData.password}
+          type="password"
+          on:change={e => {
+            formData = {
+              ...formData,
+              password: e.detail,
+            }
+          }}
+          validate={() => {
+            let fieldError = {}
+
+            fieldError["password"] = !formData.password
+              ? "Please enter a password"
+              : undefined
+
+            fieldError["confirmationPassword"] =
+              !passwordsMatch(
+                formData.password,
+                formData.confirmationPassword
+              ) && formData.confirmationPassword
+                ? "Passwords must match"
+                : undefined
+
+            errors = handleError({ ...errors, ...fieldError })
+          }}
+          error={errors.password}
+          disabled={submitted}
+        />
+        <FancyInput
+          label="Repeat Password"
+          value={formData.confirmationPassword}
+          type="password"
+          on:change={e => {
+            formData = {
+              ...formData,
+              confirmationPassword: e.detail,
+            }
+          }}
+          validate={() => {
+            let fieldError = {
+              confirmationPassword:
+                !passwordsMatch(
+                  formData.password,
+                  formData.confirmationPassword
+                ) && formData.password
+                  ? "Passwords must match"
+                  : undefined,
+            }
+            errors = handleError({ ...errors, ...fieldError })
+          }}
+          error={errors.confirmationPassword}
+          disabled={submitted}
+        />
+      </FancyForm>
+    </Layout>
+    <Layout gap="XS" noPadding justifyItems="center">
+      <Button
+        cta
+        disabled={Object.keys(errors).length > 0 || submitted}
+        on:click={save}
+      >
+        Create super admin user
+      </Button>
+    </Layout>
+    <Layout gap="XS" noPadding justifyItems="center">
+      <div class="user-actions">
+        {#if !cloud && !imported}
           <ActionButton
             quiet
             on:click={() => {
@@ -91,28 +167,13 @@
             Import from cloud
           </ActionButton>
         {/if}
-      </Layout>
+      </div>
     </Layout>
-  </div>
+  </Layout>
-</section>
+</TestimonialPage>
 
 <style>
-  section {
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    height: 100%;
-  }
-  .container {
-    margin: 0 auto;
-    width: 260px;
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: stretch;
-  }
   img {
     width: 48px;
-    margin: 0 auto;
   }
 </style>

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -1,6 +1,7 @@
 <script>
   import { store, automationStore } from "builderStore"
   import { roles, flags } from "stores/backend"
+  import { auth } from "stores/portal"
   import {
     ActionMenu,
     MenuItem,
@@ -10,6 +11,7 @@
     Heading,
     notifications,
   } from "@budibase/bbui"
+
   import RevertModal from "components/deploy/RevertModal.svelte"
   import VersionModal from "components/deploy/VersionModal.svelte"
   import DeployNavigation from "components/deploy/DeployNavigation.svelte"
@@ -17,6 +19,9 @@
   import { isActive, goto, layout, redirect } from "@roxi/routify"
   import { capitalise } from "helpers"
   import { onMount, onDestroy } from "svelte"
+  import TourWrap from "components/portal/onboarding/TourWrap.svelte"
+  import TourPopover from "components/portal/onboarding/TourPopover.svelte"
+  import { TOUR_KEYS, TOURS } from "components/portal/onboarding/tours.js"
 
   export let application
 
@@ -62,6 +67,23 @@
     })
   }
 
+  const initTour = async () => {
+    if (!$auth.user?.onboardedAt) {
+      // Determine the correct step
+      const activeNav = $layout.children.find(c => $isActive(c.path))
+      const onboardingTour = TOURS[TOUR_KEYS.TOUR_BUILDER_ONBOARDING]
+      const targetStep = activeNav
+        ? onboardingTour.find(step => step.route === activeNav?.path)
+        : null
+      await store.update(state => ({
+        ...state,
+        onboarding: true,
+        tourKey: TOUR_KEYS.TOUR_BUILDER_ONBOARDING,
+        tourStepKey: targetStep?.id,
+      }))
+    }
+  }
+
   onMount(async () => {
     if (!hasSynced && application) {
       try {
@@ -69,6 +91,7 @@
         // check if user has beta access
         // const betaResponse = await API.checkBetaAccess($auth?.user?.email)
         // betaAccess = betaResponse.access
+        initTour()
       } catch (error) {
         notifications.error("Failed to sync with production database")
       }
@@ -88,6 +111,7 @@
   <!-- This should probably be some kind of loading state? -->
   <div class="loading" />
 {:then _}
+  <TourPopover />
   <div class="root">
     <div class="top-nav">
       <div class="topleftnav">
@@ -140,12 +164,15 @@
       <div class="topcenternav">
         <Tabs {selected} size="M">
           {#each $layout.children as { path, title }}
-            <Tab
+            <TourWrap tourStepKey={`builder-${title}-section`}>
-              quiet
+              <Tab
-              selected={$isActive(path)}
+                quiet
-              on:click={topItemNavigate(path)}
+                selected={$isActive(path)}
-              title={capitalise(title)}
+                on:click={topItemNavigate(path)}
-            />
+                title={capitalise(title)}
+                id={`builder-${title}-tab`}
+              />
+            </TourWrap>
           {/each}
         </Tabs>
       </div>

packages/builder/src/pages/builder/auth/_components/GoogleButton.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { ActionButton } from "@budibase/bbui"
+  import { FancyButton } from "@budibase/bbui"
   import GoogleLogo from "assets/google-logo.png"
   import { auth, organisation } from "stores/portal"
 
@@ -10,31 +10,11 @@
 </script>
 
 {#if show}
-  <ActionButton
+  <FancyButton
+    icon={GoogleLogo}
     on:click={() =>
       window.open(`/api/global/auth/${tenantId}/google`, "_blank")}
   >
-    <div class="inner">
+    Log in with Google
-      <img src={GoogleLogo} alt="google icon" />
+  </FancyButton>
-      <p>Sign in with Google</p>
-    </div>
-  </ActionButton>
 {/if}
-
-<style>
-  .inner {
-    display: flex;
-    flex-direction: row;
-    align-items: center;
-    justify-content: center;
-    padding-top: var(--spacing-xs);
-    padding-bottom: var(--spacing-xs);
-  }
-  .inner img {
-    width: 18px;
-    margin: 3px 10px 3px 3px;
-  }
-  .inner p {
-    margin: 0;
-  }
-</style>

packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { ActionButton, notifications } from "@budibase/bbui"
+  import { notifications, FancyButton } from "@budibase/bbui"
   import OidcLogo from "assets/oidc-logo.png"
   import Auth0Logo from "assets/auth0-logo.png"
   import MicrosoftLogo from "assets/microsoft-logo.png"
@@ -33,34 +33,14 @@
 </script>
 
 {#if show}
-  <ActionButton
+  <FancyButton
+    icon={src}
     on:click={() =>
       window.open(
         `/api/global/auth/${$auth.tenantId}/oidc/configs/${$oidc.uuid}`,
         "_blank"
       )}
   >
-    <div class="inner">
+    {`Log in with ${$oidc.name || "OIDC"}`}
-      <img {src} alt="oidc icon" />
+  </FancyButton>
-      <p>{`Sign in with ${$oidc.name || "OIDC"}`}</p>
-    </div>
-  </ActionButton>
 {/if}
-
-<style>
-  .inner {
-    display: flex;
-    flex-direction: row;
-    align-items: center;
-    justify-content: center;
-    padding-top: var(--spacing-xs);
-    padding-bottom: var(--spacing-xs);
-  }
-  .inner img {
-    width: 18px;
-    margin: 3px 10px 3px 3px;
-  }
-  .inner p {
-    margin: 0;
-  }
-</style>

packages/builder/src/pages/builder/auth/_components/utils.js

@@ -0,0 +1,17 @@
+export const handleError = err => {
+  let update = { ...err }
+  return Object.keys(update).reduce((acc, key) => {
+    if (update[key]) {
+      acc[key] = update[key]
+    }
+    return acc
+  }, {})
+}
+
+export const passwordsMatch = (password, confirmation) => {
+  let confirm = confirmation?.trim()
+  let pwd = password?.trim()
+  return (
+    typeof confirm === "string" && typeof pwd === "string" && confirm == pwd
+  )
+}

packages/builder/src/pages/builder/auth/forgot.svelte

@@ -1,25 +1,35 @@
 <script>
   import {
     notifications,
-    Input,
     Button,
     Layout,
     Body,
     Heading,
-    ActionButton,
+    Icon,
   } from "@budibase/bbui"
   import { organisation, auth } from "stores/portal"
   import Logo from "assets/bb-emblem.svg"
   import { onMount } from "svelte"
   import { goto } from "@roxi/routify"
+  import { TestimonialPage } from "@budibase/frontend-core/src/components"
+  import { FancyForm, FancyInput } from "@budibase/bbui"
 
   let email = ""
+  let form
+  let error
+  let submitted = false
 
   async function forgot() {
+    form.validate()
+    if (error) {
+      return
+    }
+    submitted = true
     try {
       await auth.forgotPassword(email)
       notifications.success("Email sent - please check your inbox")
     } catch (err) {
+      submitted = false
       notifications.error("Unable to send reset password link")
     }
   }
@@ -33,45 +43,64 @@
   })
 </script>
 
-<div class="login">
+<TestimonialPage>
-  <div class="main">
+  <Layout gap="S" noPadding>
-    <Layout>
+    <img alt="logo" src={$organisation.logoUrl || Logo} />
-      <Layout noPadding justifyItems="center">
+    <span class="heading-wrap">
-        <img alt="logo" src={$organisation.logoUrl || Logo} />
+      <Heading size="M">
-      </Layout>
+        <div class="heading-content">
-      <Layout gap="XS" noPadding>
+          <span class="back-chev" on:click={() => $goto("../")}>
-        <Heading textAlign="center">Forgotten your password?</Heading>
+            <Icon name="ChevronLeft" size="XL" />
-        <Body size="S" textAlign="center">
+          </span>
-          No problem! Just enter your account's email address and we'll send you
+          Forgotten your password?
-          a link to reset it.
+        </div>
-        </Body>
+      </Heading>
-        <Input label="Email" bind:value={email} />
+    </span>
-      </Layout>
+    <Layout gap="XS" noPadding>
-      <Layout gap="XS" nopadding>
+      <Body size="M">
-        <Button cta on:click={forgot} disabled={!email}>
+        No problem! Just enter your account's email address and we'll send you a
-          Reset your password
+        link to reset it.
-        </Button>
+      </Body>
-        <ActionButton quiet on:click={() => $goto("../")}>Back</ActionButton>
-      </Layout>
     </Layout>
-  </div>
+
-</div>
+    <Layout gap="S" noPadding>
+      <FancyForm bind:this={form}>
+        <FancyInput
+          label="Email"
+          value={email}
+          on:change={e => {
+            email = e.detail
+          }}
+          validate={() => {
+            if (!email) {
+              return "Please enter your email"
+            }
+            return null
+          }}
+          {error}
+          disabled={submitted}
+        />
+      </FancyForm>
+    </Layout>
+    <div>
+      <Button disabled={!email || error || submitted} cta on:click={forgot}>
+        Reset password
+      </Button>
+    </div>
+  </Layout>
+</TestimonialPage>
 
 <style>
-  .login {
-    width: 100%;
-    height: 100%;
-    display: flex;
-    flex-direction: column;
-    justify-content: center;
-    align-items: center;
-  }
-
-  .main {
-    width: 300px;
-  }
-
   img {
     width: 48px;
   }
+  .back-chev {
+    display: inline-block;
+    cursor: pointer;
+    margin-left: -5px;
+  }
+  .heading-content {
+    display: flex;
+    align-items: center;
+  }
 </style>

packages/builder/src/pages/builder/auth/login.svelte

@@ -5,7 +5,6 @@
     Button,
     Divider,
     Heading,
-    Input,
     Layout,
     notifications,
     Link,
@@ -14,22 +13,30 @@
   import { auth, organisation, oidc, admin } from "stores/portal"
   import GoogleButton from "./_components/GoogleButton.svelte"
   import OIDCButton from "./_components/OIDCButton.svelte"
+  import { handleError } from "./_components/utils"
   import Logo from "assets/bb-emblem.svg"
+  import { TestimonialPage } from "@budibase/frontend-core/src/components"
+  import { FancyForm, FancyInput } from "@budibase/bbui"
   import { onMount } from "svelte"
 
-  let username = ""
-  let password = ""
   let loaded = false
+  let form
+  let errors = {}
+  let formData = {}
 
   $: company = $organisation.company || "Budibase"
-  $: multiTenancyEnabled = $admin.multiTenancy
   $: cloud = $admin.cloud
 
   async function login() {
+    form.validate()
+    if (Object.keys(errors).length > 0) {
+      console.log("errors")
+      return
+    }
     try {
       await auth.login({
-        username: username.trim(),
+        username: formData?.username.trim(),
-        password,
+        password: formData?.password,
       })
       if ($auth?.user?.forceResetPassword) {
         $goto("./reset")
@@ -57,75 +64,96 @@
 </script>
 
 <svelte:window on:keydown={handleKeydown} />
-<div class="login">
+
-  <div class="main">
+<TestimonialPage>
-    <Layout>
+  <Layout gap="S" noPadding>
-      <Layout noPadding justifyItems="center">
+    <Layout justifyItems="center" noPadding>
-        <img alt="logo" src={$organisation.logoUrl || Logo} />
-        <Heading textAlign="center">Sign in to {company}</Heading>
-      </Layout>
       {#if loaded}
-        <GoogleButton />
+        <img alt="logo" src={$organisation.logoUrl || Logo} />
-        <OIDCButton oidcIcon={$oidc.logo} oidcName={$oidc.name} />
-      {/if}
-      <Divider noGrid />
-      <Layout gap="XS" noPadding>
-        <Body size="S" textAlign="center">Sign in with email</Body>
-        <Input label="Email" bind:value={username} />
-        <Input
-          label="Password"
-          type="password"
-          on:change
-          bind:value={password}
-        />
-      </Layout>
-      <Layout gap="XS" noPadding>
-        <Button cta disabled={!username && !password} on:click={login}
-          >Sign in to {company}</Button
-        >
-        <ActionButton quiet on:click={() => $goto("./forgot")}>
-          Forgot password?
-        </ActionButton>
-        {#if multiTenancyEnabled && !cloud}
-          <ActionButton
-            quiet
-            on:click={() => {
-              admin.unload()
-              $goto("./org")
-            }}
-          >
-            Change organisation
-          </ActionButton>
-        {/if}
-      </Layout>
-      {#if cloud}
-        <Body size="xs" textAlign="center">
-          By using Budibase Cloud
-          <br />
-          you are agreeing to our
-          <Link href="https://budibase.com/eula" target="_blank"
-            >License Agreement</Link
-          >
-        </Body>
       {/if}
+      <Heading size="M">Log in to Budibase</Heading>
     </Layout>
-  </div>
+    <Layout gap="S" noPadding>
-</div>
+      {#if loaded && ($organisation.google || $organisation.oidc)}
+        <FancyForm>
+          <OIDCButton oidcIcon={$oidc.logo} oidcName={$oidc.name} />
+          <GoogleButton />
+        </FancyForm>
+        <Divider />
+      {/if}
+      <FancyForm bind:this={form}>
+        <FancyInput
+          label="Your work email"
+          value={formData.username}
+          on:change={e => {
+            formData = {
+              ...formData,
+              username: e.detail,
+            }
+          }}
+          validate={() => {
+            let fieldError = {
+              username: !formData.username
+                ? "Please enter a valid email"
+                : undefined,
+            }
+            errors = handleError({ ...errors, ...fieldError })
+          }}
+          error={errors.username}
+        />
+        <FancyInput
+          label="Password"
+          value={formData.password}
+          type="password"
+          on:change={e => {
+            formData = {
+              ...formData,
+              password: e.detail,
+            }
+          }}
+          validate={() => {
+            let fieldError = {
+              password: !formData.password
+                ? "Please enter your password"
+                : undefined,
+            }
+            errors = handleError({ ...errors, ...fieldError })
+          }}
+          error={errors.password}
+        />
+      </FancyForm>
+    </Layout>
+    <Layout gap="XS" noPadding justifyItems="center">
+      <Button cta disabled={Object.keys(errors).length > 0} on:click={login}>
+        Log in to {company}
+      </Button>
+    </Layout>
+    <Layout gap="XS" noPadding justifyItems="center">
+      <div class="user-actions">
+        <ActionButton quiet on:click={() => $goto("./forgot")}>
+          Forgot password
+        </ActionButton>
+      </div>
+    </Layout>
+
+    {#if cloud}
+      <Body size="xs" textAlign="center">
+        By using Budibase Cloud
+        <br />
+        you are agreeing to our
+        <Link href="https://budibase.com/eula" target="_blank" secondary={true}>
+          License Agreement
+        </Link>
+      </Body>
+    {/if}
+  </Layout>
+</TestimonialPage>
 
 <style>
-  .login {
+  .user-actions {
-    width: 100%;
-    height: 100%;
     display: flex;
-    flex-direction: column;
-    justify-content: center;
     align-items: center;
   }
-
-  .main {
-    width: 300px;
-  }
-
   img {
     width: 48px;
   }

packages/builder/src/pages/builder/auth/reset.svelte

@@ -1,31 +1,43 @@
 <script>
   import { Body, Button, Heading, Layout, notifications } from "@budibase/bbui"
   import { goto, params } from "@roxi/routify"
-  import PasswordRepeatInput from "components/common/users/PasswordRepeatInput.svelte"
   import { auth, organisation } from "stores/portal"
   import Logo from "assets/bb-emblem.svg"
+  import { TestimonialPage } from "@budibase/frontend-core/src/components"
+  import { FancyForm, FancyInput } from "@budibase/bbui"
   import { onMount } from "svelte"
+  import { handleError, passwordsMatch } from "./_components/utils"
 
   const resetCode = $params["?code"]
-  let password, error
+  let form
+  let formData = {}
+  let errors = {}
+  let loaded = false
 
+  $: submitted = false
   $: forceResetPassword = $auth?.user?.forceResetPassword
 
   async function reset() {
+    form.validate()
+    if (Object.keys(errors).length > 0) {
+      return
+    }
+    submitted = true
     try {
       if (forceResetPassword) {
         await auth.updateSelf({
-          password,
+          password: formData.password,
           forceResetPassword: false,
         })
         $goto("../portal/")
       } else {
-        await auth.resetPassword(password, resetCode)
+        await auth.resetPassword(formData.password, resetCode)
         notifications.success("Password reset successfully")
         // send them to login if reset successful
         $goto("./login")
       }
     } catch (err) {
+      submitted = false
       notifications.error("Unable to reset password")
     }
   }
@@ -37,47 +49,92 @@
     } catch (error) {
       notifications.error("Error getting org config")
     }
+    loaded = true
   })
 </script>
 
-<div class="login">
+<TestimonialPage>
-  <div class="main">
+  <Layout gap="S" noPadding>
-    <Layout>
+    {#if loaded}
-      <Layout noPadding justifyItems="center">
+      <img alt="logo" src={$organisation.logoUrl || Logo} />
-        <img src={$organisation.logoUrl || Logo} alt="Organisation logo" />
+    {/if}
-      </Layout>
+    <Layout gap="XS" noPadding>
-      <Layout gap="XS" noPadding>
+      <Heading size="M">Reset your password</Heading>
-        <Heading textAlign="center">Reset your password</Heading>
+      <Body size="M">Please enter the new password you'd like to use.</Body>
-        <Body size="S" textAlign="center">
-          Please enter the new password you'd like to use.
-        </Body>
-        <PasswordRepeatInput bind:password bind:error />
-      </Layout>
-      <Button
-        cta
-        on:click={reset}
-        disabled={error || (forceResetPassword ? false : !resetCode)}
-      >
-        Reset your password
-      </Button>
     </Layout>
-  </div>
+
-</div>
+    <Layout gap="S" noPadding>
+      <FancyForm bind:this={form}>
+        <FancyInput
+          label="Password"
+          value={formData.password}
+          type="password"
+          on:change={e => {
+            formData = {
+              ...formData,
+              password: e.detail,
+            }
+          }}
+          validate={() => {
+            let fieldError = {}
+
+            fieldError["password"] = !formData.password
+              ? "Please enter a password"
+              : undefined
+
+            fieldError["confirmationPassword"] =
+              !passwordsMatch(
+                formData.password,
+                formData.confirmationPassword
+              ) && formData.confirmationPassword
+                ? "Passwords must match"
+                : undefined
+
+            errors = handleError({ ...errors, ...fieldError })
+          }}
+          error={errors.password}
+          disabled={submitted}
+        />
+        <FancyInput
+          label="Repeat Password"
+          value={formData.confirmationPassword}
+          type="password"
+          on:change={e => {
+            formData = {
+              ...formData,
+              confirmationPassword: e.detail,
+            }
+          }}
+          validate={() => {
+            const isValid =
+              !passwordsMatch(
+                formData.password,
+                formData.confirmationPassword
+              ) && formData.password
+
+            let fieldError = {
+              confirmationPassword: isValid ? "Passwords must match" : null,
+            }
+
+            errors = handleError({ ...errors, ...fieldError })
+          }}
+          error={errors.confirmationPassword}
+          disabled={submitted}
+        />
+      </FancyForm>
+    </Layout>
+    <div>
+      <Button
+        disabled={Object.keys(errors).length > 0 ||
+          (forceResetPassword ? false : !resetCode)}
+        cta
+        on:click={reset}>Reset your password</Button
+      >
+    </div>
+  </Layout>
+</TestimonialPage>
 
 <style>
-  .login {
-    width: 100%;
-    height: 100%;
-    display: flex;
-    flex-direction: column;
-    justify-content: center;
-    align-items: center;
-  }
-
-  .main {
-    width: 260px;
-  }
-
   img {
     width: 48px;
   }

packages/builder/src/pages/builder/invite/index.svelte

@@ -1,70 +1,192 @@
 <script>
   import { Layout, Heading, Body, Button, notifications } from "@budibase/bbui"
   import { goto, params } from "@roxi/routify"
-  import { users, organisation } from "stores/portal"
+  import { users, organisation, auth } from "stores/portal"
-  import PasswordRepeatInput from "components/common/users/PasswordRepeatInput.svelte"
   import Logo from "assets/bb-emblem.svg"
+  import { TestimonialPage } from "@budibase/frontend-core/src/components"
+  import { FancyForm, FancyInput } from "@budibase/bbui"
   import { onMount } from "svelte"
+  import { handleError, passwordsMatch } from "../auth/_components/utils"
 
   const inviteCode = $params["?code"]
-  let password, error
+  let form
+  let formData = {}
+  let onboarding = false
+  let errors = {}
 
   $: company = $organisation.company || "Budibase"
 
   async function acceptInvite() {
+    form.validate()
+    if (Object.keys(errors).length > 0) {
+      return
+    }
+    onboarding = true
     try {
-      await users.acceptInvite(inviteCode, password)
+      const { password, firstName, lastName } = formData
+      await users.acceptInvite(inviteCode, password, firstName, lastName)
       notifications.success("Invitation accepted successfully")
-      $goto("../auth/login")
+      await login()
     } catch (error) {
       notifications.error(error.message)
+      onboarding = false
+    }
+  }
+
+  async function getInvite() {
+    try {
+      const invite = await users.fetchInvite(inviteCode)
+      if (invite?.email) {
+        formData.email = invite?.email
+      }
+    } catch (error) {
+      notifications.error(error.message)
+    }
+  }
+
+  async function login() {
+    try {
+      await auth.login({
+        username: formData.email.trim(),
+        password: formData.password.trim(),
+      })
+      notifications.success("Logged in successfully")
+      $goto("../portal")
+    } catch (err) {
+      notifications.error(err.message ? err.message : "Invalid credentials") //not likely, considering.
     }
   }
 
   onMount(async () => {
     try {
       await organisation.init()
+      await getInvite()
     } catch (error) {
-      notifications.error("Error getting org config")
+      notifications.error("Error getting invite config")
     }
   })
 </script>
 
-<section>
+<TestimonialPage>
-  <div class="container">
+  <Layout gap="S" noPadding>
-    <Layout>
+    <img alt="logo" src={$organisation.logoUrl || Logo} />
-      <img alt="logo" src={$organisation.logoUrl || Logo} />
+    <Layout gap="XS" noPadding>
-      <Layout gap="XS" justifyItems="center" noPadding>
+      <Heading size="M">Join {company}</Heading>
-        <Heading size="M">Invitation to {company}</Heading>
+      <Body size="M">Create your account to access your budibase apps!</Body>
-        <Body textAlign="center" size="M">
-          Please enter a password to get started.
-        </Body>
-      </Layout>
-      <PasswordRepeatInput bind:error bind:password />
-      <Button disabled={error} cta on:click={acceptInvite}>
-        Accept invite
-      </Button>
     </Layout>
-  </div>
+
-</section>
+    <Layout gap="S" noPadding>
+      <FancyForm bind:this={form}>
+        <FancyInput
+          label="Email"
+          value={formData.email}
+          disabled={true}
+          error={errors.email}
+        />
+        <FancyInput
+          label="First name"
+          value={formData.firstName}
+          on:change={e => {
+            formData = {
+              ...formData,
+              firstName: e.detail,
+            }
+          }}
+          validate={() => {
+            let fieldError = {
+              firstName: !formData.firstName
+                ? "Please enter your first name"
+                : undefined,
+            }
+
+            errors = handleError({ ...errors, ...fieldError })
+          }}
+          error={errors.firstName}
+          disabled={onboarding}
+        />
+        <FancyInput
+          label="Last name (optional)"
+          value={formData.lastName}
+          on:change={e => {
+            formData = {
+              ...formData,
+              lastName: e.detail,
+            }
+          }}
+          disabled={onboarding}
+        />
+        <FancyInput
+          label="Password"
+          value={formData.password}
+          type="password"
+          on:change={e => {
+            formData = {
+              ...formData,
+              password: e.detail,
+            }
+          }}
+          validate={() => {
+            let fieldError = {}
+
+            fieldError["password"] = !formData.password
+              ? "Please enter a password"
+              : undefined
+
+            fieldError["confirmationPassword"] =
+              !passwordsMatch(
+                formData.password,
+                formData.confirmationPassword
+              ) && formData.confirmationPassword
+                ? "Passwords must match"
+                : undefined
+
+            errors = handleError({ ...errors, ...fieldError })
+          }}
+          error={errors.password}
+          disabled={onboarding}
+        />
+        <FancyInput
+          label="Repeat password"
+          value={formData.confirmationPassword}
+          type="password"
+          on:change={e => {
+            formData = {
+              ...formData,
+              confirmationPassword: e.detail,
+            }
+          }}
+          validate={() => {
+            let fieldError = {
+              confirmationPassword:
+                !passwordsMatch(
+                  formData.password,
+                  formData.confirmationPassword
+                ) && formData.password
+                  ? "Passwords must match"
+                  : undefined,
+            }
+
+            errors = handleError({ ...errors, ...fieldError })
+          }}
+          error={errors.confirmationPassword}
+          disabled={onboarding}
+        />
+      </FancyForm>
+    </Layout>
+    <div>
+      <Button
+        disabled={Object.keys(errors).length > 0 || onboarding}
+        cta
+        on:click={acceptInvite}
+      >
+        Create account
+      </Button>
+    </div>
+  </Layout>
+</TestimonialPage>
 
 <style>
-  section {
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    height: 100%;
-  }
-  .container {
-    margin: 0 auto;
-    width: 300px;
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: stretch;
-  }
   img {
     width: 40px;
-    margin: 0 auto;
   }
 </style>

packages/builder/src/pages/builder/portal/settings/environment/_components/EditVariableColumn.svelte

@@ -0,0 +1,35 @@
+<script>
+  import { ActionButton, Modal } from "@budibase/bbui"
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { environment } from "stores/portal"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
+
+  export let row
+
+  let editVariableModal
+  let deleteDialog
+
+  const save = async data => {
+    await environment.updateVariable(data)
+    editVariableModal.hide()
+  }
+</script>
+
+<ActionButton size="S" on:click={editVariableModal.show}>Edit</ActionButton>
+
+<Modal bind:this={editVariableModal}>
+  <CreateEditVariableModal {row} {save} />
+</Modal>
+
+<ConfirmDialog
+  bind:this={deleteDialog}
+  onOk={async () => {
+    await environment.deleteVariable(row.name)
+  }}
+  okText="Delete Environment Variable"
+  title="Confirm Deletion"
+>
+  Are you sure you wish to delete the environment variable
+  <i>{row.name}?</i>
+  This action cannot be undone.
+</ConfirmDialog>

packages/builder/src/pages/builder/portal/settings/environment/index.svelte

@@ -0,0 +1,145 @@
+<script>
+  import {
+    Layout,
+    Heading,
+    Body,
+    Button,
+    Divider,
+    Modal,
+    Table,
+    Tags,
+    Tag,
+    InlineAlert,
+    notifications,
+  } from "@budibase/bbui"
+  import { environment, licensing, auth, admin } from "stores/portal"
+  import { onMount } from "svelte"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
+  import EditVariableColumn from "./_components/EditVariableColumn.svelte"
+
+  let modal
+
+  const customRenderers = [{ column: "edit", component: EditVariableColumn }]
+
+  $: noEncryptionKey = $environment.status?.encryptionKeyAvailable === false
+  $: schema = buildSchema(noEncryptionKey)
+
+  onMount(async () => {
+    await environment.checkStatus()
+    await environment.loadVariables()
+  })
+
+  const buildSchema = noEncryptionKey => {
+    const schema = {
+      name: {
+        width: "2fr",
+      },
+    }
+    if (!noEncryptionKey) {
+      schema.edit = {
+        width: "auto",
+        borderLeft: true,
+        displayName: "",
+      }
+    }
+    return schema
+  }
+
+  const save = async data => {
+    try {
+      await environment.createVariable(data)
+      modal.hide()
+    } catch (err) {
+      notifications.error(`Error saving variable: ${err.message}`)
+    }
+  }
+</script>
+
+<Layout noPadding>
+  <Layout gap="XS" noPadding>
+    <div class="title">
+      <Heading size="M">Environment Variables</Heading>
+      {#if !$licensing.environmentVariablesEnabled}
+        <Tags>
+          <Tag icon="LockClosed">Business plan</Tag>
+        </Tags>
+      {/if}
+    </div>
+    <Body
+      >Add and manage environment variables for development and production</Body
+    >
+  </Layout>
+  <Divider size="S" />
+
+  {#if $licensing.environmentVariablesEnabled}
+    {#if noEncryptionKey}
+      <InlineAlert
+        message="Your Budibase installation does not have a key for encryption, please update your app service's environment variables to contain an 'ENCRYPTION_KEY' value."
+        header="No encryption key found"
+        type="error"
+      />
+    {/if}
+    <div>
+      <Button on:click={modal.show} cta disabled={noEncryptionKey}
+        >Add Variable</Button
+      >
+    </div>
+
+    <Layout noPadding>
+      <Table
+        {schema}
+        data={$environment.variables}
+        allowEditColumns={false}
+        allowEditRows={false}
+        allowSelectRows={false}
+        {customRenderers}
+      />
+    </Layout>
+  {:else}
+    <div class="buttons">
+      <Button
+        primary
+        disabled={!$auth.accountPortalAccess && $admin.cloud}
+        on:click={async () => {
+          await environment.upgradePanelOpened()
+          $licensing.goToUpgradePage()
+        }}
+      >
+        Upgrade
+      </Button>
+      <!--Show the view plans button-->
+      <Button
+        secondary
+        on:click={() => {
+          window.open("https://budibase.com/pricing/", "_blank")
+        }}
+      >
+        View Plans
+      </Button>
+    </div>
+  {/if}
+</Layout>
+
+<Modal bind:this={modal}>
+  <CreateEditVariableModal {save} />
+</Modal>
+
+<style>
+  .buttons {
+    display: flex;
+    gap: var(--spacing-l);
+  }
+  .title {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: flex-start;
+    gap: var(--spacing-m);
+  }
+
+  .buttons {
+    display: flex;
+    flex-direction: row;
+    gap: var(--spacing-m);
+  }
+</style>

packages/builder/src/stores/backend/datasources.js

@@ -28,7 +28,7 @@ export function createDatasourcesStore() {
     }))
   }
 
-  const updateDatasource = async response => {
+  const updateDatasource = response => {
     const { datasource, error } = response
     store.update(state => {
       const currentIdx = state.list.findIndex(ds => ds._id === datasource._id)
@@ -52,7 +52,7 @@ export function createDatasourcesStore() {
       datasourceId: datasource?._id,
       tablesFilter,
     })
-    return await updateDatasource(response)
+    return updateDatasource(response)
   }
 
   const save = async (body, fetchSchema = false) => {

packages/builder/src/stores/portal/environment.js

@@ -0,0 +1,68 @@
+import { writable } from "svelte/store"
+import { API } from "api"
+import { Constants } from "@budibase/frontend-core"
+
+export function createEnvironmentStore() {
+  const { subscribe, update } = writable({
+    variables: [],
+    status: {},
+  })
+
+  async function checkStatus() {
+    const status = await API.checkEnvironmentVariableStatus()
+    update(store => {
+      store.status = status
+      return store
+    })
+  }
+
+  async function loadVariables() {
+    const envVars = await API.fetchEnvironmentVariables()
+    const mappedVars = envVars.variables.map(name => ({ name }))
+    update(store => {
+      store.variables = mappedVars
+      return store
+    })
+  }
+
+  async function createVariable(data) {
+    await API.createEnvironmentVariable(data)
+    let mappedVar = { name: data.name }
+    update(store => {
+      store.variables = [mappedVar, ...store.variables]
+      return store
+    })
+  }
+
+  async function deleteVariable(varName) {
+    await API.deleteEnvironmentVariable(varName)
+    update(store => {
+      store.variables = store.variables.filter(
+        envVar => envVar.name !== varName
+      )
+      return store
+    })
+  }
+
+  async function updateVariable(data) {
+    await API.updateEnvironmentVariable(data)
+  }
+
+  async function upgradePanelOpened() {
+    await API.publishEvent(
+      Constants.EventPublishType.ENV_VAR_UPGRADE_PANEL_OPENED
+    )
+  }
+
+  return {
+    subscribe,
+    checkStatus,
+    loadVariables,
+    createVariable,
+    deleteVariable,
+    updateVariable,
+    upgradePanelOpened,
+  }
+}
+
+export const environment = createEnvironmentStore()

packages/builder/src/stores/portal/index.js

@@ -11,4 +11,5 @@ export { groups } from "./groups"
 export { plugins } from "./plugins"
 export { backups } from "./backups"
 export { overview } from "./overview"
+export { environment } from "./environment"
 export { menu } from "./menu"

packages/builder/src/stores/portal/licensing.js

@@ -60,6 +60,9 @@ export const createLicensingStore = () => {
       const backupsEnabled = license.features.includes(
         Constants.Features.BACKUPS
       )
+      const environmentVariablesEnabled = license.features.includes(
+        Constants.Features.ENVIRONMENT_VARIABLES
+      )
 
       store.update(state => {
         return {
@@ -68,6 +71,7 @@ export const createLicensingStore = () => {
           isFreePlan,
           groupsEnabled,
           backupsEnabled,
+          environmentVariablesEnabled,
         }
       })
     },

packages/builder/src/stores/portal/menu.js

@@ -50,6 +50,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         title: "Organisation",
         href: "/builder/portal/settings/organisation",
       },
+      {
+        title: "Environment",
+        href: "/builder/portal/settings/environment",
+      },
     ]
     if (!$admin.cloud) {
       settingsSubPages.push({

packages/builder/src/stores/portal/users.js

@@ -29,13 +29,19 @@ export function createUsersStore() {
   async function invite(payload) {
     return API.inviteUsers(payload)
   }
-  async function acceptInvite(inviteCode, password) {
+  async function acceptInvite(inviteCode, password, firstName, lastName) {
     return API.acceptInvite({
       inviteCode,
       password,
+      firstName,
+      lastName,
     })
   }
 
+  async function fetchInvite(inviteCode) {
+    return API.getUserInvite(inviteCode)
+  }
+
   async function create(data) {
     let mappedUsers = data.users.map(user => {
       const body = {
@@ -101,6 +107,7 @@ export function createUsersStore() {
     fetch,
     invite,
     acceptInvite,
+    fetchInvite,
     create,
     save,
     bulkDelete,

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.2.12-alpha.44",
+  "version": "2.2.12-alpha.50",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.2.12-alpha.44",
+    "@budibase/backend-core": "2.2.12-alpha.50",
-    "@budibase/string-templates": "2.2.12-alpha.44",
+    "@budibase/string-templates": "2.2.12-alpha.50",
-    "@budibase/types": "2.2.12-alpha.44",
+    "@budibase/types": "2.2.12-alpha.50",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.2.12-alpha.44",
+  "version": "2.2.12-alpha.50",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.44",
+    "@budibase/bbui": "2.2.12-alpha.50",
-    "@budibase/frontend-core": "2.2.12-alpha.44",
+    "@budibase/frontend-core": "2.2.12-alpha.50",
-    "@budibase/string-templates": "2.2.12-alpha.44",
+    "@budibase/string-templates": "2.2.12-alpha.50",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",
@@ -48,6 +48,7 @@
   "devDependencies": {
     "@rollup/plugin-alias": "^3.1.5",
     "@rollup/plugin-commonjs": "^18.0.0",
+    "@rollup/plugin-image": "^3.0.2",
     "@rollup/plugin-node-resolve": "^11.2.1",
     "postcss": "^8.2.10",
     "rollup": "^2.44.0",

packages/client/rollup.config.js

@@ -5,6 +5,7 @@ import svelte from "rollup-plugin-svelte"
 import { terser } from "rollup-plugin-terser"
 import postcss from "rollup-plugin-postcss"
 import svg from "rollup-plugin-svg"
+import image from "@rollup/plugin-image"
 import json from "rollup-plugin-json"
 import nodePolyfills from "rollup-plugin-polyfill-node"
 import path from "path"
@@ -87,6 +88,7 @@ export default {
       dedupe: ["svelte", "svelte/internal"],
     }),
     svg(),
+    image(),
     json(),
     production && terser(),
     !production && visualizer(),

packages/client/yarn.lock

@@ -83,6 +83,14 @@
     magic-string "^0.25.7"
     resolve "^1.17.0"
 
+"@rollup/plugin-image@^3.0.2":
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/@rollup/plugin-image/-/plugin-image-3.0.2.tgz#8a66389510517495c5d10d392140cdefa43b27c2"
+  integrity sha512-eGVrD6lummWH5ENo9LWX3JY62uBb9okUNQ2htXkugrG6WjACrMUVhWvss+0wW3fwJWmFYpoEny3yL4spEdh15g==
+  dependencies:
+    "@rollup/pluginutils" "^5.0.1"
+    mini-svg-data-uri "^1.4.4"
+
 "@rollup/plugin-inject@^4.0.0":
   version "4.0.4"
   resolved "https://registry.yarnpkg.com/@rollup/plugin-inject/-/plugin-inject-4.0.4.tgz#fbeee66e9a700782c4f65c8b0edbafe58678fbc2"
@@ -113,6 +121,15 @@
     estree-walker "^1.0.1"
     picomatch "^2.2.2"
 
+"@rollup/pluginutils@^5.0.1":
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-5.0.2.tgz#012b8f53c71e4f6f9cb317e311df1404f56e7a33"
+  integrity sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==
+  dependencies:
+    "@types/estree" "^1.0.0"
+    estree-walker "^2.0.2"
+    picomatch "^2.3.1"
+
 "@socket.io/component-emitter@~3.1.0":
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553"
@@ -182,6 +199,11 @@
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f"
   integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==
 
+"@types/estree@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.0.tgz#5fb2e536c1ae9bf35366eed879e827fa59ca41c2"
+  integrity sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ==
+
 "@types/node@*":
   version "16.11.7"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.7.tgz#36820945061326978c42a01e56b61cd223dfdc42"
@@ -599,7 +621,7 @@ estree-walker@^1.0.1:
   resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-1.0.1.tgz#31bc5d612c96b704106b477e6dd5d8aa138cb700"
   integrity sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg==
 
-estree-walker@^2.0.1:
+estree-walker@^2.0.1, estree-walker@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-2.0.2.tgz#52f010178c2a4c117a7757cfe942adb7d2da4cac"
   integrity sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==
@@ -845,6 +867,11 @@ merge-stream@^2.0.0:
   resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60"
   integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==
 
+mini-svg-data-uri@^1.4.4:
+  version "1.4.4"
+  resolved "https://registry.yarnpkg.com/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz#8ab0aabcdf8c29ad5693ca595af19dd2ead09939"
+  integrity sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==
+
 minimatch@^3.0.2, minimatch@^3.0.4:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
@@ -955,6 +982,11 @@ picomatch@^2.2.2:
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972"
   integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==
 
+picomatch@^2.3.1:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
+  integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
+
 pify@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/pify/-/pify-5.0.0.tgz#1f5eca3f5e87ebec28cc6d54a0e4aaf00acc127f"

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.2.12-alpha.44",
+  "version": "2.2.12-alpha.50",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.44",
+    "@budibase/bbui": "2.2.12-alpha.50",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/frontend-core/src/api/environmentVariables.js

@@ -0,0 +1,36 @@
+export const buildEnvironmentVariableEndpoints = API => ({
+  checkEnvironmentVariableStatus: async () => {
+    return await API.get({
+      url: `/api/env/variables/status`,
+    })
+  },
+
+  /**
+   * Fetches a list of environment variables
+   */
+  fetchEnvironmentVariables: async () => {
+    return await API.get({
+      url: `/api/env/variables`,
+      json: false,
+    })
+  },
+
+  createEnvironmentVariable: async data => {
+    return await API.post({
+      url: `/api/env/variables`,
+      body: data,
+    })
+  },
+  deleteEnvironmentVariable: async varName => {
+    return await API.delete({
+      url: `/api/env/variables/${varName}`,
+    })
+  },
+
+  updateEnvironmentVariable: async data => {
+    return await API.patch({
+      url: `/api/env/variables/${data.name}`,
+      body: data,
+    })
+  },
+})

packages/frontend-core/src/api/events.js

@@ -0,0 +1,13 @@
+export const buildEventEndpoints = API => ({
+  /**
+   * Publish a specific event to the backend.
+   */
+  publishEvent: async eventType => {
+    return await API.post({
+      url: `/api/global/event/publish`,
+      body: {
+        type: eventType,
+      },
+    })
+  },
+})

packages/frontend-core/src/api/index.js

@@ -26,7 +26,8 @@ import { buildLicensingEndpoints } from "./licensing"
 import { buildGroupsEndpoints } from "./groups"
 import { buildPluginEndpoints } from "./plugins"
 import { buildBackupsEndpoints } from "./backups"
-
+import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
+import { buildEventEndpoints } from "./events"
 const defaultAPIClientConfig = {
   /**
    * Certain definitions can't change at runtime for client apps, such as the
@@ -247,5 +248,7 @@ export const createAPIClient = config => {
     ...buildGroupsEndpoints(API),
     ...buildPluginEndpoints(API),
     ...buildBackupsEndpoints(API),
+    ...buildEnvironmentVariableEndpoints(API),
+    ...buildEventEndpoints(API),
   }
 }

packages/frontend-core/src/api/rows.js

@@ -67,12 +67,13 @@ export const buildRowEndpoints = API => ({
    * @param format the format to export (csv or json)
    * @param columns which columns to export (all if undefined)
    */
-  exportRows: async ({ tableId, rows, format, columns }) => {
+  exportRows: async ({ tableId, rows, format, columns, search }) => {
     return await API.post({
       url: `/api/${tableId}/rows/exportRows?format=${format}`,
       body: {
         rows,
         columns,
+        ...search,
       },
       parseResponse: async response => {
         return await response.text()

packages/frontend-core/src/api/user.js

@@ -146,6 +146,16 @@ export const buildUserEndpoints = API => ({
     })
   },
 
+  /**
+   * Retrieves the invitation associated with a provided code.
+   * @param code The unique code for the target invite
+   */
+  getUserInvite: async code => {
+    return await API.get({
+      url: `/api/global/users/invite/${code}`,
+    })
+  },
+
   /**
    * Invites multiple users to the current tenant.
    * @param users An array of users to invite
@@ -168,13 +178,17 @@ export const buildUserEndpoints = API => ({
    * Accepts an invite to join the platform and creates a user.
    * @param inviteCode the invite code sent in the email
    * @param password the password for the newly created user
+   * @param firstName the first name of the new user
+   * @param lastName the last name of the new user
    */
-  acceptInvite: async ({ inviteCode, password }) => {
+  acceptInvite: async ({ inviteCode, password, firstName, lastName }) => {
     return await API.post({
       url: "/api/global/users/invite/accept",
       body: {
         inviteCode,
         password,
+        firstName,
+        lastName,
       },
     })
   },

packages/frontend-core/src/components/SplitPage.svelte

@@ -1,10 +1,15 @@
+<script>
+  import BG from "../../assets/bg.png"
+</script>
+
 <div class="split-page">
   <div class="left">
     <div class="content">
       <slot />
     </div>
   </div>
-  <div class="right">
+  <div class="right spectrum spectrum--darkest">
+    <img src={BG} alt="background" />
     <slot name="right" />
   </div>
 </div>
@@ -25,15 +30,19 @@
     overflow-y: auto;
   }
   .right {
-    background: linear-gradient(
+    position: relative;
-      to bottom right,
+  }
-      var(--spectrum-global-color-gray-300) 0%,
+  .right img {
-      var(--background) 100%
+    position: absolute;
-    );
+    top: 0;
+    left: 0;
+    height: 100%;
+    width: 100%;
   }
   .content {
     width: 100%;
     max-width: 400px;
+    min-height: 480px;
   }
 
   @media (max-width: 740px) {

packages/frontend-core/src/components/TestimonialPage.svelte

@@ -1,6 +1,34 @@
 <script>
   import SplitPage from "./SplitPage.svelte"
   import { Layout } from "@budibase/bbui"
+  import Bulgaria from "../../assets/bulgaria.png"
+  import Covanta from "../../assets/covanta.png"
+  import Schnellecke from "../../assets/schnellecke.png"
+
+  const testimonials = [
+    {
+      text: "Budibase was the only solution that checked all the boxes for Covanta. Covanta expects to realize $3.2MM in savings due to the elimination of redundant data entry.",
+      name: "Charles Link",
+      role: "Senior Director, Data and Analytics",
+      image: Covanta,
+      imageSize: 105,
+    },
+    {
+      text: "Budibase was mission-critical for us and went a long way in preventing what could have become a humanitarian crisis here in Bulgaria.",
+      name: "Bozhidar Bozhanov",
+      role: "Government of Bulgaria",
+      image: Bulgaria,
+      imageSize: 49,
+    },
+    {
+      text: "Centralization of authentication, quick turnaround time for requests, integration with different database systems has given it the edge and it’s now used daily for internal development for those apps that you know you need but don’t feel value in losing days of development to reinvent the wheel.",
+      name: "Davide Lenzarini",
+      role: "IT manager",
+      image: Schnellecke,
+      imageSize: 141,
+    },
+  ]
+  const testimonial = testimonials[Math.floor(Math.random() * 3)]
 </script>
 
 <SplitPage>
@@ -8,19 +36,17 @@
   <div class="wrapper" slot="right">
     <div class="testimonial">
       <Layout noPadding gap="S">
+        <img
+          width={testimonial.imageSize}
+          alt="a-happy-budibase-user"
+          src={testimonial.image}
+        />
         <div class="text">
-          "Here is an example of how Budibase changed my life for the better and
+          "{testimonial.text}"
-          now all I do is eat, sleep, build apps, repeat."
         </div>
-        <div class="user">
+        <div class="author">
-          <img
+          <div class="name">{testimonial.name}</div>
-            alt="a-happy-budibase-user"
+          <div class="company">{testimonial.role}</div>
-            src="https://icon-library.com/images/male-user-icon/male-user-icon-24.jpg"
-          />
-          <div class="author">
-            <div class="name">No-code Enthusiast</div>
-            <div class="company">Bedroom TLD</div>
-          </div>
         </div>
       </Layout>
     </div>
@@ -35,23 +61,13 @@
     place-items: center;
   }
   .testimonial {
-    width: 280px;
+    width: 380px;
     padding: 40px;
   }
   .text {
     font-size: var(--font-size-l);
     font-style: italic;
   }
-  img {
-    width: 40px;
-  }
-  .user {
-    display: flex;
-    flex-direction: row;
-    justify-content: flex-start;
-    align-items: center;
-    gap: var(--spacing-s);
-  }
   .name {
     font-weight: bold;
     color: var(--spectrum-global-color-gray-900);

packages/frontend-core/src/constants.js

@@ -114,6 +114,7 @@ export const ApiVersion = "1"
 export const Features = {
   USER_GROUPS: "userGroups",
   BACKUPS: "appBackups",
+  ENVIRONMENT_VARIABLES: "environmentVariables",
 }
 
 // Role IDs
@@ -174,3 +175,7 @@ export const Themes = [
     base: "darkest",
   },
 ]
+
+export const EventPublishType = {
+  ENV_VAR_UPGRADE_PANEL_OPENED: "environment_variable_upgrade_panel_opened",
+}

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.2.12-alpha.44",
+  "version": "2.2.12-alpha.50",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/__mocks__/node-fetch.ts

@@ -1,5 +1,7 @@
+// @ts-ignore
 import fs from "fs"
 module FetchMock {
+  // @ts-ignore
   const fetch = jest.requireActual("node-fetch")
   let failCount = 0
 

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.2.12-alpha.44",
+  "version": "2.2.12-alpha.50",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.2.12-alpha.44",
+    "@budibase/backend-core": "2.2.12-alpha.50",
-    "@budibase/client": "2.2.12-alpha.44",
+    "@budibase/client": "2.2.12-alpha.50",
-    "@budibase/pro": "2.2.12-alpha.44",
+    "@budibase/pro": "2.2.12-alpha.50",
-    "@budibase/string-templates": "2.2.12-alpha.44",
+    "@budibase/string-templates": "2.2.12-alpha.50",
-    "@budibase/types": "2.2.12-alpha.44",
+    "@budibase/types": "2.2.12-alpha.50",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/scripts/dev/manage.js

@@ -29,6 +29,7 @@ async function init() {
       ACCOUNT_PORTAL_URL: "http://localhost:10001",
       ACCOUNT_PORTAL_API_KEY: "budibase",
       JWT_SECRET: "testsecret",
+      ENCRYPTION_KEY: "testsecret",
       REDIS_PASSWORD: "budibase",
       MINIO_ACCESS_KEY: "budibase",
       MINIO_SECRET_KEY: "budibase",

packages/server/src/api/controllers/application.ts

@@ -112,12 +112,11 @@ function checkAppName(
   }
 }
 
-async function createInstance(template: any, includeSampleData: boolean) {
+async function createInstance(
-  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
+  appId: string,
-  const baseAppId = generateAppID(tenantId)
+  template: any,
-  const appId = generateDevAppID(baseAppId)
+  includeSampleData: boolean
-  await context.updateAppId(appId)
+) {
-
   const db = context.getAppDB()
   await db.put({
     _id: "_design/database",
@@ -250,82 +249,90 @@ async function performAppCreate(ctx: BBContext) {
     instanceConfig.file = ctx.request.files.templateFile
   }
   const includeSampleData = isQsTrue(ctx.request.body.sampleData)
-  const instance = await createInstance(instanceConfig, includeSampleData)
+  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
-  const appId = instance._id
+  const appId = generateDevAppID(generateAppID(tenantId))
-  const db = context.getAppDB()
 
-  let newApplication: App = {
+  return await context.doInAppContext(appId, async () => {
-    _id: DocumentType.APP_METADATA,
+    const instance = await createInstance(
-    _rev: undefined,
+      appId,
-    appId,
+      instanceConfig,
-    type: "app",
+      includeSampleData
-    version: packageJson.version,
+    )
-    componentLibraries: ["@budibase/standard-components"],
+    const db = context.getAppDB()
-    name: name,
-    url: url,
-    template: templateKey,
-    instance,
-    tenantId: tenancy.getTenantId(),
-    updatedAt: new Date().toISOString(),
-    createdAt: new Date().toISOString(),
-    status: AppStatus.DEV,
-    navigation: {
-      navigation: "Top",
-      title: name,
-      navWidth: "Large",
-      navBackground: "var(--spectrum-global-color-gray-100)",
-      links: [
-        {
-          url: "/home",
-          text: "Home",
-        },
-      ],
-    },
-    theme: "spectrum--light",
-    customTheme: {
-      buttonBorderRadius: "16px",
-    },
-  }
 
-  // If we used a template or imported an app there will be an existing doc.
+    let newApplication: App = {
-  // Fetch and migrate some metadata from the existing app.
+      _id: DocumentType.APP_METADATA,
-  try {
+      _rev: undefined,
-    const existing: App = await db.get(DocumentType.APP_METADATA)
+      appId,
-    const keys: (keyof App)[] = [
+      type: "app",
-      "_rev",
+      version: packageJson.version,
-      "navigation",
+      componentLibraries: ["@budibase/standard-components"],
-      "theme",
+      name: name,
-      "customTheme",
+      url: url,
-      "icon",
+      template: templateKey,
-    ]
+      instance,
-    keys.forEach(key => {
+      tenantId: tenancy.getTenantId(),
-      if (existing[key]) {
+      updatedAt: new Date().toISOString(),
-        // @ts-ignore
+      createdAt: new Date().toISOString(),
-        newApplication[key] = existing[key]
+      status: AppStatus.DEV,
-      }
+      navigation: {
-    })
+        navigation: "Top",
-
+        title: name,
-    // Migrate navigation settings and screens if required
+        navWidth: "Large",
-    if (existing) {
+        navBackground: "var(--spectrum-global-color-gray-100)",
-      const navigation = await migrateAppNavigation()
+        links: [
-      if (navigation) {
+          {
-        newApplication.navigation = navigation
+            url: "/home",
-      }
+            text: "Home",
+          },
+        ],
+      },
+      theme: "spectrum--light",
+      customTheme: {
+        buttonBorderRadius: "16px",
+      },
     }
-  } catch (err) {
-    // Nothing to do
-  }
 
-  const response = await db.put(newApplication, { force: true })
+    // If we used a template or imported an app there will be an existing doc.
-  newApplication._rev = response.rev
+    // Fetch and migrate some metadata from the existing app.
+    try {
+      const existing: App = await db.get(DocumentType.APP_METADATA)
+      const keys: (keyof App)[] = [
+        "_rev",
+        "navigation",
+        "theme",
+        "customTheme",
+        "icon",
+      ]
+      keys.forEach(key => {
+        if (existing[key]) {
+          // @ts-ignore
+          newApplication[key] = existing[key]
+        }
+      })
 
-  /* istanbul ignore next */
+      // Migrate navigation settings and screens if required
-  if (!env.isTest()) {
+      if (existing) {
-    await createApp(appId)
+        const navigation = await migrateAppNavigation()
-  }
+        if (navigation) {
+          newApplication.navigation = navigation
+        }
+      }
+    } catch (err) {
+      // Nothing to do
+    }
 
-  await cache.app.invalidateAppMetadata(appId, newApplication)
+    const response = await db.put(newApplication, { force: true })
-  return newApplication
+    newApplication._rev = response.rev
+
+    /* istanbul ignore next */
+    if (!env.isTest()) {
+      await createApp(appId)
+    }
+
+    await cache.app.invalidateAppMetadata(appId, newApplication)
+    return newApplication
+  })
 }
 
 async function creationEvents(request: any, app: App) {

packages/server/src/api/controllers/cloud.ts

@@ -86,7 +86,7 @@ export async function importApps(ctx: Ctx) {
   if (Array.isArray(file)) {
     ctx.throw(400, "Single file is required")
   }
-  if (file.type !== "application/gzip") {
+  if (file.type !== "application/gzip" && file.type !== "application/x-gzip") {
     ctx.throw(400, "Import file must be a gzipped tarball.")
   }
 

packages/server/src/api/controllers/datasource.ts

@@ -12,9 +12,11 @@ import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
-import { BBContext, Datasource, Row } from "@budibase/types"
+import { UserCtx, Datasource, Row } from "@budibase/types"
+import sdk from "../../sdk"
+import { mergeConfigs } from "../../sdk/app/datasources/datasources"
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
   const internalTables = await db.allDocs(
@@ -43,25 +45,23 @@ export async function fetch(ctx: BBContext) {
     )
   ).rows.map(row => row.doc)
 
-  const allDatasources = [bbInternalDb, ...datasources]
+  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
+    bbInternalDb,
+    ...datasources,
+  ])
 
   for (let datasource of allDatasources) {
-    if (datasource.config && datasource.config.auth) {
-      // strip secrets from response so they don't show in the network request
-      delete datasource.config.auth
-    }
-
     if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
-      datasource.entities = internal[datasource._id]
+      datasource.entities = internal[datasource._id!]
     }
   }
 
   ctx.body = [bbInternalDb, ...datasources]
 }
 
-export async function buildSchemaFromDb(ctx: BBContext) {
+export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const datasource = await db.get(ctx.params.datasourceId)
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
 
   let { tables, error } = await buildSchemaHelper(datasource)
@@ -146,11 +146,11 @@ async function invalidateVariables(
   await invalidateDynamicVariables(toInvalidate)
 }
 
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
-  let datasource = await db.get(datasourceId)
+  let datasource = await sdk.datasources.get(datasourceId)
-  const auth = datasource.config.auth
+  const auth = datasource.config?.auth
   await invalidateVariables(datasource, ctx.request.body)
 
   const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
@@ -159,10 +159,13 @@ export async function update(ctx: BBContext) {
     ? { name: ctx.request.body?.name }
     : ctx.request.body
 
-  datasource = { ...datasource, ...dataSourceBody }
+  datasource = {
+    ...datasource,
+    ...sdk.datasources.mergeConfigs(dataSourceBody, datasource),
+  }
   if (auth && !ctx.request.body.auth) {
     // don't strip auth config from DB
-    datasource.config.auth = auth
+    datasource.config!.auth = auth
   }
 
   const response = await db.put(datasource)
@@ -179,10 +182,12 @@ export async function update(ctx: BBContext) {
 
   ctx.status = 200
   ctx.message = "Datasource saved successfully."
-  ctx.body = { datasource }
+  ctx.body = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
@@ -213,7 +218,9 @@ export async function save(ctx: BBContext) {
     }
   }
 
-  const response: any = { datasource }
+  const response: any = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
   if (schemaError) {
     response.error = schemaError
   }
@@ -251,11 +258,11 @@ async function destroyInternalTablesBySourceId(datasourceId: string) {
   }
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
 
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   // Delete all queries for the datasource
 
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
@@ -279,13 +286,14 @@ export async function destroy(ctx: BBContext) {
   ctx.status = 200
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const database = context.getAppDB()
-  ctx.body = await database.get(ctx.params.datasourceId)
+  const datasource = await database.get(ctx.params.datasourceId)
+  ctx.body = await sdk.datasources.removeSecretSingle(datasource)
 }
 
 // dynamic query functionality
-export async function query(ctx: BBContext) {
+export async function query(ctx: UserCtx) {
   const queryJson = ctx.request.body
   try {
     ctx.body = await getDatasourceAndQuery(queryJson)
@@ -313,7 +321,7 @@ function updateError(error: any, newError: any, tables: string[]) {
 
 async function buildSchemaHelper(datasource: Datasource) {
   const Connector = await getIntegration(datasource.source)
-
+  datasource = await sdk.datasources.enrich(datasource)
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
   await connector.buildSchema(datasource._id, datasource.entities)

packages/server/src/api/controllers/query/index.ts

@@ -7,6 +7,8 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
 import env from "../../../environment"
 import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
+import sdk from "../../../sdk"
+import { QueryEvent } from "../../../threads/definitions"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
@@ -81,7 +83,7 @@ export async function save(ctx: any) {
   const db = context.getAppDB()
   const query = ctx.request.body
 
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
 
   let eventFn
   if (!query._id) {
@@ -126,9 +128,9 @@ function getAuthConfig(ctx: any) {
 }
 
 export async function preview(ctx: any) {
-  const db = context.getAppDB()
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
-
+    ctx.request.body.datasourceId
-  const datasource = await db.get(ctx.request.body.datasourceId)
+  )
   const query = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
@@ -137,20 +139,22 @@ export async function preview(ctx: any) {
   const authConfigCtx: any = getAuthConfig(ctx)
 
   try {
-    const runFn = () =>
+    const inputs: QueryEvent = {
-      Runner.run({
+      appId: ctx.appId,
-        appId: ctx.appId,
+      datasource,
-        datasource,
+      queryVerb,
-        queryVerb,
+      fields,
-        fields,
+      parameters,
-        parameters,
+      transformer,
-        transformer,
+      queryId,
-        queryId,
+      // have to pass down to the thread runner - can't put into context now
-        ctx: {
+      environmentVariables: envVars,
-          user: ctx.user,
+      ctx: {
-          auth: { ...authConfigCtx },
+        user: ctx.user,
-        },
+        auth: { ...authConfigCtx },
-      })
+      },
+    }
+    const runFn = () => Runner.run(inputs)
 
     const { rows, keys, info, extra } = await quotas.addQuery(runFn, {
       datasourceId: datasource._id,
@@ -201,7 +205,9 @@ async function execute(
   const db = context.getAppDB()
 
   const query = await db.get(ctx.params.queryId)
-  const datasource = await db.get(query.datasourceId)
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
+    query.datasourceId
+  )
 
   let authConfigCtx: any = {}
   if (!opts.isAutomation) {
@@ -219,21 +225,23 @@ async function execute(
 
   // call the relevant CRUD method on the integration class
   try {
-    const runFn = () =>
+    const inputs: QueryEvent = {
-      Runner.run({
+      appId: ctx.appId,
-        appId: ctx.appId,
+      datasource,
-        datasource,
+      queryVerb: query.queryVerb,
-        queryVerb: query.queryVerb,
+      fields: query.fields,
-        fields: query.fields,
+      pagination: ctx.request.body.pagination,
-        pagination: ctx.request.body.pagination,
+      parameters: enrichedParameters,
-        parameters: enrichedParameters,
+      transformer: query.transformer,
-        transformer: query.transformer,
+      queryId: ctx.params.queryId,
-        queryId: ctx.params.queryId,
+      // have to pass down to the thread runner - can't put into context now
-        ctx: {
+      environmentVariables: envVars,
-          user: ctx.user,
+      ctx: {
-          auth: { ...authConfigCtx },
+        user: ctx.user,
-        },
+        auth: { ...authConfigCtx },
-      })
+      },
+    }
+    const runFn = () => Runner.run(inputs)
 
     const { rows, pagination, extra } = await quotas.addQuery(runFn, {
       datasourceId: datasource._id,
@@ -266,18 +274,18 @@ export async function executeV2(
 const removeDynamicVariables = async (queryId: any) => {
   const db = context.getAppDB()
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
-  const dynamicVariables = datasource.config.dynamicVariables
+  const dynamicVariables = datasource.config?.dynamicVariables as any[]
 
   if (dynamicVariables) {
     // delete dynamic variables from the datasource
-    datasource.config.dynamicVariables = dynamicVariables.filter(
+    datasource.config!.dynamicVariables = dynamicVariables!.filter(
       (dv: any) => dv.queryId !== queryId
     )
     await db.put(datasource)
 
     // invalidate the deleted variables
-    const variablesToDelete = dynamicVariables.filter(
+    const variablesToDelete = dynamicVariables!.filter(
       (dv: any) => dv.queryId === queryId
     )
     await invalidateDynamicVariables(variablesToDelete)
@@ -289,7 +297,7 @@ export async function destroy(ctx: any) {
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
   ctx.message = `Query deleted.`
   ctx.status = 200

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -25,6 +25,7 @@ import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
 import { removeKeyNumbering } from "./utils"
+import sdk from "../../../sdk"
 
 export interface ManyRelationship {
   tableId?: string
@@ -664,8 +665,7 @@ export class ExternalRequest {
       throw "Unable to run without a table name"
     }
     if (!this.datasource) {
-      const db = context.getAppDB()
+      this.datasource = await sdk.datasources.get(datasourceId!)
-      this.datasource = await db.get(datasourceId)
       if (!this.datasource || !this.datasource.entities) {
         throw "No tables found, fetch tables before query."
       }

packages/server/src/api/controllers/row/external.ts

@@ -19,6 +19,9 @@ import {
   Table,
   Datasource,
 } from "@budibase/types"
+import sdk from "../../../sdk"
+
+const { cleanExportRows } = require("./utils")
 
 export async function handleRequest(
   operation: Operation,
@@ -99,7 +102,7 @@ export async function destroy(ctx: BBContext) {
 export async function bulkDestroy(ctx: BBContext) {
   const { rows } = ctx.request.body
   const tableId = ctx.params.tableId
-  let promises = []
+  let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
       handleRequest(Operation.DELETE, tableId, {
@@ -179,27 +182,30 @@ export async function validate(ctx: BBContext) {
 
 export async function exportRows(ctx: BBContext) {
   const { datasourceId } = breakExternalTableId(ctx.params.tableId)
-  const db = context.getAppDB()
   const format = ctx.query.format
   const { columns } = ctx.request.body
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId!)
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
-  ctx.request.body = {
+
-    query: {
+  if (ctx.request.body.rows) {
-      oneOf: {
+    ctx.request.body = {
-        _id: ctx.request.body.rows.map(
+      query: {
-          (row: string) => JSON.parse(decodeURI(row))[0]
+        oneOf: {
-        ),
+          _id: ctx.request.body.rows.map(
+            (row: string) => JSON.parse(decodeURI(row))[0]
+          ),
+        },
       },
-    },
+    }
   }
 
   let result = await search(ctx)
   let rows: Row[] = []
 
   // Filter data to only specified columns if required
+
   if (columns && columns.length) {
     for (let i = 0; i < result.rows.length; i++) {
       rows[i] = {}
@@ -211,22 +217,26 @@ export async function exportRows(ctx: BBContext) {
     rows = result.rows
   }
 
-  let headers = Object.keys(rows[0])
+  // @ts-ignore
+  let schema = datasource.entities[tableName].schema
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+
+  let headers = Object.keys(schema)
+
   // @ts-ignore
   const exporter = exporters[format]
   const filename = `export.${format}`
 
   // send down the file
   ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, rows))
+  return apiFileReturn(exporter(headers, exportRows))
 }
 
 export async function fetchEnrichedRow(ctx: BBContext) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const db = context.getAppDB()
+  const datasource: Datasource = await sdk.datasources.get(datasourceId!)
-  const datasource: Datasource = await db.get(datasourceId)
   if (!tableName) {
     ctx.throw(400, "Unable to find table.")
   }

packages/server/src/api/controllers/row/internal.ts

@@ -27,7 +27,7 @@ import {
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import { csv, json, jsonWithSchema, Format, isFormat } from "../view/exporters"
+import { csv, json, jsonWithSchema, Format } from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Ctx,
@@ -38,6 +38,8 @@ import {
   Table,
 } from "@budibase/types"
 
+const { cleanExportRows } = require("./utils")
+
 const CALCULATION_TYPES = {
   SUM: "sum",
   COUNT: "count",
@@ -357,6 +359,14 @@ export async function search(ctx: Ctx) {
   params.version = ctx.version
   params.tableId = tableId
 
+  let table
+  if (params.sort && !params.sortType) {
+    table = await db.get(tableId)
+    const schema = table.schema
+    const sortField = schema[params.sort]
+    params.sortType = sortField.type == "number" ? "number" : "string"
+  }
+
   let response
   if (paginate) {
     response = await paginatedSearch(query, params)
@@ -370,7 +380,7 @@ export async function search(ctx: Ctx) {
     if (tableId === InternalTables.USER_METADATA) {
       response.rows = await getGlobalUsersFromMetadata(response.rows)
     }
-    const table = await db.get(tableId)
+    table = table || (await db.get(tableId))
     response.rows = await outputProcessing(table, response.rows)
   }
 
@@ -389,16 +399,25 @@ export async function exportRows(ctx: Ctx) {
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
   let format = ctx.query.format
-  const { columns } = ctx.request.body
+  const { columns, query } = ctx.request.body
-  let response = (
+
-    await db.allDocs({
+  let result
-      include_docs: true,
+  if (rowIds) {
-      keys: rowIds,
+    let response = (
-    })
+      await db.allDocs({
-  ).rows.map(row => row.doc)
+        include_docs: true,
+        keys: rowIds,
+      })
+    ).rows.map(row => row.doc)
+
+    result = await outputProcessing(table, response)
+  } else if (query) {
+    let searchResponse = await exports.search(ctx)
+    result = searchResponse.rows
+  }
 
-  let result = (await outputProcessing(table, response)) as Row[]
   let rows: Row[] = []
+  let schema = table.schema
 
   // Filter data to only specified columns if required
   if (columns && columns.length) {
@@ -412,12 +431,16 @@ export async function exportRows(ctx: Ctx) {
     rows = result
   }
 
+  let exportRows = cleanExportRows(rows, schema, format, columns)
   if (format === Format.CSV) {
     ctx.attachment("export.csv")
-    return apiFileReturn(csv(Object.keys(rows[0]), rows))
+    return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
   } else if (format === Format.JSON) {
     ctx.attachment("export.json")
-    return apiFileReturn(json(rows))
+    return apiFileReturn(json(exportRows))
+  } else if (format === Format.JSON_WITH_SCHEMA) {
+    ctx.attachment("export.json")
+    return apiFileReturn(jsonWithSchema(schema, exportRows))
   } else {
     throw "Format not recognised"
   }

packages/server/src/api/controllers/row/utils.ts

@@ -7,7 +7,9 @@ import { BBContext, Row, Table } from "@budibase/types"
 export { removeKeyNumbering } from "../../../integrations/base/utils"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
+import { Format } from "../view/exporters"
 import { Ctx } from "@budibase/types"
+import sdk from "../../../sdk"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -21,8 +23,7 @@ validateJs.extend(validateJs.validators.datetime, {
 
 export async function getDatasourceAndQuery(json: any) {
   const datasourceId = json.endpoint.datasourceId
-  const db = context.getAppDB()
+  const datasource = await sdk.datasources.get(datasourceId)
-  const datasource = await db.get(datasourceId)
   return makeExternalQuery(datasource, json)
 }
 
@@ -117,3 +118,40 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
+
+export function cleanExportRows(
+  rows: any[],
+  schema: any,
+  format: string,
+  columns: string[]
+) {
+  let cleanRows = [...rows]
+
+  const relationships = Object.entries(schema)
+    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
+    .map(entry => entry[0])
+
+  relationships.forEach(column => {
+    cleanRows.forEach(row => {
+      delete row[column]
+    })
+    delete schema[column]
+  })
+
+  // Intended to avoid 'undefined' in export
+  if (format === Format.CSV) {
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      if (columns?.length && columns.indexOf(key) > 0) {
+        continue
+      }
+      for (let row of cleanRows) {
+        if (row[key] == null) {
+          row[key] = ""
+        }
+      }
+    }
+  }
+
+  return cleanRows
+}

packages/server/src/api/controllers/static/index.ts

@@ -1,20 +1,21 @@
 require("svelte/register")
 
-const send = require("koa-send")
+import { resolve, join } from "../../../utilities/centralPath"
-const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 import { ObjectStoreBuckets } from "../../../constants"
-const { processString } = require("@budibase/string-templates")
+import { processString } from "@budibase/string-templates"
-const {
+import {
   loadHandlebarsFile,
   NODE_MODULES_PATH,
   TOP_LEVEL_PATH,
-} = require("../../../utilities/fileSystem")
+} from "../../../utilities/fileSystem"
 import env from "../../../environment"
-const { DocumentType } = require("../../../db/utils")
+import { DocumentType } from "../../../db/utils"
-const { context, objectStore, utils } = require("@budibase/backend-core")
+import { context, objectStore, utils } from "@budibase/backend-core"
-const AWS = require("aws-sdk")
+import AWS from "aws-sdk"
-const fs = require("fs")
+import fs from "fs"
+import sdk from "../../../sdk"
+const send = require("koa-send")
 
 async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   const response = await objectStore.upload({
@@ -110,7 +111,7 @@ export const serveApp = async function (ctx: any) {
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
       usedPlugins: plugins,
     })
 
@@ -135,7 +136,7 @@ export const serveBuilderPreview = async function (ctx: any) {
     let appId = context.getAppId()
     const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
     ctx.body = await processString(previewHbs, {
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
     })
   } else {
     // just return the app info for jest to assert on
@@ -150,13 +151,11 @@ export const serveClientLibrary = async function (ctx: any) {
 }
 
 export const getSignedUploadURL = async function (ctx: any) {
-  const database = context.getAppDB()
-
   // Ensure datasource is valid
   let datasource
   try {
     const { datasourceId } = ctx.params
-    datasource = await database.get(datasourceId)
+    datasource = await sdk.datasources.get(datasourceId, { enriched: true })
     if (!datasource) {
       ctx.throw(400, "The specified datasource could not be found")
     }
@@ -172,8 +171,8 @@ export const getSignedUploadURL = async function (ctx: any) {
   // Determine type of datasource and generate signed URL
   let signedUrl
   let publicUrl
-  const awsRegion = datasource?.config?.region || "eu-west-1"
+  const awsRegion = (datasource?.config?.region || "eu-west-1") as string
-  if (datasource.source === "S3") {
+  if (datasource?.source === "S3") {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
@@ -182,8 +181,8 @@ export const getSignedUploadURL = async function (ctx: any) {
     try {
       const s3 = new AWS.S3({
         region: awsRegion,
-        accessKeyId: datasource?.config?.accessKeyId,
+        accessKeyId: datasource?.config?.accessKeyId as string,
-        secretAccessKey: datasource?.config?.secretAccessKey,
+        secretAccessKey: datasource?.config?.secretAccessKey as string,
         apiVersion: "2006-03-01",
         signatureVersion: "v4",
       })

packages/server/src/api/controllers/table/external.ts

@@ -219,7 +219,7 @@ export async function save(ctx: BBContext) {
   }
 
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   if (!datasource.entities) {
     datasource.entities = {}
   }
@@ -322,15 +322,17 @@ export async function destroy(ctx: BBContext) {
   const datasourceId = getDatasourceId(tableToDelete)
 
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId!)
   const tables = datasource.entities
 
   const operation = Operation.DELETE_TABLE
-  await makeTableRequest(datasource, operation, tableToDelete, tables)
+  if (tables) {
+    await makeTableRequest(datasource, operation, tableToDelete, tables)
+    cleanupRelationships(tableToDelete, tables)
+    delete tables[tableToDelete.name]
+    datasource.entities = tables
+  }
 
-  cleanupRelationships(tableToDelete, tables)
-
-  delete datasource.entities[tableToDelete.name]
   await db.put(datasource)
 
   return tableToDelete

packages/server/src/api/controllers/view/index.ts

@@ -3,7 +3,6 @@ import { apiFileReturn } from "../../../utilities/fileSystem"
 import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
 import { deleteView, getView, getViews, saveView } from "./utils"
 import { fetchView } from "../row"
-import { FieldTypes } from "../../../constants"
 import { context, events } from "@budibase/backend-core"
 import { DocumentType } from "../../../db/utils"
 import sdk from "../../../sdk"
@@ -15,6 +14,7 @@ import {
   TableSchema,
   View,
 } from "@budibase/types"
+import { cleanExportRows } from "../row/utils"
 
 const { cloneDeep, isEqual } = require("lodash")
 
@@ -162,39 +162,17 @@ export async function exportView(ctx: BBContext) {
     schema = table.schema
   }
 
-  // remove any relationships
+  let exportRows = cleanExportRows(rows, schema, format, [])
-  const relationships = Object.entries(schema)
-    .filter(entry => entry[1].type === FieldTypes.LINK)
-    .map(entry => entry[0])
-  // iterate relationship columns and remove from and row and schema
-  relationships.forEach(column => {
-    rows.forEach(row => {
-      delete row[column]
-    })
-    delete schema[column]
-  })
-
-  // make sure no "undefined" entries appear in the CSV
-  if (format === Format.CSV) {
-    const schemaKeys = Object.keys(schema)
-    for (let key of schemaKeys) {
-      for (let row of rows) {
-        if (row[key] == null) {
-          row[key] = ""
-        }
-      }
-    }
-  }
 
   if (format === Format.CSV) {
     ctx.attachment(`${viewName}.csv`)
-    ctx.body = apiFileReturn(csv(Object.keys(schema), rows))
+    ctx.body = apiFileReturn(csv(Object.keys(schema), exportRows))
   } else if (format === Format.JSON) {
     ctx.attachment(`${viewName}.json`)
-    ctx.body = apiFileReturn(json(rows))
+    ctx.body = apiFileReturn(json(exportRows))
   } else if (format === Format.JSON_WITH_SCHEMA) {
     ctx.attachment(`${viewName}.json`)
-    ctx.body = apiFileReturn(jsonWithSchema(schema, rows))
+    ctx.body = apiFileReturn(jsonWithSchema(schema, exportRows))
   } else {
     throw "Format not recognised"
   }

packages/server/src/api/controllers/webhook.ts

@@ -39,60 +39,62 @@ export async function destroy(ctx: BBContext) {
 }
 
 export async function buildSchema(ctx: BBContext) {
-  await context.updateAppId(ctx.params.instance)
+  await context.doInAppContext(ctx.params.instance, async () => {
-  const db = context.getAppDB()
+    const db = context.getAppDB()
-  const webhook = (await db.get(ctx.params.id)) as Webhook
+    const webhook = (await db.get(ctx.params.id)) as Webhook
-  webhook.bodySchema = toJsonSchema(ctx.request.body)
+    webhook.bodySchema = toJsonSchema(ctx.request.body)
-  // update the automation outputs
+    // update the automation outputs
-  if (webhook.action.type === WebhookActionType.AUTOMATION) {
+    if (webhook.action.type === WebhookActionType.AUTOMATION) {
-    let automation = (await db.get(webhook.action.target)) as Automation
+      let automation = (await db.get(webhook.action.target)) as Automation
-    const autoOutputs = automation.definition.trigger.schema.outputs
+      const autoOutputs = automation.definition.trigger.schema.outputs
-    let properties = webhook.bodySchema.properties
+      let properties = webhook.bodySchema.properties
-    // reset webhook outputs
+      // reset webhook outputs
-    autoOutputs.properties = {
+      autoOutputs.properties = {
-      body: autoOutputs.properties.body,
+        body: autoOutputs.properties.body,
-    }
-    for (let prop of Object.keys(properties)) {
-      autoOutputs.properties[prop] = {
-        type: properties[prop].type,
-        description: AUTOMATION_DESCRIPTION,
       }
+      for (let prop of Object.keys(properties)) {
+        autoOutputs.properties[prop] = {
+          type: properties[prop].type,
+          description: AUTOMATION_DESCRIPTION,
+        }
+      }
+      await db.put(automation)
     }
-    await db.put(automation)
+    ctx.body = await db.put(webhook)
-  }
+  })
-  ctx.body = await db.put(webhook)
 }
 
 export async function trigger(ctx: BBContext) {
   const prodAppId = dbCore.getProdAppID(ctx.params.instance)
-  await context.updateAppId(prodAppId)
+  await context.doInAppContext(prodAppId, async () => {
-  try {
+    try {
-    const db = context.getAppDB()
+      const db = context.getAppDB()
-    const webhook = (await db.get(ctx.params.id)) as Webhook
+      const webhook = (await db.get(ctx.params.id)) as Webhook
-    // validate against the schema
+      // validate against the schema
-    if (webhook.bodySchema) {
+      if (webhook.bodySchema) {
-      validate(ctx.request.body, webhook.bodySchema)
+        validate(ctx.request.body, webhook.bodySchema)
-    }
+      }
-    const target = await db.get(webhook.action.target)
+      const target = await db.get(webhook.action.target)
-    if (webhook.action.type === WebhookActionType.AUTOMATION) {
+      if (webhook.action.type === WebhookActionType.AUTOMATION) {
-      // trigger with both the pure request and then expand it
+        // trigger with both the pure request and then expand it
-      // incase the user has produced a schema to bind to
+        // incase the user has produced a schema to bind to
-      await triggers.externalTrigger(target, {
+        await triggers.externalTrigger(target, {
-        body: ctx.request.body,
+          body: ctx.request.body,
-        ...ctx.request.body,
+          ...ctx.request.body,
-        appId: prodAppId,
+          appId: prodAppId,
-      })
+        })
-    }
+      }
-    ctx.status = 200
-    ctx.body = {
-      message: "Webhook trigger fired successfully",
-    }
-  } catch (err: any) {
-    if (err.status === 404) {
       ctx.status = 200
       ctx.body = {
-        message: "Application not deployed yet.",
+        message: "Webhook trigger fired successfully",
+      }
+    } catch (err: any) {
+      if (err.status === 404) {
+        ctx.status = 200
+        ctx.body = {
+          message: "Application not deployed yet.",
+        }
       }
     }
-  }
+  })
 }

packages/server/src/api/routes/index.ts

@@ -33,6 +33,7 @@ export { default as publicRoutes } from "./public"
 
 const appBackupRoutes = pro.appBackups
 const scheduleRoutes = pro.schedules
+const environmentVariableRoutes = pro.environmentVariables
 
 export const mainRoutes: Router[] = [
   appBackupRoutes,
@@ -63,6 +64,7 @@ export const mainRoutes: Router[] = [
   migrationRoutes,
   pluginRoutes,
   scheduleRoutes,
+  environmentVariableRoutes,
   // these need to be handled last as they still use /api/:tableId
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,

packages/server/src/api/routes/row.ts

@@ -3,7 +3,7 @@ import * as rowController from "../controllers/row"
 import authorized from "../../middleware/authorized"
 import { paramResource, paramSubResource } from "../../middleware/resourceId"
 import { permissions } from "@budibase/backend-core"
-const { internalSearchValidator } = require("./utils/validators")
+import { internalSearchValidator } from "./utils/validators"
 const { PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -2,7 +2,8 @@ jest.mock("pg")
 import * as setup from "./utilities"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import { checkCacheForDynamicVariable } from "../../../threads/utils"
-import { events } from "@budibase/backend-core"
+import { context, events } from "@budibase/backend-core"
+import sdk from "../../../sdk"
 
 let { basicDatasource } = setup.structures
 const pg = require("pg")
@@ -184,4 +185,37 @@ describe("/datasources", () => {
       })
     })
   })
+
+  describe("check secret replacement", () => {
+    async function makeDatasource() {
+      datasource = basicDatasource()
+      datasource.datasource.config.password = "testing"
+      const res = await request
+        .post(`/api/datasources`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      return res.body.datasource
+    }
+
+    it("should save a datasource with password", async () => {
+      const datasource = await makeDatasource()
+      expect(datasource.config.password).toBe("--secret-value--")
+    })
+
+    it("should not the password on update with the --secret-value--", async () => {
+      const datasource = await makeDatasource()
+      await request
+        .put(`/api/datasources/${datasource._id}`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      await context.doInAppContext(config.getAppId(), async () => {
+        const dbDatasource: any = await sdk.datasources.get(datasource._id)
+        expect(dbDatasource.config.password).toBe("testing")
+      })
+    })
+  })
 })

packages/server/src/definitions/automations.ts

@@ -24,6 +24,7 @@ export interface TriggerOutput {
 
 export interface AutomationContext extends AutomationResults {
   steps: any[]
+  env?: Record<string, string>
   trigger: any
 }
 

packages/server/src/definitions/datasource.ts

@@ -7,44 +7,3 @@
 export interface QueryOptions {
   disableReturning?: boolean
 }
-
-export enum AuthType {
-  BASIC = "basic",
-  BEARER = "bearer",
-}
-
-interface AuthConfig {
-  _id: string
-  name: string
-  type: AuthType
-  config: BasicAuthConfig | BearerAuthConfig
-}
-
-export interface BasicAuthConfig {
-  username: string
-  password: string
-}
-
-export interface BearerAuthConfig {
-  token: string
-}
-
-export interface RestConfig {
-  url: string
-  rejectUnauthorized: boolean
-  defaultHeaders: {
-    [key: string]: any
-  }
-  legacyHttpParser: boolean
-  authConfigs: AuthConfig[]
-  staticVariables: {
-    [key: string]: string
-  }
-  dynamicVariables: [
-    {
-      name: string
-      queryId: string
-      value: string
-    }
-  ]
-}

packages/server/src/index.ts

@@ -1,6 +1,6 @@
 import { bootstrap } from "global-agent"
 const fixPath = require("fix-path")
-const { checkDevelopmentEnvironment } = require("./utilities/fileSystem")
+import { checkDevelopmentEnvironment } from "./utilities/fileSystem"
 
 function runServer() {
   // this will shutdown the system if development environment not ready

packages/server/src/integrations/base/query.ts

@@ -1,10 +1,12 @@
 import { QueryJson, Datasource } from "@budibase/types"
-const { getIntegration } = require("../index")
+import { getIntegration } from "../index"
+import sdk from "../../sdk"
 
 export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
 ) {
+  datasource = await sdk.datasources.enrich(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function
   if (Integration.prototype.query) {

packages/server/src/integrations/base/sqlTable.ts

@@ -3,7 +3,7 @@ import { Operation, QueryJson, RenameColumn, Table } from "@budibase/types"
 import { breakExternalTableId } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
-const { FieldTypes, RelationshipTypes } = require("../../constants")
+import { FieldTypes, RelationshipTypes } from "../../constants"
 
 function generateSchema(
   schema: CreateTableBuilder,

packages/server/src/integrations/dynamodb.ts

@@ -5,8 +5,8 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-const AWS = require("aws-sdk")
+import AWS from "aws-sdk"
-const { AWS_REGION } = require("../db/dynamoClient")
+import { AWS_REGION } from "../db/dynamoClient"
 
 interface DynamoDBConfig {
   region: string
@@ -182,7 +182,7 @@ class DynamoDBIntegration implements IntegrationBase {
     return response
   }
 
-  async describe(query: { table: string }) {
+  async describe(query: { table: string }): Promise<any> {
     const params = {
       TableName: query.table,
     }