Merge branch 'master' into BUDI-7655/migration-backend

LICENSE

@@ -1,7 +1,9 @@
-Copyright 2019-2021, Budibase Ltd.
+Copyright 2019-2023, Budibase Ltd.
 
 Each Budibase package has its own license, please check the license file in each package.
 
 You can consider Budibase to be GPLv3 licensed overall.
 
 The apps that you build with Budibase do not package any GPLv3 licensed code, thus do not fall under those restrictions.
+
+Budibase ships with Structured Query Server, by The Neighbourhoodie Software GmbH. This license for this can be found at ./SQS_LICENSE

SQS_LICENSE

@@ -0,0 +1,31 @@
+FORM OF CUSTOMER LICENCE
+
+Budibase hereby grants the Customer a worldwide, royalty free, non-exclusive,
+perpetual (for the lifetime of the intellectual property rights contained in the Product)
+right and title to utilise the binary code of the The Neighbourhoodie Software GmbH
+Structured Query Server software product (Product) for its own internal business
+purposes (the Purpose) only (the Licence). The Product has the function of bringing a
+CouchDB database (NoSQL database) into an SQL database form (SQLite) and thereby
+making it usable for complex queries - which originally could only be displayed in an
+SQL database. By indexing in SQLite and a server that is tailored to it, the Product
+enables the use of CouchDB with SQL queries.
+The Licence shall not permit sub-licensing, resale or transfer of the Product to third
+parties, other than sub-licensing to the Customer’s direct contractors for the purposes
+of utilizing the Product as contemplated above.
+The Licence shall not permit the adaptation, modification, decompilation, reverse
+engineering or similar activities with respect to the Product.
+This licence is granted to the Customer only, although Customer and its Affiliates’
+employees, servants and agents shall be entitled to utilize the Product within the scope
+of the Licence for the Customer’s Purpose only.
+Reproduction is not permitted to users, except for reproductions that are necessary for
+the use of the product under the licence described above. These conditions apply to the
+product regardless of the form in which we make the product available and on which
+devices it is installed and/or with which devices it is ultimately used. Depending on the
+product variant or intended use, certain technical requirements in the IT infrastructure
+must be satisfied as a prerequisite for use.
+The law of the Northern Ireland applies exclusively to this licence, and the courts of
+Northern Ireland shall have exclusive jurisdiction, save that we reserve a right to sue
+you in the jurisdiction in which you are based. The application of the UN Sales
+Convention (CISG) is excluded.
+The invalidity of any part of this licence does not affect the validity of the remaining
+regulations.

charts/budibase/README.md

@@ -87,6 +87,7 @@ couchdb:
     storageClass: "nfs-client"
   adminPassword: admin
 
+services:
   objectStore:
     storageClass: "nfs-client"
   redis:

charts/budibase/README.md.gotmpl

@@ -86,6 +86,7 @@ couchdb:
     storageClass: "nfs-client"
   adminPassword: admin
 
+services:
   objectStore:
     storageClass: "nfs-client"
   redis:

charts/budibase/templates/proxy-service-deployment.yaml

@@ -16,6 +16,7 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: budibase-proxy
+  minReadySeconds: 10
   strategy:
     type: RollingUpdate
   template:

hosting/couchdb/runner.sh

@@ -30,10 +30,18 @@ elif [[ "${TARGETBUILD}" = "single" ]]; then
     # mount, so we use that for all persistent data.
     sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
     sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then
+    # We remove the database_dir and view_index_dir settings from the local.ini
+    # in docker-compose because it will default to /opt/couchdb/data which is what
+    # our docker-compose was using prior to us switching to using our own CouchDB
+    # image.
+    sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
 elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then
     # In Kubernetes the directory /opt/couchdb/data has a persistent volume
     # mount for storing database data.
-    sed -i "s#DATA_DIR#/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
+    sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
 
     # We remove the database_dir and view_index_dir settings from the local.ini
     # in Kubernetes because it will default to /opt/couchdb/data which is what

hosting/docker-compose.yaml

@@ -57,7 +57,6 @@ services:
     depends_on:
       - redis-service
       - minio-service
-      - couch-init
 
   minio-service:
     restart: unless-stopped
@@ -70,7 +69,7 @@ services:
       MINIO_BROWSER: "off"
     command: server /data --console-address ":9001"
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      test: "timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1"
       interval: 30s
       timeout: 20s
       retries: 3
@@ -98,26 +97,15 @@ services:
 
   couchdb-service:
     restart: unless-stopped
-    image: ibmcom/couchdb3
+    image: budibase/couchdb
+    pull_policy: always
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
+      - TARGETBUILD=docker-compose
     volumes:
       - couchdb3_data:/opt/couchdb/data
 
-  couch-init:
-    image: curlimages/curl
-    environment:
-      PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984"
-    depends_on:
-      - couchdb-service
-    command:
-      [
-        "sh",
-        "-c",
-        "sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;",
-      ]
-
   redis-service:
     restart: unless-stopped
     image: redis

hosting/nginx.dev.conf

@@ -42,7 +42,7 @@ http {
   server { 
     listen       10000 default_server;
     server_name  _;
-    client_max_body_size 1000m;
+    client_max_body_size 50000m;
     ignore_invalid_headers off;
     proxy_buffering off;
 

hosting/proxy/nginx.prod.conf

@@ -249,4 +249,30 @@ http {
     gzip_comp_level 6;
     gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
   }
+
+  # From https://docs.datadoghq.com/integrations/nginx/?tab=kubernetes
+  server {
+    listen 81;
+    server_name localhost;
+
+    access_log off;
+    allow 127.0.0.1;
+    deny all;
+
+    location /nginx_status {
+      # Choose your status module
+
+      # freely available with open source NGINX
+      stub_status;
+
+      # for open source NGINX < version 1.7.5
+      # stub_status on;
+
+      # available only with NGINX Plus
+      # status;
+
+      # ensures the version information can be retrieved
+      server_tokens on;
+    }
+  }
 }

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.31",
+  "version": "2.13.35",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/auth/auth.ts

@@ -19,6 +19,7 @@ import {
   GoogleInnerConfig,
   OIDCInnerConfig,
   PlatformLogoutOpts,
+  SessionCookie,
   SSOProviderType,
 } from "@budibase/types"
 import * as events from "../events"
@@ -44,7 +45,6 @@ export const buildAuthMiddleware = authenticated
 export const buildTenancyMiddleware = tenancy
 export const buildCsrfMiddleware = csrf
 export const passport = _passport
-export const jwt = require("jsonwebtoken")
 
 // Strategies
 _passport.use(new LocalStrategy(local.options, local.authenticate))
@@ -191,10 +191,10 @@ export async function platformLogout(opts: PlatformLogoutOpts) {
 
   if (!ctx) throw new Error("Koa context must be supplied to logout.")
 
-  const currentSession = getCookie(ctx, Cookie.Auth)
+  const currentSession = getCookie<SessionCookie>(ctx, Cookie.Auth)
   let sessions = await getSessionsForUser(userId)
 
-  if (keepActiveSession) {
+  if (currentSession && keepActiveSession) {
     sessions = sessions.filter(
       session => session.sessionId !== currentSession.sessionId
     )

packages/backend-core/src/middleware/authenticated.ts

@@ -13,7 +13,7 @@ import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"
-import { Ctx, EndpointMatcher } from "@budibase/types"
+import { Ctx, EndpointMatcher, SessionCookie } from "@budibase/types"
 import { InvalidAPIKeyError, ErrorCode } from "../errors"
 
 const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
@@ -98,7 +98,9 @@ export default function (
       // check the actual user is authenticated first, try header or cookie
       let headerToken = ctx.request.headers[Header.TOKEN]
 
-      const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
+      const authCookie =
+        getCookie<SessionCookie>(ctx, Cookie.Auth) ||
+        openJwt<SessionCookie>(headerToken)
       let apiKey = ctx.request.headers[Header.API_KEY]
 
       if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) {

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -3,7 +3,7 @@ import { Cookie } from "../../../constants"
 import * as configs from "../../../configs"
 import * as cache from "../../../cache"
 import * as utils from "../../../utils"
-import { UserCtx, SSOProfile } from "@budibase/types"
+import { UserCtx, SSOProfile, DatasourceAuthCookie } from "@budibase/types"
 import { ssoSaveUserNoOp } from "../sso/sso"
 
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
@@ -58,7 +58,14 @@ export async function postAuth(
   const platformUrl = await configs.getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const authStateCookie = utils.getCookie(ctx, Cookie.DatasourceAuth)
+  const authStateCookie = utils.getCookie<{ appId: string }>(
+    ctx,
+    Cookie.DatasourceAuth
+  )
+
+  if (!authStateCookie) {
+    throw new Error("Unable to fetch datasource auth cookie")
+  }
 
   return passport.authenticate(
     new GoogleStrategy(

packages/backend-core/src/objectStore/objectStore.ts

@@ -305,20 +305,33 @@ export async function retrieveDirectory(bucketName: string, path: string) {
   let writePath = join(budibaseTempDir(), v4())
   fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
-  let fullObjects = await Promise.all(
+  let streams = await Promise.all(
-    objects.map(obj => retrieve(bucketName, obj.Key!))
+    objects.map(obj => getReadStream(bucketName, obj.Key!))
   )
   let count = 0
+  const writePromises: Promise<Error>[] = []
   for (let obj of objects) {
     const filename = obj.Key!
-    const data = fullObjects[count++]
+    const stream = streams[count++]
     const possiblePath = filename.split("/")
-    if (possiblePath.length > 1) {
+    const dirs = possiblePath.slice(0, possiblePath.length - 1)
-      const dirs = possiblePath.slice(0, possiblePath.length - 1)
+    const possibleDir = join(writePath, ...dirs)
-      fs.mkdirSync(join(writePath, ...dirs), { recursive: true })
+    if (possiblePath.length > 1 && !fs.existsSync(possibleDir)) {
+      fs.mkdirSync(possibleDir, { recursive: true })
     }
-    fs.writeFileSync(join(writePath, ...possiblePath), data)
+    const writeStream = fs.createWriteStream(join(writePath, ...possiblePath), {
+      mode: 0o644,
+    })
+    stream.pipe(writeStream)
+    writePromises.push(
+      new Promise((resolve, reject) => {
+        stream.on("finish", resolve)
+        stream.on("error", reject)
+        writeStream.on("error", reject)
+      })
+    )
   }
+  await Promise.all(writePromises)
   return writePath
 }
 

packages/backend-core/src/security/encryption.ts

@@ -73,6 +73,9 @@ export async function encryptFile(
   const outputFileName = `${filename}.enc`
 
   const filePath = join(dir, filename)
+  if (fs.lstatSync(filePath).isDirectory()) {
+    throw new Error("Unable to encrypt directory")
+  }
   const inputFile = fs.createReadStream(filePath)
   const outputFile = fs.createWriteStream(join(dir, outputFileName))
 
@@ -110,6 +113,9 @@ export async function decryptFile(
   outputPath: string,
   secret: string
 ) {
+  if (fs.lstatSync(inputPath).isDirectory()) {
+    throw new Error("Unable to encrypt directory")
+  }
   const { salt, iv } = await getSaltAndIV(inputPath)
   const inputFile = fs.createReadStream(inputPath, {
     start: SALT_LENGTH + IV_LENGTH,

packages/backend-core/src/utils/utils.ts

@@ -11,8 +11,7 @@ import {
   TenantResolutionStrategy,
 } from "@budibase/types"
 import type { SetOption } from "cookies"
-
+import jwt, { Secret } from "jsonwebtoken"
-const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR
 const PROD_APP_PREFIX = "/app/"
@@ -60,10 +59,7 @@ export function isServingApp(ctx: Ctx) {
     return true
   }
   // prod app
-  if (ctx.path.startsWith(PROD_APP_PREFIX)) {
+  return ctx.path.startsWith(PROD_APP_PREFIX)
-    return true
-  }
-  return false
 }
 
 export function isServingBuilder(ctx: Ctx): boolean {
@@ -138,16 +134,16 @@ function parseAppIdFromUrl(url?: string) {
  * opens the contents of the specified encrypted JWT.
  * @return the contents of the token.
  */
-export function openJwt(token: string) {
+export function openJwt<T>(token?: string): T | undefined {
   if (!token) {
-    return token
+    return undefined
   }
   try {
-    return jwt.verify(token, env.JWT_SECRET)
+    return jwt.verify(token, env.JWT_SECRET as Secret) as T
   } catch (e) {
     if (env.JWT_SECRET_FALLBACK) {
       // fallback to enable rotation
-      return jwt.verify(token, env.JWT_SECRET_FALLBACK)
+      return jwt.verify(token, env.JWT_SECRET_FALLBACK) as T
     } else {
       throw e
     }
@@ -159,13 +155,9 @@ export function isValidInternalAPIKey(apiKey: string) {
     return true
   }
   // fallback to enable rotation
-  if (
+  return !!(
-    env.INTERNAL_API_KEY_FALLBACK &&
+    env.INTERNAL_API_KEY_FALLBACK && env.INTERNAL_API_KEY_FALLBACK === apiKey
-    env.INTERNAL_API_KEY_FALLBACK === apiKey
+  )
-  ) {
-    return true
-  }
-  return false
 }
 
 /**
@@ -173,14 +165,14 @@ export function isValidInternalAPIKey(apiKey: string) {
  * @param ctx The request which is to be manipulated.
  * @param name The name of the cookie to get.
  */
-export function getCookie(ctx: Ctx, name: string) {
+export function getCookie<T>(ctx: Ctx, name: string) {
   const cookie = ctx.cookies.get(name)
 
   if (!cookie) {
-    return cookie
+    return undefined
   }
 
-  return openJwt(cookie)
+  return openJwt<T>(cookie)
 }
 
 /**
@@ -197,7 +189,7 @@ export function setCookie(
   opts = { sign: true }
 ) {
   if (value && opts && opts.sign) {
-    value = jwt.sign(value, env.JWT_SECRET)
+    value = jwt.sign(value, env.JWT_SECRET as Secret)
   }
 
   const config: SetOption = {

packages/bbui/src/DetailSummary/DetailSummary.svelte

@@ -1,20 +1,17 @@
 <script>
   import Icon from "../Icon/Icon.svelte"
-  import { createEventDispatcher } from "svelte"
 
   export let name
-  export let show = false
+  export let initiallyShow = false
   export let collapsible = true
 
-  const dispatch = createEventDispatcher()
+  let show = initiallyShow
+
   const onHeaderClick = () => {
     if (!collapsible) {
       return
     }
     show = !show
-    if (show) {
-      dispatch("open")
-    }
   }
 </script>
 

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -53,7 +53,7 @@
   $: {
     if (selectedImage?.url) {
       selectedUrl = selectedImage?.url
-    } else if (selectedImage) {
+    } else if (selectedImage && isImage) {
       try {
         let reader = new FileReader()
         reader.readAsDataURL(selectedImage)

packages/builder/src/builderStore/index.js

@@ -8,6 +8,7 @@ import { derived, get } from "svelte/store"
 import { findComponent, findComponentPath } from "./componentUtils"
 import { RoleUtils } from "@budibase/frontend-core"
 import { createHistoryStore } from "builderStore/store/history"
+import { cloneDeep } from "lodash/fp"
 
 export const store = getFrontendStore()
 export const automationStore = getAutomationStore()
@@ -69,7 +70,14 @@ export const selectedComponent = derived(
     if (!$selectedScreen || !$store.selectedComponentId) {
       return null
     }
-    return findComponent($selectedScreen?.props, $store.selectedComponentId)
+    const selected = findComponent(
+      $selectedScreen?.props,
+      $store.selectedComponentId
+    )
+
+    const clone = selected ? cloneDeep(selected) : selected
+    store.actions.components.migrateSettings(clone)
+    return clone
   }
 )
 

packages/builder/src/builderStore/store/frontend.js

@@ -85,6 +85,7 @@ const INITIAL_FRONTEND_STATE = {
   selectedScreenId: null,
   selectedComponentId: null,
   selectedLayoutId: null,
+  hoverComponentId: null,
 
   // Client state
   selectedComponentInstance: null,
@@ -112,7 +113,7 @@ export const getFrontendStore = () => {
     }
     let clone = cloneDeep(screen)
     const result = patchFn(clone)
-
+    // An explicit false result means skip this change
     if (result === false) {
       return
     }
@@ -601,6 +602,36 @@ export const getFrontendStore = () => {
         // Finally try an external table
         return validTables.find(table => table.sourceType === DB_TYPE_EXTERNAL)
       },
+      migrateSettings: enrichedComponent => {
+        const componentPrefix = "@budibase/standard-components"
+        let migrated = false
+
+        if (enrichedComponent?._component == `${componentPrefix}/formblock`) {
+          // Use default config if the 'buttons' prop has never been initialised
+          if (!("buttons" in enrichedComponent)) {
+            enrichedComponent["buttons"] =
+              Utils.buildDynamicButtonConfig(enrichedComponent)
+            migrated = true
+          } else if (enrichedComponent["buttons"] == null) {
+            // Ignore legacy config if 'buttons' has been reset by 'resetOn'
+            const { _id, actionType, dataSource } = enrichedComponent
+            enrichedComponent["buttons"] = Utils.buildDynamicButtonConfig({
+              _id,
+              actionType,
+              dataSource,
+            })
+            migrated = true
+          }
+
+          // Ensure existing Formblocks position their buttons at the top.
+          if (!("buttonPosition" in enrichedComponent)) {
+            enrichedComponent["buttonPosition"] = "top"
+            migrated = true
+          }
+        }
+
+        return migrated
+      },
       enrichEmptySettings: (component, opts) => {
         if (!component?._component) {
           return
@@ -672,7 +703,6 @@ export const getFrontendStore = () => {
               component[setting.key] = setting.defaultValue
             }
           }
-
           // Validate non-empty settings
           else {
             if (setting.type === "dataProvider") {
@@ -722,6 +752,9 @@ export const getFrontendStore = () => {
           useDefaultValues: true,
         })
 
+        // Migrate nested component settings
+        store.actions.components.migrateSettings(instance)
+
         // Add any extra properties the component needs
         let extras = {}
         if (definition.hasChildren) {
@@ -845,7 +878,16 @@ export const getFrontendStore = () => {
           if (!component) {
             return false
           }
-          return patchFn(component, screen)
+
+          // Mutates the fetched component with updates
+          const patchResult = patchFn(component, screen)
+
+          // Mutates the component with any required settings updates
+          const migrated = store.actions.components.migrateSettings(component)
+
+          // Returning an explicit false signifies that we should skip this
+          // update. If we migrated something, ensure we never skip.
+          return migrated ? null : patchResult
         }
         await store.actions.screens.patch(patchScreen, screenId)
       },
@@ -1247,9 +1289,13 @@ export const getFrontendStore = () => {
           const settings = getComponentSettings(component._component)
           const updatedSetting = settings.find(setting => setting.key === name)
 
-          const resetFields = settings.filter(
+          // Can be a single string or array of strings
-            setting => name === setting.resetOn
+          const resetFields = settings.filter(setting => {
-          )
+            return (
+              name === setting.resetOn ||
+              (Array.isArray(setting.resetOn) && setting.resetOn.includes(name))
+            )
+          })
           resetFields?.forEach(setting => {
             component[setting.key] = null
           })
@@ -1271,6 +1317,7 @@ export const getFrontendStore = () => {
             })
           }
           component[name] = value
+          return true
         }
       },
       requestEjectBlock: componentId => {
@@ -1278,7 +1325,6 @@ export const getFrontendStore = () => {
       },
       handleEjectBlock: async (componentId, ejectedDefinition) => {
         let nextSelectedComponentId
-
         await store.actions.screens.patch(screen => {
           const block = findComponent(screen.props, componentId)
           const parent = findComponentParent(screen.props, componentId)

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -57,16 +57,11 @@
       }}
       class="buttons"
     >
-      <Icon hoverable size="M" name="Play" />
+      <Icon size="M" name="Play" />
       <div>Run test</div>
     </div>
     <div class="buttons">
-      <Icon
+      <Icon disabled={!$automationStore.testResults} size="M" name="Multiple" />
-        disabled={!$automationStore.testResults}
-        hoverable
-        size="M"
-        name="Multiple"
-      />
       <div
         class:disabled={!$automationStore.testResults}
         on:click={() => {

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte

@@ -97,6 +97,7 @@
   class:typing={typing && !automationNameError}
   class:typing-error={automationNameError}
   class="blockSection"
+  on:click={() => dispatch("toggle")}
 >
   <div class="splitHeader">
     <div class="center-items">
@@ -138,7 +139,20 @@
             on:input={e => {
               automationName = e.target.value.trim()
             }}
-            on:click={startTyping}
+            on:click={e => {
+              e.stopPropagation()
+              startTyping()
+            }}
+            on:keydown={async e => {
+              if (e.key === "Enter") {
+                typing = false
+                if (automationNameError) {
+                  automationName = stepNames[block.id] || block?.name
+                } else {
+                  await saveName()
+                }
+              }
+            }}
             on:blur={async () => {
               typing = false
               if (automationNameError) {
@@ -168,7 +182,11 @@
             </StatusLight>
           </div>
           <Icon
-            on:click={() => dispatch("toggle")}
+            e.stopPropagation()
+            on:click={e => {
+              e.stopPropagation()
+              dispatch("toggle")
+            }}
             hoverable
             name={open ? "ChevronUp" : "ChevronDown"}
           />
@@ -195,7 +213,10 @@
         {/if}
         {#if !showTestStatus}
           <Icon
-            on:click={() => dispatch("toggle")}
+            on:click={e => {
+              e.stopPropagation()
+              dispatch("toggle")
+            }}
             hoverable
             name={open ? "ChevronUp" : "ChevronDown"}
           />

packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte

@@ -1,11 +1,9 @@
 <script>
   import {
     ModalContent,
-    Tabs,
-    Tab,
     TextArea,
-    Label,
     notifications,
+    ActionButton,
   } from "@budibase/bbui"
   import { automationStore, selectedAutomation } from "builderStore"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
@@ -55,50 +53,69 @@
       notifications.error(error)
     }
   }
+
+  const toggle = () => {
+    selectedValues = !selectedValues
+    selectedJSON = !selectedJSON
+  }
+  let selectedValues = true
+  let selectedJSON = false
 </script>
 
 <ModalContent
   title="Add test data"
-  confirmText="Test"
+  confirmText="Run test"
-  size="M"
+  size="L"
   showConfirmButton={true}
   disabled={isError}
   onConfirm={testAutomation}
   cancelText="Cancel"
 >
-  <Tabs selected="Form" quiet>
+  <div class="size">
-    <Tab icon="Form" title="Form">
+    <div class="options">
-      <div class="tab-content-padding">
+      <ActionButton quiet selected={selectedValues} on:click={toggle}
-        <AutomationBlockSetup
+        >Use values</ActionButton
-          {testData}
+      >
-          {schemaProperties}
+      <ActionButton quiet selected={selectedJSON} on:click={toggle}
-          isTestModal
+        >Use JSON</ActionButton
-          block={trigger}
+      >
-        />
+    </div>
-      </div></Tab
+  </div>
-    >
+
-    <Tab icon="FileJson" title="JSON">
+  {#if selectedValues}
-      <div class="tab-content-padding">
+    <div class="tab-content-padding">
-        <Label>JSON</Label>
+      <AutomationBlockSetup
-        <div class="text-area-container">
+        {testData}
-          <TextArea
+        {schemaProperties}
-            value={JSON.stringify($selectedAutomation.testData, null, 2)}
+        isTestModal
-            error={failedParse}
+        block={trigger}
-            on:change={e => parseTestJSON(e)}
+      />
-          />
+    </div>
-        </div>
+  {/if}
-      </div>
+  {#if selectedJSON}
-    </Tab>
+    <div class="text-area-container">
-  </Tabs>
+      <TextArea
+        value={JSON.stringify($selectedAutomation.testData, null, 2)}
+        error={failedParse}
+        on:change={e => parseTestJSON(e)}
+      />
+    </div>
+  {/if}
 </ModalContent>
 
 <style>
   .text-area-container :global(textarea) {
-    min-height: 200px;
+    min-height: 300px;
-    height: 200px;
+    height: 300px;
   }
 
   .tab-content-padding {
-    padding: 0 var(--spacing-xl);
+    padding: 0 var(--spacing-s);
+  }
+
+  .options {
+    display: flex;
+    align-items: center;
+    gap: 8px;
   }
 </style>

packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte

@@ -9,7 +9,7 @@
 <div class="title">
   <div class="title-text">
     <Icon name="MultipleCheck" />
-    <div style="padding-left: var(--spacing-l)">Test Details</div>
+    <div style="padding-left: var(--spacing-l); ">Test Details</div>
   </div>
   <div style="padding-right: var(--spacing-xl)">
     <Icon
@@ -40,6 +40,7 @@
     display: flex;
     flex-direction: row;
     align-items: center;
+    padding-top: var(--spacing-s);
   }
 
   .title :global(h1) {

packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte

@@ -1,20 +1,44 @@
 <script>
   import AutomationList from "./AutomationList.svelte"
   import CreateAutomationModal from "./CreateAutomationModal.svelte"
-  import { Modal, Button, Layout } from "@budibase/bbui"
+  import { Modal, Icon } from "@budibase/bbui"
   import Panel from "components/design/Panel.svelte"
 
   export let modal
   export let webhookModal
 </script>
 
-<Panel title="Automations" borderRight>
+<Panel title="Automations" borderRight noHeaderBorder titleCSS={false}>
-  <Layout paddingX="L" paddingY="XL" gap="S">
+  <span class="panel-title-content" slot="panel-title-content">
-    <Button cta on:click={modal.show}>Add automation</Button>
+    <div class="header">
-  </Layout>
+      <div>Automations</div>
+      <div on:click={modal.show} class="add-automation-button">
+        <Icon name="Add" />
+      </div>
+    </div>
+  </span>
   <AutomationList />
 </Panel>
 
 <Modal bind:this={modal}>
   <CreateAutomationModal {webhookModal} />
 </Modal>
+
+<style>
+  .header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    gap: var(--spacing-m);
+  }
+
+  .add-automation-button {
+    margin-left: 130px;
+    color: var(--grey-7);
+    cursor: pointer;
+  }
+
+  .add-automation-button:hover {
+    color: var(--ink);
+  }
+</style>

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -149,7 +149,6 @@
   }
   const initialiseField = (field, savingColumn) => {
     isCreating = !field
-
     if (field && !savingColumn) {
       editableColumn = cloneDeep(field)
       originalName = editableColumn.name ? editableColumn.name + "" : null
@@ -171,7 +170,8 @@
           relationshipPart2 = part2
         }
       }
-    } else if (!savingColumn) {
+    }
+    if (!savingColumn && !originalName) {
       let highestNumber = 0
       Object.keys(table.schema).forEach(columnName => {
         const columnNumber = extractColumnNumber(columnName)
@@ -307,12 +307,6 @@
       dispatch("updatecolumns")
       gridDispatch("close-edit-column")
 
-      if (saveColumn.type === LINK_TYPE) {
-        // Fetching the new tables
-        tables.fetch()
-        // Fetching the new relationships
-        datasources.fetch()
-      }
       if (originalName) {
         notifications.success("Column updated successfully")
       } else {
@@ -339,11 +333,6 @@
         confirmDeleteDialog.hide()
         dispatch("updatecolumns")
         gridDispatch("close-edit-column")
-
-        if (editableColumn.type === LINK_TYPE) {
-          // Updating the relationships
-          datasources.fetch()
-        }
       }
     } catch (error) {
       notifications.error(`Error deleting column: ${error.message}`)
@@ -540,8 +529,16 @@
 <Layout noPadding gap="S">
   {#if mounted}
     <Input
+      value={editableColumn.name}
       autofocus
-      bind:value={editableColumn.name}
+      on:input={e => {
+        if (
+          !uneditable &&
+          !(linkEditDisabled && editableColumn.type === LINK_TYPE)
+        ) {
+          editableColumn.name = e.target.value
+        }
+      }}
       disabled={uneditable ||
         (linkEditDisabled && editableColumn.type === LINK_TYPE)}
       error={errors?.name}

packages/builder/src/components/common/NavItem.svelte

@@ -23,6 +23,7 @@
   export let showTooltip = false
   export let selectedBy = null
   export let compact = false
+  export let hovering = false
 
   const scrollApi = getContext("scroll")
   const dispatch = createEventDispatcher()
@@ -61,6 +62,7 @@
 
 <div
   class="nav-item"
+  class:hovering
   class:border
   class:selected
   class:withActions
@@ -71,6 +73,8 @@
   on:dragstart
   on:dragover
   on:drop
+  on:mouseenter
+  on:mouseleave
   on:click={onClick}
   ondragover="return false"
   ondragenter="return false"
@@ -152,15 +156,17 @@
     --avatars-background: var(--spectrum-global-color-gray-200);
   }
   .nav-item.selected {
-    background-color: var(--spectrum-global-color-gray-300);
+    background-color: var(--spectrum-global-color-gray-300) !important;
     --avatars-background: var(--spectrum-global-color-gray-300);
     color: var(--ink);
   }
-  .nav-item:hover {
+  .nav-item:hover,
-    background-color: var(--spectrum-global-color-gray-300);
+  .hovering {
+    background-color: var(--spectrum-global-color-gray-200);
     --avatars-background: var(--spectrum-global-color-gray-300);
   }
-  .nav-item:hover .actions {
+  .nav-item:hover .actions,
+  .hovering .actions {
     visibility: visible;
   }
   .nav-item-content {

packages/builder/src/components/design/Panel.svelte

@@ -16,7 +16,8 @@
   export let wide = false
   export let extraWide = false
   export let closeButtonIcon = "Close"
-
+  export let noHeaderBorder = false
+  export let titleCSS = true
   $: customHeaderContent = $$slots["panel-header-content"]
   $: customTitleContent = $$slots["panel-title-content"]
 </script>
@@ -32,6 +33,7 @@
     class="header"
     class:custom={customHeaderContent}
     class:borderBottom={borderBottomHeader}
+    class:noHeaderBorder
   >
     {#if showBackButton}
       <Icon name="ArrowLeft" hoverable on:click={onClickBackButton} />
@@ -41,7 +43,7 @@
         <Icon name={icon} />
       </AbsTooltip>
     {/if}
-    <div class="title">
+    <div class:title={titleCSS}>
       {#if customTitleContent}
         <slot name="panel-title-content" />
       {:else}
@@ -106,6 +108,10 @@
     padding: 0 var(--spacing-l);
     gap: var(--spacing-m);
   }
+
+  .noHeaderBorder {
+    border-bottom: none !important;
+  }
   .header.borderBottom {
     border-bottom: var(--border-light);
   }

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/FieldSetting.svelte

@@ -49,7 +49,15 @@
     <div class="field-label">{item.label || item.field}</div>
   </div>
   <div class="list-item-right">
-    <Toggle on:change={onToggle(item)} text="" value={item.active} thin />
+    <Toggle
+      on:click={e => {
+        e.stopPropagation()
+      }}
+      on:change={onToggle(item)}
+      text=""
+      value={item.active}
+      thin
+    />
   </div>
 </div>
 

packages/builder/src/components/start/ExportAppModal.svelte

@@ -13,7 +13,7 @@
   export let app
   export let published
   let includeInternalTablesRows = true
-  let encypt = true
+  let encrypt = true
 
   let password = null
   const validation = createValidationStore()
@@ -27,9 +27,9 @@
   $: stepConfig = {
     [Step.CONFIG]: {
       title: published ? "Export published app" : "Export latest app",
-      confirmText: encypt ? "Continue" : exportButtonText,
+      confirmText: encrypt ? "Continue" : exportButtonText,
       onConfirm: () => {
-        if (!encypt) {
+        if (!encrypt) {
           exportApp()
         } else {
           currentStep = Step.SET_PASSWORD
@@ -46,7 +46,7 @@
         if (!$validation.valid) {
           return keepOpen
         }
-        exportApp(password)
+        await exportApp(password)
       },
       isValid: $validation.valid,
     },
@@ -109,13 +109,13 @@
         text="Export rows from internal tables"
         bind:value={includeInternalTablesRows}
       />
-      <Toggle text="Encrypt my export" bind:value={encypt} />
+      <Toggle text="Encrypt my export" bind:value={encrypt} />
     </Body>
-    {#if !encypt}
+    <InlineAlert
-      <InlineAlert
+      header={encrypt
-        header="Do not share your budibase application exports publicly as they may contain sensitive information such as database credentials or secret keys."
+        ? "Please note Budibase does not encrypt attachments during the export process to ensure efficient export of large attachments."
-      />
+        : "Do not share your Budibase application exports publicly as they may contain sensitive information such as database credentials or secret keys."}
-    {/if}
+    />
   {/if}
   {#if currentStep === Step.SET_PASSWORD}
     <Input

packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte

@@ -110,7 +110,7 @@
   }
 
   .setup {
-    padding-top: var(--spectrum-global-dimension-size-200);
+    padding-top: 9px;
     border-left: var(--border-light);
     display: flex;
     flex-direction: column;

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -32,6 +32,7 @@
     const generalSettings = settings.filter(
       setting => !setting.section && setting.tag === tag
     )
+
     const customSections = settings.filter(
       setting => setting.section && setting.tag === tag
     )
@@ -151,7 +152,7 @@
   {#if section.visible}
     <DetailSummary
       name={showSectionTitle ? section.name : ""}
-      show={section.collapsed !== true}
+      initiallyShow={section.collapsed !== true}
     >
       {#if section.info}
         <div class="section-info">

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -36,12 +36,14 @@
 
   // Determine selected component ID
   $: selectedComponentId = $store.selectedComponentId
+  $: hoverComponentId = $store.hoverComponentId
 
   $: previewData = {
     appId: $store.appId,
     layout,
     screen,
     selectedComponentId,
+    hoverComponentId,
     theme: $store.theme,
     customTheme: $store.customTheme,
     previewDevice: $store.previewDevice,
@@ -117,6 +119,8 @@
       error = event.error || "An unknown error occurred"
     } else if (type === "select-component" && data.id) {
       $store.selectedComponentId = data.id
+    } else if (type === "hover-component" && data.id) {
+      $store.hoverComponentId = data.id
     } else if (type === "update-prop") {
       await store.actions.components.updateSetting(data.prop, data.value)
     } else if (type === "update-styles") {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte

@@ -89,6 +89,17 @@
     }
     return findComponentPath($selectedComponent, component._id)?.length > 0
   }
+
+  const handleMouseover = componentId => {
+    if ($store.hoverComponentId !== componentId) {
+      $store.hoverComponentId = componentId
+    }
+  }
+  const handleMouseout = componentId => {
+    if ($store.hoverComponentId === componentId) {
+      $store.hoverComponentId = null
+    }
+  }
 </script>
 
 <ul>
@@ -109,6 +120,9 @@
         on:dragover={dragover(component, index)}
         on:iconClick={() => toggleNodeOpen(component._id)}
         on:drop={onDrop}
+        hovering={$store.hoverComponentId === component._id}
+        on:mouseenter={() => handleMouseover(component._id)}
+        on:mouseleave={() => handleMouseout(component._id)}
         text={getComponentText(component)}
         icon={getComponentIcon(component)}
         iconTooltip={getComponentName(component)}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/index.svelte

@@ -32,6 +32,17 @@
   const handleScroll = e => {
     scrolling = e.target.scrollTop !== 0
   }
+
+  const handleMouseover = componentId => {
+    if ($store.hoverComponentId !== componentId) {
+      $store.hoverComponentId = componentId
+    }
+  }
+  const handleMouseout = componentId => {
+    if ($store.hoverComponentId === componentId) {
+      $store.hoverComponentId = null
+    }
+  }
 </script>
 
 <div class="components">
@@ -57,6 +68,12 @@
             on:click={() => {
               $store.selectedComponentId = `${$store.selectedScreenId}-screen`
             }}
+            hovering={$store.hoverComponentId ===
+              `${$store.selectedScreenId}-screen`}
+            on:mouseenter={() =>
+              handleMouseover(`${$store.selectedScreenId}-screen`)}
+            on:mouseleave={() =>
+              handleMouseout(`${$store.selectedScreenId}-screen`)}
             id={`component-screen`}
             selectedBy={$userSelectedResourceMap[
               `${$store.selectedScreenId}-screen`
@@ -78,6 +95,12 @@
             on:click={() => {
               $store.selectedComponentId = `${$store.selectedScreenId}-navigation`
             }}
+            hovering={$store.hoverComponentId ===
+              `${$store.selectedScreenId}-navigation`}
+            on:mouseenter={() =>
+              handleMouseover(`${$store.selectedScreenId}-navigation`)}
+            on:mouseleave={() =>
+              handleMouseout(`${$store.selectedScreenId}-navigation`)}
             id={`component-nav`}
             selectedBy={$userSelectedResourceMap[
               `${$store.selectedScreenId}-navigation`

packages/builder/src/stores/backend/tables.js

@@ -81,13 +81,21 @@ export function createTablesStore() {
     replaceTable(savedTable._id, savedTable)
     select(savedTable._id)
     // make sure tables up to date (related)
-    let tableIdsToFetch = []
+    let newTableIds = []
     for (let column of Object.values(updatedTable?.schema || {})) {
       if (column.type === FIELDS.LINK.type) {
-        tableIdsToFetch.push(column.tableId)
+        newTableIds.push(column.tableId)
       }
     }
-    tableIdsToFetch = [...new Set(tableIdsToFetch)]
+
+    let oldTableIds = []
+    for (let column of Object.values(oldTable?.schema || {})) {
+      if (column.type === FIELDS.LINK.type) {
+        oldTableIds.push(column.tableId)
+      }
+    }
+
+    const tableIdsToFetch = [...new Set([...newTableIds, ...oldTableIds])]
     // too many tables to fetch, just get all
     if (tableIdsToFetch.length > 3) {
       await fetch()

packages/cli/package.json

@@ -15,7 +15,8 @@
     "pkg": "pkg . --out-path build --no-bytecode --public --public-packages \"*\" -C GZip",
     "build": "yarn prebuild && yarn rename && yarn tsc && yarn pkg && yarn postbuild",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
-    "postbuild": "rm -rf prebuilds 2> /dev/null"
+    "postbuild": "rm -rf prebuilds 2> /dev/null",
+    "start": "ts-node ./src/index.ts"
   },
   "pkg": {
     "targets": [

packages/client/manifest.json

@@ -6112,54 +6112,32 @@
           }
         ]
       },
+      {
+        "tag": "style",
+        "type": "select",
+        "label": "Button position",
+        "key": "buttonPosition",
+        "options": [
+          {
+            "label": "Bottom",
+            "value": "bottom"
+          },
+          {
+            "label": "Top",
+            "value": "top"
+          }
+        ],
+        "defaultValue": "bottom"
+      },
       {
         "section": true,
         "name": "Buttons",
-        "dependsOn": {
-          "setting": "actionType",
-          "value": "View",
-          "invert": true
-        },
         "settings": [
           {
-            "type": "text",
+            "type": "buttonConfiguration",
-            "key": "saveButtonLabel",
+            "key": "buttons",
-            "label": "Save button",
             "nested": true,
-            "defaultValue": "Save"
+            "resetOn": ["actionType", "dataSource"]
-          },
-          {
-            "type": "text",
-            "key": "deleteButtonLabel",
-            "label": "Delete button",
-            "nested": true,
-            "defaultValue": "Delete",
-            "dependsOn": {
-              "setting": "actionType",
-              "value": "Update"
-            }
-          },
-          {
-            "type": "url",
-            "label": "Navigate after button press",
-            "key": "actionUrl",
-            "placeholder": "Choose a screen",
-            "dependsOn": {
-              "setting": "actionType",
-              "value": "View",
-              "invert": true
-            }
-          },
-          {
-            "type": "boolean",
-            "label": "Hide notifications",
-            "key": "notificationOverride",
-            "defaultValue": false,
-            "dependsOn": {
-              "setting": "actionType",
-              "value": "View",
-              "invert": true
-            }
           }
         ]
       },

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -5,6 +5,7 @@
   import BlockComponent from "components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
   import { enrichSearchColumns, enrichFilter } from "utils/blocks.js"
+  import { Utils } from "@budibase/frontend-core"
 
   export let title
   export let dataSource
@@ -33,6 +34,7 @@
   export let notificationOverride
 
   const { fetchDatasourceSchema, API } = getContext("sdk")
+  const component = getContext("component")
   const stateKey = `ID_${generate()}`
 
   let formId
@@ -259,16 +261,25 @@
             name="Details form block"
             type="formblock"
             bind:id={detailsFormBlockId}
+            context="form-edit"
             props={{
               dataSource,
-              saveButtonLabel: sidePanelSaveLabel || "Save", //always show
+              buttonPosition: "top",
-              deleteButtonLabel: deleteLabel,
+              buttons: Utils.buildDynamicButtonConfig({
+                _id: $component.id + "-form-edit",
+                showDeleteButton: deleteLabel !== "",
+                showSaveButton: true,
+                saveButtonLabel: sidePanelSaveLabel || "Save",
+                deleteButtonLabel: deleteLabel,
+                notificationOverride,
+                actionType: "Update",
+                dataSource,
+              }),
               actionType: "Update",
               rowId: `{{ ${safe("state")}.${safe(stateKey)} }}`,
               fields: sidePanelFields || normalFields,
               title: editTitle,
               labelPosition: "left",
-              notificationOverride,
             }}
           />
         </BlockComponent>
@@ -284,16 +295,23 @@
           <BlockComponent
             name="New row form block"
             type="formblock"
+            context="form-new"
             props={{
               dataSource,
-              showSaveButton: true,
+              buttonPosition: "top",
-              showDeleteButton: false,
+              buttons: Utils.buildDynamicButtonConfig({
-              saveButtonLabel: sidePanelSaveLabel || "Save", //always show
+                _id: $component.id + "-form-new",
+                showDeleteButton: false,
+                showSaveButton: true,
+                saveButtonLabel: "Save",
+                notificationOverride,
+                actionType: "Create",
+                dataSource,
+              }),
               actionType: "Create",
               fields: sidePanelFields || normalFields,
               title: "Create Row",
               labelPosition: "left",
-              notificationOverride,
             }}
           />
         </BlockComponent>

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -4,28 +4,31 @@
   import Block from "components/Block.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
   import InnerFormBlock from "./InnerFormBlock.svelte"
+  import { Utils } from "@budibase/frontend-core"
 
   export let actionType
   export let dataSource
   export let size
   export let disabled
   export let fields
+  export let buttons
+  export let buttonPosition
+
   export let title
   export let description
-  export let showDeleteButton
-  export let showSaveButton
-  export let saveButtonLabel
-  export let deleteButtonLabel
   export let rowId
   export let actionUrl
   export let noRowsMessage
   export let notificationOverride
 
-  // Accommodate old config to ensure delete button does not reappear
+  // Legacy
-  $: deleteLabel = showDeleteButton === false ? "" : deleteButtonLabel?.trim()
+  export let showDeleteButton
-  $: saveLabel = showSaveButton === false ? "" : saveButtonLabel?.trim()
+  export let showSaveButton
+  export let saveButtonLabel
+  export let deleteButtonLabel
 
   const { fetchDatasourceSchema } = getContext("sdk")
+  const component = getContext("component")
 
   const convertOldFieldFormat = fields => {
     if (!fields) {
@@ -98,11 +101,23 @@
     fields: fieldsOrDefault,
     title,
     description,
-    saveButtonLabel: saveLabel,
-    deleteButtonLabel: deleteLabel,
     schema,
     repeaterId,
     notificationOverride,
+    buttons:
+      buttons ||
+      Utils.buildDynamicButtonConfig({
+        _id: $component.id,
+        showDeleteButton,
+        showSaveButton,
+        saveButtonLabel,
+        deleteButtonLabel,
+        notificationOverride,
+        actionType,
+        actionUrl,
+        dataSource,
+      }),
+    buttonPosition: buttons ? buttonPosition : "top",
   }
   const fetchSchema = async () => {
     schema = (await fetchDatasourceSchema(dataSource)) || {}

packages/client/src/components/app/blocks/form/InnerFormBlock.svelte

@@ -1,22 +1,18 @@
 <script>
   import BlockComponent from "components/BlockComponent.svelte"
   import Placeholder from "components/app/Placeholder.svelte"
-  import { makePropSafe as safe } from "@budibase/string-templates"
   import { getContext } from "svelte"
 
   export let dataSource
-  export let actionUrl
   export let actionType
   export let size
   export let disabled
   export let fields
   export let title
   export let description
-  export let saveButtonLabel
+  export let buttons
-  export let deleteButtonLabel
+  export let buttonPosition = "bottom"
   export let schema
-  export let repeaterId
-  export let notificationOverride
 
   const FieldTypeToComponentMap = {
     string: "stringfield",
@@ -37,74 +33,7 @@
 
   let formId
 
-  $: onSave = [
+  $: renderHeader = buttons || title
-    {
-      "##eventHandlerType": "Validate Form",
-      parameters: {
-        componentId: formId,
-      },
-    },
-    {
-      "##eventHandlerType": "Save Row",
-      parameters: {
-        providerId: formId,
-        tableId: dataSource?.resourceId,
-        notificationOverride,
-      },
-    },
-    {
-      "##eventHandlerType": "Close Screen Modal",
-    },
-    {
-      "##eventHandlerType": "Close Side Panel",
-    },
-    // Clear a create form once submitted
-    ...(actionType !== "Create"
-      ? []
-      : [
-          {
-            "##eventHandlerType": "Clear Form",
-            parameters: {
-              componentId: formId,
-            },
-          },
-        ]),
-    {
-      "##eventHandlerType": "Navigate To",
-      parameters: {
-        url: actionUrl,
-      },
-    },
-  ]
-  $: onDelete = [
-    {
-      "##eventHandlerType": "Delete Row",
-      parameters: {
-        confirm: true,
-        tableId: dataSource?.resourceId,
-        rowId: `{{ ${safe(repeaterId)}.${safe("_id")} }}`,
-        revId: `{{ ${safe(repeaterId)}.${safe("_rev")} }}`,
-        notificationOverride,
-      },
-    },
-    {
-      "##eventHandlerType": "Close Screen Modal",
-    },
-    {
-      "##eventHandlerType": "Close Side Panel",
-    },
-    {
-      "##eventHandlerType": "Navigate To",
-      parameters: {
-        url: actionUrl,
-      },
-    },
-  ]
-
-  $: renderDeleteButton = deleteButtonLabel && actionType === "Update"
-  $: renderSaveButton = saveButtonLabel && actionType !== "View"
-  $: renderButtons = renderDeleteButton || renderSaveButton
-  $: renderHeader = renderButtons || title
 
   const getComponentForField = field => {
     const fieldSchemaName = field.field || field.name
@@ -184,42 +113,14 @@
               props={{ text: title || "" }}
               order={0}
             />
-            {#if renderButtons}
+            {#if buttonPosition == "top"}
               <BlockComponent
-                type="container"
+                type="buttongroup"
                 props={{
-                  direction: "row",
+                  buttons,
-                  hAlign: "stretch",
-                  vAlign: "center",
-                  gap: "M",
-                  wrap: true,
                 }}
-                order={1}
+                order={0}
-              >
+              />
-                {#if renderDeleteButton}
-                  <BlockComponent
-                    type="button"
-                    props={{
-                      text: deleteButtonLabel,
-                      onClick: onDelete,
-                      quiet: true,
-                      type: "secondary",
-                    }}
-                    order={0}
-                  />
-                {/if}
-                {#if renderSaveButton}
-                  <BlockComponent
-                    type="button"
-                    props={{
-                      text: saveButtonLabel,
-                      onClick: onSave,
-                      type: "cta",
-                    }}
-                    order={1}
-                  />
-                {/if}
-              </BlockComponent>
             {/if}
           </BlockComponent>
         </BlockComponent>
@@ -245,6 +146,20 @@
         </BlockComponent>
       {/key}
     </BlockComponent>
+    {#if buttonPosition === "bottom"}
+      <BlockComponent
+        type="buttongroup"
+        props={{
+          buttons,
+        }}
+        styles={{
+          normal: {
+            "margin-top": "16",
+          },
+        }}
+        order={1}
+      />
+    {/if}
   </BlockComponent>
 {:else}
   <Placeholder

packages/client/src/components/preview/HoverIndicator.svelte

@@ -3,8 +3,7 @@
   import IndicatorSet from "./IndicatorSet.svelte"
   import { builderStore, dndIsDragging } from "stores"
 
-  let componentId
+  $: componentId = $builderStore.hoverComponentId
-
   $: zIndex = componentId === $builderStore.selectedComponentId ? 900 : 920
 
   const onMouseOver = e => {
@@ -24,12 +23,12 @@
     }
 
     if (newId !== componentId) {
-      componentId = newId
+      builderStore.actions.hoverComponent(newId)
     }
   }
 
   const onMouseLeave = () => {
-    componentId = null
+    builderStore.actions.hoverComponent(null)
   }
 
   onMount(() => {

packages/client/src/index.js

@@ -32,6 +32,7 @@ const loadBudibase = async () => {
     layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
     screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
     selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
+    hoverComponentId: window["##BUDIBASE_HOVER_COMPONENT_ID##"],
     previewId: window["##BUDIBASE_PREVIEW_ID##"],
     theme: window["##BUDIBASE_PREVIEW_THEME##"],
     customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"],

packages/client/src/stores/builder.js

@@ -8,6 +8,7 @@ const createBuilderStore = () => {
     inBuilder: false,
     screen: null,
     selectedComponentId: null,
+    hoverComponentId: null,
     editMode: false,
     previewId: null,
     theme: null,
@@ -23,6 +24,16 @@ const createBuilderStore = () => {
   }
   const store = writable(initialState)
   const actions = {
+    hoverComponent: id => {
+      if (id === get(store).hoverComponentId) {
+        return
+      }
+      store.update(state => ({
+        ...state,
+        hoverComponentId: id,
+      }))
+      eventStore.actions.dispatchEvent("hover-component", { id })
+    },
     selectComponent: id => {
       if (id === get(store).selectedComponentId) {
         return

packages/frontend-core/src/utils/utils.js

@@ -1,3 +1,6 @@
+import { makePropSafe as safe } from "@budibase/string-templates"
+import { Helpers } from "@budibase/bbui"
+
 /**
  * Utility to wrap an async function and ensure all invocations happen
  * sequentially.
@@ -106,3 +109,135 @@ export const domDebounce = callback => {
     }
   }
 }
+
+/**
+ * Build the default FormBlock button configs per actionType
+ * Parse any legacy button config and mirror its the outcome
+ *
+ * @param {any} props
+ * */
+export const buildDynamicButtonConfig = props => {
+  const {
+    _id,
+    actionType,
+    dataSource,
+    notificationOverride,
+    actionUrl,
+    showDeleteButton,
+    deleteButtonLabel,
+    showSaveButton,
+    saveButtonLabel,
+  } = props || {}
+
+  if (!_id) {
+    console.log("MISSING ID")
+    return
+  }
+  const formId = `${_id}-form`
+  const repeaterId = `${_id}-repeater`
+  const resourceId = dataSource?.resourceId
+
+  // Accommodate old config to ensure delete button does not reappear
+  const deleteText = showDeleteButton === false ? "" : deleteButtonLabel?.trim()
+  const saveText = showSaveButton === false ? "" : saveButtonLabel?.trim()
+
+  const onSave = [
+    {
+      "##eventHandlerType": "Validate Form",
+      parameters: {
+        componentId: formId,
+      },
+    },
+    {
+      "##eventHandlerType": "Save Row",
+      parameters: {
+        providerId: formId,
+        tableId: resourceId,
+        notificationOverride,
+      },
+    },
+    {
+      "##eventHandlerType": "Close Screen Modal",
+    },
+    {
+      "##eventHandlerType": "Close Side Panel",
+    },
+    // Clear a create form once submitted
+    ...(actionType !== "Create"
+      ? []
+      : [
+          {
+            "##eventHandlerType": "Clear Form",
+            parameters: {
+              componentId: formId,
+            },
+          },
+        ]),
+
+    ...(actionUrl
+      ? [
+          {
+            "##eventHandlerType": "Navigate To",
+            parameters: {
+              url: actionUrl,
+            },
+          },
+        ]
+      : []),
+  ]
+
+  const onDelete = [
+    {
+      "##eventHandlerType": "Delete Row",
+      parameters: {
+        confirm: true,
+        tableId: resourceId,
+        rowId: `{{ ${safe(repeaterId)}.${safe("_id")} }}`,
+        revId: `{{ ${safe(repeaterId)}.${safe("_rev")} }}`,
+        notificationOverride,
+      },
+    },
+    {
+      "##eventHandlerType": "Close Screen Modal",
+    },
+    {
+      "##eventHandlerType": "Close Side Panel",
+    },
+
+    ...(actionUrl
+      ? [
+          {
+            "##eventHandlerType": "Navigate To",
+            parameters: {
+              url: actionUrl,
+            },
+          },
+        ]
+      : []),
+  ]
+
+  const defaultButtons = []
+
+  if (["Update", "Create"].includes(actionType) && showSaveButton !== false) {
+    defaultButtons.push({
+      text: saveText || "Save",
+      _id: Helpers.uuid(),
+      _component: "@budibase/standard-components/button",
+      onClick: onSave,
+      type: "cta",
+    })
+  }
+
+  if (actionType == "Update" && showDeleteButton !== false) {
+    defaultButtons.push({
+      text: deleteText || "Delete",
+      _id: Helpers.uuid(),
+      _component: "@budibase/standard-components/button",
+      onClick: onDelete,
+      quiet: true,
+      type: "secondary",
+    })
+  }
+
+  return defaultButtons
+}

packages/server/src/api/controllers/query/index.ts

@@ -9,7 +9,7 @@ import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { QueryEvent } from "../../../threads/definitions"
-import { ConfigType, Query, UserCtx } from "@budibase/types"
+import { ConfigType, Query, UserCtx, SessionCookie } from "@budibase/types"
 import { ValidQueryNameRegex } from "@budibase/shared-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
@@ -113,7 +113,7 @@ function getOAuthConfigCookieId(ctx: UserCtx) {
 }
 
 function getAuthConfig(ctx: UserCtx) {
-  const authCookie = utils.getCookie(ctx, constants.Cookie.Auth)
+  const authCookie = utils.getCookie<SessionCookie>(ctx, constants.Cookie.Auth)
   let authConfigCtx: any = {}
   authConfigCtx["configId"] = getOAuthConfigCookieId(ctx)
   authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null

packages/server/src/api/controllers/row/internal.ts

@@ -2,7 +2,7 @@ import * as linkRows from "../../../db/linkedRows"
 import { generateRowID, InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import {
-  cleanupAttachments,
+  AttachmentCleanup,
   inputProcessing,
   outputProcessing,
 } from "../../../utilities/rowProcessor"
@@ -79,7 +79,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     table,
   })) as Row
   // check if any attachments removed
-  await cleanupAttachments(table, { oldRow, row })
+  await AttachmentCleanup.rowUpdate(table, { row, oldRow })
 
   if (isUserTable) {
     // the row has been updated, need to put it into the ctx
@@ -119,7 +119,7 @@ export async function save(ctx: UserCtx) {
     throw { validation: validateResult.errors }
   }
 
-  // make sure link rows are up to date
+  // make sure link rows are up-to-date
   row = (await linkRows.updateLinks({
     eventType: linkRows.EventType.ROW_SAVE,
     row,
@@ -165,7 +165,7 @@ export async function destroy(ctx: UserCtx) {
     tableId,
   })
   // remove any attachments that were on the row from object storage
-  await cleanupAttachments(table, { row })
+  await AttachmentCleanup.rowDelete(table, [row])
   // remove any static formula
   await updateRelatedFormula(table, row)
 
@@ -216,7 +216,7 @@ export async function bulkDestroy(ctx: UserCtx) {
     await db.bulkDocs(processedRows.map(row => ({ ...row, _deleted: true })))
   }
   // remove any attachments that were on the rows from object storage
-  await cleanupAttachments(table, { rows: processedRows })
+  await AttachmentCleanup.rowDelete(table, processedRows)
   await updateRelatedFormula(table, processedRows)
   await Promise.all(updates)
   return { response: { ok: true }, rows: processedRows }

packages/server/src/api/controllers/static/templates/preview.hbs

@@ -63,6 +63,7 @@
       // Extract data from message
       const {
         selectedComponentId,
+        hoverComponentId,
         layout,
         screen,
         appId,
@@ -81,6 +82,7 @@
       window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
       window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
       window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId
+      window["##BUDIBASE_HOVER_COMPONENT_ID##"] = hoverComponentId
       window["##BUDIBASE_PREVIEW_ID##"] = Math.random()
       window["##BUDIBASE_PREVIEW_THEME##"] = theme
       window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"] = customTheme
@@ -108,4 +110,4 @@
   </script>
 </head>
 <body></body>
-</html>
+</html>

packages/server/src/api/controllers/table/utils.ts

@@ -11,7 +11,7 @@ import {
 } from "../../../constants"
 import {
   inputProcessing,
-  cleanupAttachments,
+  AttachmentCleanup,
 } from "../../../utilities/rowProcessor"
 import { getViews, saveView } from "../view/utils"
 import viewTemplate from "../view/viewBuilder"
@@ -82,7 +82,10 @@ export async function checkForColumnUpdates(
     })
 
     // cleanup any attachments from object storage for deleted attachment columns
-    await cleanupAttachments(updatedTable, { oldTable, rows: rawRows })
+    await AttachmentCleanup.tableUpdate(updatedTable, rawRows, {
+      oldTable,
+      rename: columnRename,
+    })
     // Update views
     await checkForViewUpdates(updatedTable, deletedColumns, columnRename)
   }

packages/server/src/environment.ts

@@ -59,6 +59,7 @@ const environment = {
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+  MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/server/src/integrations/microsoftSqlServer.ts

@@ -17,7 +17,7 @@ import {
 import {
   getSqlQuery,
   buildExternalTableId,
-  convertSqlType,
+  generateColumnDefinition,
   finaliseExternalTables,
   SqlClient,
   checkExternalTables,
@@ -429,15 +429,12 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
         const hasDefault = def.COLUMN_DEFAULT
         const isAuto = !!autoColumns.find(col => col === name)
         const required = !!requiredColumns.find(col => col === name)
-        schema[name] = {
+        schema[name] = generateColumnDefinition({
           autocolumn: isAuto,
-          name: name,
+          name,
-          constraints: {
+          presence: required && !isAuto && !hasDefault,
-            presence: required && !isAuto && !hasDefault,
-          },
-          ...convertSqlType(def.DATA_TYPE),
           externalType: def.DATA_TYPE,
-        }
+        })
       }
       tables[tableName] = {
         _id: buildExternalTableId(datasourceId, tableName),

packages/server/src/integrations/mysql.ts

@@ -12,12 +12,13 @@ import {
   SourceName,
   Schema,
   TableSourceType,
+  FieldType,
 } from "@budibase/types"
 import {
   getSqlQuery,
   SqlClient,
   buildExternalTableId,
-  convertSqlType,
+  generateColumnDefinition,
   finaliseExternalTables,
   checkExternalTables,
 } from "./utils"
@@ -305,16 +306,17 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
             (column.Extra === "auto_increment" ||
               column.Extra.toLowerCase().includes("generated"))
           const required = column.Null !== "YES"
-          const constraints = {
+          schema[columnName] = generateColumnDefinition({
-            presence: required && !isAuto && !hasDefault,
-          }
-          schema[columnName] = {
             name: columnName,
             autocolumn: isAuto,
-            constraints,
+            presence: required && !isAuto && !hasDefault,
-            ...convertSqlType(column.Type),
             externalType: column.Type,
-          }
+            options: column.Type.startsWith("enum")
+              ? column.Type.substring(5, column.Type.length - 1)
+                  .split(",")
+                  .map(str => str.replace(/^'(.*)'$/, "$1"))
+              : undefined,
+          })
         }
         if (!tables[tableName]) {
           tables[tableName] = {

packages/server/src/integrations/oracle.ts

@@ -15,7 +15,7 @@ import {
 import {
   buildExternalTableId,
   checkExternalTables,
-  convertSqlType,
+  generateColumnDefinition,
   finaliseExternalTables,
   getSqlQuery,
   SqlClient,
@@ -250,14 +250,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     )
   }
 
-  private internalConvertType(column: OracleColumn) {
-    if (this.isBooleanType(column)) {
-      return { type: FieldTypes.BOOLEAN }
-    }
-
-    return convertSqlType(column.type)
-  }
-
   /**
    * Fetches the tables from the oracle table and assigns them to the datasource.
    * @param datasourceId - datasourceId to fetch
@@ -302,13 +294,15 @@ class OracleIntegration extends Sql implements DatasourcePlus {
           const columnName = oracleColumn.name
           let fieldSchema = table.schema[columnName]
           if (!fieldSchema) {
-            fieldSchema = {
+            fieldSchema = generateColumnDefinition({
               autocolumn: OracleIntegration.isAutoColumn(oracleColumn),
               name: columnName,
-              constraints: {
+              presence: false,
-                presence: false,
+              externalType: oracleColumn.type,
-              },
+            })
-              ...this.internalConvertType(oracleColumn),
+
+            if (this.isBooleanType(oracleColumn)) {
+              fieldSchema.type = FieldTypes.BOOLEAN
             }
 
             table.schema[columnName] = fieldSchema

packages/server/src/integrations/postgres.ts

@@ -16,7 +16,7 @@ import {
 import {
   getSqlQuery,
   buildExternalTableId,
-  convertSqlType,
+  generateColumnDefinition,
   finaliseExternalTables,
   SqlClient,
   checkExternalTables,
@@ -162,6 +162,14 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   WHERE pg_namespace.nspname = '${this.config.schema}';
   `
 
+  ENUM_VALUES = () => `
+  SELECT t.typname,  
+      e.enumlabel
+  FROM pg_type t 
+  JOIN pg_enum e on t.oid = e.enumtypid  
+  JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace;
+  `
+
   constructor(config: PostgresConfig) {
     super(SqlClient.POSTGRES)
     this.config = config
@@ -303,6 +311,18 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
 
       const tables: { [key: string]: Table } = {}
 
+      // Fetch enum values
+      const enumsResponse = await this.client.query(this.ENUM_VALUES())
+      const enumValues = enumsResponse.rows?.reduce((acc, row) => {
+        if (!acc[row.typname]) {
+          return {
+            [row.typname]: [row.enumlabel],
+          }
+        }
+        acc[row.typname].push(row.enumlabel)
+        return acc
+      }, {})
+
       for (let column of columnsResponse.rows) {
         const tableName: string = column.table_name
         const columnName: string = column.column_name
@@ -333,16 +353,13 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
           column.is_generated && column.is_generated !== "NEVER"
         const isAuto: boolean = hasNextVal || identity || isGenerated
         const required = column.is_nullable === "NO"
-        const constraints = {
+        tables[tableName].schema[columnName] = generateColumnDefinition({
-          presence: required && !hasDefault && !isGenerated,
-        }
-        tables[tableName].schema[columnName] = {
           autocolumn: isAuto,
           name: columnName,
-          constraints,
+          presence: required && !hasDefault && !isGenerated,
-          ...convertSqlType(column.data_type),
           externalType: column.data_type,
-        }
+          options: enumValues?.[column.udt_name],
+        })
       }
 
       let finalizedTables = finaliseExternalTables(tables, entities)

packages/server/src/integrations/utils.ts

@@ -67,6 +67,10 @@ const SQL_BOOLEAN_TYPE_MAP = {
   tinyint: FieldType.BOOLEAN,
 }
 
+const SQL_OPTIONS_TYPE_MAP = {
+  "user-defined": FieldType.OPTIONS,
+}
+
 const SQL_MISC_TYPE_MAP = {
   json: FieldType.JSON,
   bigint: FieldType.BIGINT,
@@ -78,6 +82,7 @@ const SQL_TYPE_MAP = {
   ...SQL_STRING_TYPE_MAP,
   ...SQL_BOOLEAN_TYPE_MAP,
   ...SQL_MISC_TYPE_MAP,
+  ...SQL_OPTIONS_TYPE_MAP,
 }
 
 export enum SqlClient {
@@ -178,25 +183,49 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
   }
 }
 
-export function convertSqlType(type: string) {
+export function generateColumnDefinition(config: {
+  externalType: string
+  autocolumn: boolean
+  name: string
+  presence: boolean
+  options?: string[]
+}) {
+  let { externalType, autocolumn, name, presence, options } = config
   let foundType = FieldType.STRING
-  const lcType = type.toLowerCase()
+  const lowerCaseType = externalType.toLowerCase()
   let matchingTypes = []
   for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
-    if (lcType.includes(external)) {
+    if (lowerCaseType.includes(external)) {
       matchingTypes.push({ external, internal })
     }
   }
-  //Set the foundType based the longest match
+  // Set the foundType based the longest match
   if (matchingTypes.length > 0) {
     foundType = matchingTypes.reduce((acc, val) => {
       return acc.external.length >= val.external.length ? acc : val
     }).internal
   }
-  const schema: any = { type: foundType }
+
+  const constraints: {
+    presence: boolean
+    inclusion?: string[]
+  } = {
+    presence,
+  }
+  if (foundType === FieldType.OPTIONS) {
+    constraints.inclusion = options
+  }
+
+  const schema: any = {
+    type: foundType,
+    externalType,
+    autocolumn,
+    name,
+    constraints,
+  }
   if (foundType === FieldType.DATETIME) {
-    schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType)
+    schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lowerCaseType)
-    schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType)
+    schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lowerCaseType)
   }
   return schema
 }

packages/server/src/koa.ts

@@ -1,5 +1,5 @@
 import env from "./environment"
-import Koa, { ExtendableContext } from "koa"
+import Koa from "koa"
 import koaBody from "koa-body"
 import http from "http"
 import * as api from "./api"
@@ -27,6 +27,9 @@ export default function createKoaApp() {
       // @ts-ignore
       enableTypes: ["json", "form", "text"],
       parsedMethods: ["POST", "PUT", "PATCH", "DELETE"],
+      formidable: {
+        maxFileSize: parseInt(env.MAX_IMPORT_SIZE_MB || "100") * 1024 * 1024,
+      },
     })
   )
 

packages/server/src/sdk/app/backups/constants.ts

@@ -1,3 +1,4 @@
 export const DB_EXPORT_FILE = "db.txt"
 export const GLOBAL_DB_EXPORT_FILE = "global.txt"
 export const STATIC_APP_FILES = ["manifest.json", "budibase-client.js"]
+export const ATTACHMENT_DIRECTORY = "attachments"

packages/server/src/sdk/app/backups/exports.ts

@@ -8,13 +8,15 @@ import {
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
 } from "../../../db/utils"
-import { DB_EXPORT_FILE, STATIC_APP_FILES } from "./constants"
+import {
+  DB_EXPORT_FILE,
+  STATIC_APP_FILES,
+  ATTACHMENT_DIRECTORY,
+} from "./constants"
 import fs from "fs"
 import { join } from "path"
 import env from "../../../environment"
-
+import { v4 as uuid } from "uuid"
-const uuid = require("uuid/v4")
-
 import tar from "tar"
 
 const MemoryStream = require("memorystream")
@@ -30,12 +32,11 @@ export interface ExportOpts extends DBDumpOpts {
   encryptPassword?: string
 }
 
-function tarFilesToTmp(tmpDir: string, files: string[]) {
+async function tarFilesToTmp(tmpDir: string, files: string[]) {
   const fileName = `${uuid()}.tar.gz`
   const exportFile = join(budibaseTempDir(), fileName)
-  tar.create(
+  await tar.create(
     {
-      sync: true,
       gzip: true,
       file: exportFile,
       noDirRecurse: false,
@@ -150,19 +151,21 @@ export async function exportApp(appId: string, config?: ExportOpts) {
     for (let file of fs.readdirSync(tmpPath)) {
       const path = join(tmpPath, file)
 
-      await encryption.encryptFile(
+      // skip the attachments - too big to encrypt
-        { dir: tmpPath, filename: file },
+      if (file !== ATTACHMENT_DIRECTORY) {
-        config.encryptPassword
+        await encryption.encryptFile(
-      )
+          { dir: tmpPath, filename: file },
-
+          config.encryptPassword
-      fs.rmSync(path)
+        )
+        fs.rmSync(path)
+      }
     }
   }
 
   // if tar requested, return where the tarball is
   if (config?.tar) {
     // now the tmpPath contains both the DB export and attachments, tar this
-    const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
+    const tarPath = await tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
     // cleanup the tmp export files as tarball returned
     fs.rmSync(tmpPath, { recursive: true, force: true })
 

packages/server/src/sdk/app/backups/imports.ts

@@ -6,17 +6,20 @@ import {
   AutomationTriggerStepId,
   RowAttachment,
 } from "@budibase/types"
-import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
+import { getAutomationParams } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
-import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import {
+  DB_EXPORT_FILE,
+  GLOBAL_DB_EXPORT_FILE,
+  ATTACHMENT_DIRECTORY,
+} from "./constants"
 import { downloadTemplate } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
-
+import { v4 as uuid } from "uuid"
-const uuid = require("uuid/v4")
+import tar from "tar"
-const tar = require("tar")
 
 type TemplateType = {
   file?: {
@@ -114,12 +117,11 @@ async function getTemplateStream(template: TemplateType) {
   }
 }
 
-export function untarFile(file: { path: string }) {
+export async function untarFile(file: { path: string }) {
   const tmpPath = join(budibaseTempDir(), uuid())
   fs.mkdirSync(tmpPath)
   // extract the tarball
-  tar.extract({
+  await tar.extract({
-    sync: true,
     cwd: tmpPath,
     file: file.path,
   })
@@ -130,9 +132,11 @@ async function decryptFiles(path: string, password: string) {
   try {
     for (let file of fs.readdirSync(path)) {
       const inputPath = join(path, file)
-      const outputPath = inputPath.replace(/\.enc$/, "")
+      if (!inputPath.endsWith(ATTACHMENT_DIRECTORY)) {
-      await encryption.decryptFile(inputPath, outputPath, password)
+        const outputPath = inputPath.replace(/\.enc$/, "")
-      fs.rmSync(inputPath)
+        await encryption.decryptFile(inputPath, outputPath, password)
+        fs.rmSync(inputPath)
+      }
     }
   } catch (err: any) {
     if (err.message === "incorrect header check") {
@@ -162,7 +166,7 @@ export async function importApp(
   const isDirectory =
     template.file && fs.lstatSync(template.file.path).isDirectory()
   if (template.file && (isTar || isDirectory)) {
-    const tmpPath = isTar ? untarFile(template.file) : template.file.path
+    const tmpPath = isTar ? await untarFile(template.file) : template.file.path
     if (isTar && template.file.password) {
       await decryptFiles(tmpPath, template.file.password)
     }

packages/server/src/sdk/app/tables/internal/index.ts

@@ -19,11 +19,10 @@ import { context } from "@budibase/backend-core"
 import { getTable } from "../getters"
 import { checkAutoColumns } from "./utils"
 import * as viewsSdk from "../../views"
-import sdk from "../../../index"
 import { getRowParams } from "../../../../db/utils"
 import { quotas } from "@budibase/pro"
 import env from "../../../../environment"
-import { cleanupAttachments } from "../../../../utilities/rowProcessor"
+import { AttachmentCleanup } from "../../../../utilities/rowProcessor"
 
 export async function save(
   table: Table,
@@ -164,9 +163,10 @@ export async function destroy(table: Table) {
   await runStaticFormulaChecks(table, {
     deletion: true,
   })
-  await cleanupAttachments(table, {
+  await AttachmentCleanup.tableDelete(
-    rows: rowsData.rows.map((row: any) => row.doc),
+    table,
-  })
+    rowsData.rows.map((row: any) => row.doc)
+  )
 
   return { table }
 }

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -56,6 +56,7 @@ import {
 
 import API from "./api"
 import { cloneDeep } from "lodash"
+import jwt, { Secret } from "jsonwebtoken"
 
 mocks.licenses.init(pro)
 
@@ -391,7 +392,7 @@ class TestConfiguration {
         sessionId: "sessionid",
         tenantId: this.getTenantId(),
       }
-      const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET)
+      const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
 
       // returning necessary request headers
       await cache.user.invalidateUser(userId)
@@ -412,7 +413,7 @@ class TestConfiguration {
       sessionId: "sessionid",
       tenantId,
     }
-    const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET)
+    const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
 
     const headers: any = {
       Accept: "application/json",

packages/server/src/utilities/rowProcessor/attachments.ts

@@ -0,0 +1,99 @@
+import { FieldTypes, ObjectStoreBuckets } from "../../constants"
+import { context, db as dbCore, objectStore } from "@budibase/backend-core"
+import { RenameColumn, Row, RowAttachment, Table } from "@budibase/types"
+
+export class AttachmentCleanup {
+  static async coreCleanup(fileListFn: () => string[]): Promise<void> {
+    const appId = context.getAppId()
+    if (!dbCore.isProdAppID(appId)) {
+      const prodAppId = dbCore.getProdAppID(appId!)
+      // if prod exists, then don't allow deleting
+      const exists = await dbCore.dbExists(prodAppId)
+      if (exists) {
+        return
+      }
+    }
+    const files = fileListFn()
+    if (files.length > 0) {
+      await objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
+    }
+  }
+
+  private static async tableChange(
+    table: Table,
+    rows: Row[],
+    opts: { oldTable?: Table; rename?: RenameColumn; deleting?: boolean }
+  ) {
+    return AttachmentCleanup.coreCleanup(() => {
+      let files: string[] = []
+      const tableSchema = opts.oldTable?.schema || table.schema
+      for (let [key, schema] of Object.entries(tableSchema)) {
+        if (schema.type !== FieldTypes.ATTACHMENT) {
+          continue
+        }
+        const columnRemoved = opts.oldTable && !table.schema[key]
+        const renaming = opts.rename?.old === key
+        // old table had this column, new table doesn't - delete it
+        if ((columnRemoved && !renaming) || opts.deleting) {
+          rows.forEach(row => {
+            files = files.concat(
+              row[key].map((attachment: any) => attachment.key)
+            )
+          })
+        }
+      }
+      return files
+    })
+  }
+
+  static async tableDelete(table: Table, rows: Row[]) {
+    return AttachmentCleanup.tableChange(table, rows, { deleting: true })
+  }
+
+  static async tableUpdate(
+    table: Table,
+    rows: Row[],
+    opts: { oldTable?: Table; rename?: RenameColumn }
+  ) {
+    return AttachmentCleanup.tableChange(table, rows, opts)
+  }
+
+  static async rowDelete(table: Table, rows: Row[]) {
+    return AttachmentCleanup.coreCleanup(() => {
+      let files: string[] = []
+      for (let [key, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldTypes.ATTACHMENT) {
+          continue
+        }
+        rows.forEach(row => {
+          files = files.concat(
+            row[key].map((attachment: any) => attachment.key)
+          )
+        })
+      }
+      return files
+    })
+  }
+
+  static rowUpdate(table: Table, opts: { row: Row; oldRow: Row }) {
+    return AttachmentCleanup.coreCleanup(() => {
+      let files: string[] = []
+      for (let [key, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldTypes.ATTACHMENT) {
+          continue
+        }
+        const oldKeys =
+          opts.oldRow[key]?.map(
+            (attachment: RowAttachment) => attachment.key
+          ) || []
+        const newKeys =
+          opts.row[key]?.map((attachment: RowAttachment) => attachment.key) ||
+          []
+        files = files.concat(
+          oldKeys.filter((key: string) => newKeys.indexOf(key) === -1)
+        )
+      }
+      return files
+    })
+  }
+}

packages/server/src/utilities/rowProcessor/index.ts

@@ -1,16 +1,7 @@
 import * as linkRows from "../../db/linkedRows"
-import {
+import { FieldTypes, AutoFieldSubTypes } from "../../constants"
-  FieldTypes,
-  AutoFieldSubTypes,
-  ObjectStoreBuckets,
-} from "../../constants"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
-import {
+import { objectStore, utils } from "@budibase/backend-core"
-  context,
-  db as dbCore,
-  objectStore,
-  utils,
-} from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
 import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
@@ -22,6 +13,7 @@ import {
 import { isExternalTableID } from "../../integrations/utils"
 
 export * from "./utils"
+export * from "./attachments"
 
 type AutoColumnProcessingOpts = {
   reprocessing?: boolean
@@ -30,27 +22,6 @@ type AutoColumnProcessingOpts = {
 
 const BASE_AUTO_ID = 1
 
-/**
- * Given the old state of the row and the new one after an update, this will
- * find the keys that have been removed in the updated row.
- */
-function getRemovedAttachmentKeys(
-  oldRow: Row,
-  row: Row,
-  attachmentKey: string
-) {
-  if (!oldRow[attachmentKey]) {
-    return []
-  }
-  const oldKeys = oldRow[attachmentKey].map((attachment: any) => attachment.key)
-  // no attachments in new row, all removed
-  if (!row[attachmentKey]) {
-    return oldKeys
-  }
-  const newKeys = row[attachmentKey].map((attachment: any) => attachment.key)
-  return oldKeys.filter((key: string) => newKeys.indexOf(key) === -1)
-}
-
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
@@ -249,7 +220,9 @@ export async function outputProcessing<T extends Row[] | Row>(
           continue
         }
         row[property].forEach((attachment: RowAttachment) => {
-          attachment.url ??= objectStore.getAppFileUrl(attachment.key)
+          if (!attachment.url) {
+            attachment.url = objectStore.getAppFileUrl(attachment.key)
+          }
         })
       }
     } else if (
@@ -286,59 +259,3 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
   return (wasArray ? enriched : enriched[0]) as T
 }
-
-/**
- * Clean up any attachments that were attached to a row.
- * @param table The table from which a row is being removed.
- * @param row optional - the row being removed.
- * @param rows optional - if multiple rows being deleted can do this in bulk.
- * @param oldRow optional - if updating a row this will determine the difference.
- * @param oldTable optional - if updating a table, can supply the old table to look for
- * deleted attachment columns.
- * @return When all attachments have been removed this will return.
- */
-export async function cleanupAttachments(
-  table: Table,
-  {
-    row,
-    rows,
-    oldRow,
-    oldTable,
-  }: { row?: Row; rows?: Row[]; oldRow?: Row; oldTable?: Table }
-): Promise<any> {
-  const appId = context.getAppId()
-  if (!dbCore.isProdAppID(appId)) {
-    const prodAppId = dbCore.getProdAppID(appId!)
-    // if prod exists, then don't allow deleting
-    const exists = await dbCore.dbExists(prodAppId)
-    if (exists) {
-      return
-    }
-  }
-  let files: string[] = []
-  function addFiles(row: Row, key: string) {
-    if (row[key]) {
-      files = files.concat(row[key].map((attachment: any) => attachment.key))
-    }
-  }
-  const schemaToUse = oldTable ? oldTable.schema : table.schema
-  for (let [key, schema] of Object.entries(schemaToUse)) {
-    if (schema.type !== FieldTypes.ATTACHMENT) {
-      continue
-    }
-    // old table had this column, new table doesn't - delete it
-    if (rows && oldTable && !table.schema[key]) {
-      rows.forEach(row => addFiles(row, key))
-    } else if (oldRow && row) {
-      // if updating, need to manage the differences
-      files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
-    } else if (row) {
-      addFiles(row, key)
-    } else if (rows) {
-      rows.forEach(row => addFiles(row, key))
-    }
-  }
-  if (files.length > 0) {
-    await objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
-  }
-}

packages/server/src/utilities/rowProcessor/tests/attachments.spec.ts

@@ -0,0 +1,110 @@
+import { AttachmentCleanup } from "../attachments"
+import { FieldType, Table, Row, TableSourceType } from "@budibase/types"
+import { DEFAULT_BB_DATASOURCE_ID } from "../../../constants"
+import { objectStore } from "@budibase/backend-core"
+
+const BUCKET = "prod-budi-app-assets"
+const FILE_NAME = "file/thing.jpg"
+
+jest.mock("@budibase/backend-core", () => {
+  const actual = jest.requireActual("@budibase/backend-core")
+  return {
+    ...actual,
+    objectStore: {
+      deleteFiles: jest.fn(),
+      ObjectStoreBuckets: actual.objectStore.ObjectStoreBuckets,
+    },
+    db: {
+      isProdAppID: () => jest.fn(() => false),
+      dbExists: () => jest.fn(() => false),
+    },
+  }
+})
+
+const mockedDeleteFiles = objectStore.deleteFiles as jest.MockedFunction<
+  typeof objectStore.deleteFiles
+>
+
+function table(): Table {
+  return {
+    name: "table",
+    sourceId: DEFAULT_BB_DATASOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
+    type: "table",
+    schema: {
+      attach: {
+        name: "attach",
+        type: FieldType.ATTACHMENT,
+        constraints: {},
+      },
+    },
+  }
+}
+
+function row(fileKey: string = FILE_NAME): Row {
+  return {
+    attach: [
+      {
+        size: 1,
+        extension: "jpg",
+        key: fileKey,
+      },
+    ],
+  }
+}
+
+describe("attachment cleanup", () => {
+  beforeEach(() => {
+    mockedDeleteFiles.mockClear()
+  })
+
+  it("should be able to cleanup a table update", async () => {
+    const originalTable = table()
+    delete originalTable.schema["attach"]
+    await AttachmentCleanup.tableUpdate(originalTable, [row()], {
+      oldTable: table(),
+    })
+    expect(mockedDeleteFiles).toBeCalledWith(BUCKET, [FILE_NAME])
+  })
+
+  it("should be able to cleanup a table deletion", async () => {
+    await AttachmentCleanup.tableDelete(table(), [row()])
+    expect(mockedDeleteFiles).toBeCalledWith(BUCKET, [FILE_NAME])
+  })
+
+  it("should handle table column renaming", async () => {
+    const updatedTable = table()
+    updatedTable.schema.attach2 = updatedTable.schema.attach
+    delete updatedTable.schema.attach
+    await AttachmentCleanup.tableUpdate(updatedTable, [row()], {
+      oldTable: table(),
+      rename: { old: "attach", updated: "attach2" },
+    })
+    expect(mockedDeleteFiles).not.toBeCalled()
+  })
+
+  it("shouldn't cleanup if no table changes", async () => {
+    await AttachmentCleanup.tableUpdate(table(), [row()], { oldTable: table() })
+    expect(mockedDeleteFiles).not.toBeCalled()
+  })
+
+  it("should handle row updates", async () => {
+    const updatedRow = row()
+    delete updatedRow.attach
+    await AttachmentCleanup.rowUpdate(table(), {
+      row: updatedRow,
+      oldRow: row(),
+    })
+    expect(mockedDeleteFiles).toBeCalledWith(BUCKET, [FILE_NAME])
+  })
+
+  it("should handle row deletion", async () => {
+    await AttachmentCleanup.rowDelete(table(), [row()])
+    expect(mockedDeleteFiles).toBeCalledWith(BUCKET, [FILE_NAME])
+  })
+
+  it("shouldn't cleanup attachments if row not updated", async () => {
+    await AttachmentCleanup.rowUpdate(table(), { row: row(), oldRow: row() })
+    expect(mockedDeleteFiles).not.toBeCalled()
+  })
+})

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -3,6 +3,7 @@ import {
   FieldType,
   FieldTypeSubtypes,
   INTERNAL_TABLE_SOURCE_ID,
+  RowAttachment,
   Table,
   TableSourceType,
 } from "@budibase/types"
@@ -70,6 +71,49 @@ describe("rowProcessor - outputProcessing", () => {
     )
   })
 
+  it("should handle attachments correctly", async () => {
+    const table: Table = {
+      _id: generator.guid(),
+      name: "TestTable",
+      type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+      schema: {
+        attach: {
+          type: FieldType.ATTACHMENT,
+          name: "attach",
+          constraints: {},
+        },
+      },
+    }
+
+    const row: { attach: RowAttachment[] } = {
+      attach: [
+        {
+          size: 10,
+          name: "test",
+          extension: "jpg",
+          key: "test.jpg",
+        },
+      ],
+    }
+
+    const output = await outputProcessing(table, row, { squash: false })
+    expect(output.attach[0].url).toBe(
+      "/files/signed/prod-budi-app-assets/test.jpg"
+    )
+
+    row.attach[0].url = ""
+    const output2 = await outputProcessing(table, row, { squash: false })
+    expect(output2.attach[0].url).toBe(
+      "/files/signed/prod-budi-app-assets/test.jpg"
+    )
+
+    row.attach[0].url = "aaaa"
+    const output3 = await outputProcessing(table, row, { squash: false })
+    expect(output3.attach[0].url).toBe("aaaa")
+  })
+
   it("process output even when the field is not empty", async () => {
     const table: Table = {
       _id: generator.guid(),

packages/types/src/api/web/cookies.ts

@@ -0,0 +1,9 @@
+export interface DatasourceAuthCookie {
+  appId: string
+  provider: string
+}
+
+export interface SessionCookie {
+  sessionId: string
+  userId: string
+}

packages/types/src/api/web/index.ts

@@ -9,3 +9,4 @@ export * from "./app"
 export * from "./global"
 export * from "./pagination"
 export * from "./searchFilter"
+export * from "./cookies"

packages/worker/src/api/controllers/global/auth.ts

@@ -15,6 +15,7 @@ import {
   PasswordResetRequest,
   PasswordResetUpdateRequest,
   GoogleInnerConfig,
+  DatasourceAuthCookie,
 } from "@budibase/types"
 import env from "../../../environment"
 
@@ -148,7 +149,13 @@ export const datasourcePreAuth = async (ctx: any, next: any) => {
 }
 
 export const datasourceAuth = async (ctx: any, next: any) => {
-  const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth)
+  const authStateCookie = getCookie<DatasourceAuthCookie>(
+    ctx,
+    Cookie.DatasourceAuth
+  )
+  if (!authStateCookie) {
+    throw new Error("Unable to retrieve datasource authentication cookie")
+  }
   const provider = authStateCookie.provider
   const { middleware } = require(`@budibase/backend-core`)
   const handler = middleware.datasource[provider]

packages/worker/src/tests/TestConfiguration.ts

@@ -35,6 +35,7 @@ import {
   ConfigType,
 } from "@budibase/types"
 import API from "./api"
+import jwt, { Secret } from "jsonwebtoken"
 
 class TestConfiguration {
   server: any
@@ -209,7 +210,7 @@ class TestConfiguration {
       sessionId: "sessionid",
       tenantId: user.tenantId,
     }
-    const authCookie = auth.jwt.sign(authToken, coreEnv.JWT_SECRET)
+    const authCookie = jwt.sign(authToken, coreEnv.JWT_SECRET as Secret)
     return {
       Accept: "application/json",
       ...this.cookieHeader([`${constants.Cookie.Auth}=${authCookie}`]),
@@ -327,7 +328,7 @@ class TestConfiguration {
   // CONFIGS - OIDC
 
   getOIDConfigCookie(configId: string) {
-    const token = auth.jwt.sign(configId, coreEnv.JWT_SECRET)
+    const token = jwt.sign(configId, coreEnv.JWT_SECRET as Secret)
     return this.cookieHeader([[`${constants.Cookie.OIDC_CONFIG}=${token}`]])
   }