Merge branch 'master' into dependabot/npm_and_yarn/ejs-3.1.10

.eslintrc.json

@@ -1,4 +1,5 @@
 {
+  "root": true,
   "env": {
     "browser": true,
     "es6": true,

.prettierignore

@@ -12,4 +12,5 @@ packages/pro/coverage
 packages/account-portal/packages/ui/build
 packages/account-portal/packages/ui/.routify
 packages/account-portal/packages/server/build
+packages/account-portal/packages/server/coverage
 **/*.ivm.bundle.js

hosting/couchdb/runner.v2.sh

@@ -70,10 +70,10 @@ sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouse
 /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
 
 # Start CouchDB.
-/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 &
 
-# Start SQS.
+# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues.
-/opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 &
+/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 &
 
 # Wait for CouchDB to start up.
 while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.24.0",
+  "version": "2.24.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/bbui/src/Actions/click_outside.js

@@ -1,22 +1,25 @@
+// These class names will never trigger a callback if clicked, no matter what
 const ignoredClasses = [
   ".download-js-link",
   ".spectrum-Menu",
   ".date-time-popover",
 ]
+
+// These class names will only trigger a callback when clicked if the registered
+// component is not nested inside them. For example, clicking inside a modal
+// will not close the modal, or clicking inside a popover will not close the
+// popover.
 const conditionallyIgnoredClasses = [
   ".spectrum-Underlay",
   ".drawer-wrapper",
   ".spectrum-Popover",
 ]
 let clickHandlers = []
+let candidateTarget
 
-/**
+// Processes a "click outside" event and invokes callbacks if our source element
- * Handle a body click event
+// is valid
- */
 const handleClick = event => {
-  // Treat right clicks (context menu events) as normal clicks
-  const eventType = event.type === "contextmenu" ? "click" : event.type
-
   // Ignore click if this is an ignored class
   if (event.target.closest('[data-ignore-click-outside="true"]')) {
     return
@@ -29,11 +32,6 @@ const handleClick = event => {
 
   // Process handlers
   clickHandlers.forEach(handler => {
-    // Check that we're the right kind of click event
-    if (handler.allowedType && eventType !== handler.allowedType) {
-      return
-    }
-
     // Check that the click isn't inside the target
     if (handler.element.contains(event.target)) {
       return
@@ -51,17 +49,43 @@ const handleClick = event => {
     handler.callback?.(event)
   })
 }
-document.documentElement.addEventListener("click", handleClick, true)
+
-document.documentElement.addEventListener("mousedown", handleClick, true)
+// On mouse up we only trigger a "click outside" callback if we targetted the
-document.documentElement.addEventListener("contextmenu", handleClick, true)
+// same element that we did on mouse down. This fixes all sorts of issues where
+// we get annoying callbacks firing when we drag to select text.
+const handleMouseUp = e => {
+  if (candidateTarget === e.target) {
+    handleClick(e)
+  }
+  candidateTarget = null
+}
+
+// On mouse down we store which element was targetted for comparison later
+const handleMouseDown = e => {
+  // Only handle the primary mouse button here.
+  // We handle context menu (right click) events in another handler.
+  if (e.button !== 0) {
+    return
+  }
+  candidateTarget = e.target
+
+  // Clear any previous listeners in case of multiple down events, and register
+  // a single mouse up listener
+  document.removeEventListener("mouseup", handleMouseUp)
+  document.addEventListener("mouseup", handleMouseUp, true)
+}
+
+// Global singleton listeners for our events
+document.addEventListener("mousedown", handleMouseDown)
+document.addEventListener("contextmenu", handleClick)
 
 /**
  * Adds or updates a click handler
  */
-const updateHandler = (id, element, anchor, callback, allowedType) => {
+const updateHandler = (id, element, anchor, callback) => {
   let existingHandler = clickHandlers.find(x => x.id === id)
   if (!existingHandler) {
-    clickHandlers.push({ id, element, anchor, callback, allowedType })
+    clickHandlers.push({ id, element, anchor, callback })
   } else {
     existingHandler.callback = callback
   }
@@ -88,8 +112,7 @@ export default (element, opts) => {
     const callback =
       newOpts?.callback || (typeof newOpts === "function" ? newOpts : null)
     const anchor = newOpts?.anchor || element
-    const allowedType = newOpts?.allowedType || "click"
+    updateHandler(id, element, anchor, callback)
-    updateHandler(id, element, anchor, callback, allowedType)
   }
   update(opts)
   return {

packages/bbui/src/Form/Core/DatePicker/Calendar.svelte

@@ -4,6 +4,9 @@
   import dayjs from "dayjs"
   import NumberInput from "./NumberInput.svelte"
   import { createEventDispatcher } from "svelte"
+  import isoWeek from "dayjs/plugin/isoWeek"
+
+  dayjs.extend(isoWeek)
 
   export let value
 
@@ -43,7 +46,7 @@
       return []
     }
     let monthEnd = monthStart.endOf("month")
-    let calendarStart = monthStart.startOf("week")
+    let calendarStart = monthStart.startOf("isoWeek")
     const numWeeks = Math.ceil((monthEnd.diff(calendarStart, "day") + 1) / 7)
 
     let mondays = []

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/PromptUser.svelte

@@ -1,8 +1,10 @@
 <script>
-  import { Body, Label, Input } from "@budibase/bbui"
+  import { Body, Label } from "@budibase/bbui"
   import { onMount } from "svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let parameters
+  export let bindings
 
   onMount(() => {
     if (!parameters.confirm) {
@@ -15,11 +17,18 @@
   <Body size="S">Enter the message you wish to display to the user.</Body>
   <div class="params">
     <Label small>Title</Label>
-    <Input placeholder="Prompt User" bind:value={parameters.customTitleText} />
+    <DrawerBindableInput
+      placeholder="Title"
+      value={parameters.customTitleText}
+      on:change={e => (parameters.customTitleText = e.detail)}
+      {bindings}
+    />
     <Label small>Message</Label>
-    <Input
+    <DrawerBindableInput
       placeholder="Are you sure you want to continue?"
-      bind:value={parameters.confirmText}
+      value={parameters.confirmText}
+      on:change={e => (parameters.confirmText = e.detail)}
+      {bindings}
     />
   </div>
 </div>

packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte

@@ -21,26 +21,24 @@
   const currentStep = derived(multiStepStore, state => state.currentStep)
   const componentType = "@budibase/standard-components/multistepformblockstep"
 
+  setContext("multi-step-form-block", multiStepStore)
+
   let cachedValue
   let cachedInstance = {}
 
   $: if (!isEqual(cachedValue, value)) {
     cachedValue = value
   }
-
   $: if (!isEqual(componentInstance, cachedInstance)) {
     cachedInstance = componentInstance
   }
-
-  setContext("multi-step-form-block", multiStepStore)
-
   $: stepCount = cachedValue?.length || 0
   $: updateStore(stepCount)
   $: dataSource = getDatasourceForProvider($selectedScreen, cachedInstance)
   $: emitCurrentStep($currentStep)
   $: stepLabel = getStepLabel($multiStepStore)
   $: stepDef = getDefinition(stepLabel)
-  $: stepSettings = cachedValue?.[$currentStep] || {}
+  $: savedInstance = cachedValue?.[$currentStep] || {}
   $: defaults = Utils.buildMultiStepFormBlockDefaultProps({
     _id: cachedInstance._id,
     stepCount: $multiStepStore.stepCount,
@@ -48,14 +46,16 @@
     actionType: cachedInstance.actionType,
     dataSource: cachedInstance.dataSource,
   })
+  // For backwards compatibility we need to sometimes manually set base
+  // properties like _id and _component as we didn't used to save these
   $: stepInstance = {
-    _id: Helpers.uuid(),
+    _id: savedInstance._id || Helpers.uuid(),
-    _component: componentType,
+    _component: savedInstance._component || componentType,
     _instanceName: `Step ${currentStep + 1}`,
-    title: stepSettings.title ?? defaults?.title,
+    title: savedInstance.title ?? defaults?.title,
-    buttons: stepSettings.buttons || defaults?.buttons,
+    buttons: savedInstance.buttons || defaults?.buttons,
-    fields: stepSettings.fields,
+    fields: savedInstance.fields,
-    desc: stepSettings.desc,
+    desc: savedInstance.desc,
 
     // Needed for field configuration
     dataSource,
@@ -92,7 +92,8 @@
   }
 
   const addStep = () => {
-    value = value.toSpliced($currentStep + 1, 0, {})
+    const newInstance = componentStore.createInstance(componentType)
+    value = value.toSpliced($currentStep + 1, 0, newInstance)
     dispatch("change", value)
     multiStepStore.update(state => ({
       ...state,

packages/builder/src/components/design/settings/controls/FormStepControls.svelte

@@ -1,6 +1,6 @@
 <script>
   import { createEventDispatcher, getContext } from "svelte"
-  import { ActionButton } from "@budibase/bbui"
+  import { ActionButton, AbsTooltip } from "@budibase/bbui"
 
   const multiStepStore = getContext("multi-step-form-block")
   const dispatch = createEventDispatcher()
@@ -28,6 +28,7 @@
   </div>
 {:else}
   <div class="step-actions">
+    <AbsTooltip text="Previous step" noWrap>
       <ActionButton
         size="S"
         secondary
@@ -36,8 +37,9 @@
         on:click={() => {
           stepAction("previousStep")
         }}
-      tooltip={"Previous step"}
       />
+    </AbsTooltip>
+    <AbsTooltip text="Next step" noWrap>
       <ActionButton
         size="S"
         secondary
@@ -46,8 +48,9 @@
         on:click={() => {
           stepAction("nextStep")
         }}
-      tooltip={"Next step"}
       />
+    </AbsTooltip>
+    <AbsTooltip text="Remove step" noWrap>
       <ActionButton
         size="S"
         secondary
@@ -56,8 +59,9 @@
         on:click={() => {
           stepAction("removeStep")
         }}
-      tooltip={"Remove step"}
       />
+    </AbsTooltip>
+    <AbsTooltip text="Add step" noWrap>
       <ActionButton
         size="S"
         secondary
@@ -65,8 +69,8 @@
         on:click={() => {
           stepAction("addStep")
         }}
-      tooltip={"Add step"}
       />
+    </AbsTooltip>
   </div>
 {/if}
 

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js

@@ -75,6 +75,7 @@ const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => {
     return createComponent(
       "@budibase/standard-components/labelfield",
       {
+        _id: column.field,
         _instanceName: column.field,
         active: column.active,
         field: column.field,

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.test.js

@@ -65,6 +65,7 @@ describe("getColumns", () => {
     it("returns the selected and unselected fields in the modern format, respecting the original order", ctx => {
       expect(ctx.columns.sortable).toEqual([
         {
+          _id: "three",
           _instanceName: "three",
           active: true,
           columnType: "foo",
@@ -73,6 +74,7 @@ describe("getColumns", () => {
           label: "three label",
         },
         {
+          _id: "two",
           _instanceName: "two",
           active: true,
           columnType: "foo",
@@ -81,6 +83,7 @@ describe("getColumns", () => {
           label: "two label",
         },
         {
+          _id: "one",
           _instanceName: "one",
           active: false,
           columnType: "foo",
@@ -91,6 +94,7 @@ describe("getColumns", () => {
       ])
 
       expect(ctx.columns.primary).toEqual({
+        _id: "four",
         _instanceName: "four",
         active: true,
         columnType: "foo",
@@ -115,6 +119,7 @@ describe("getColumns", () => {
     it("returns all columns, with non-hidden columns automatically selected", ctx => {
       expect(ctx.columns.sortable).toEqual([
         {
+          _id: "two",
           _instanceName: "two",
           active: true,
           columnType: "foo",
@@ -123,6 +128,7 @@ describe("getColumns", () => {
           label: "two",
         },
         {
+          _id: "three",
           _instanceName: "three",
           active: true,
           columnType: "foo",
@@ -131,6 +137,7 @@ describe("getColumns", () => {
           label: "three",
         },
         {
+          _id: "one",
           _instanceName: "one",
           active: false,
           columnType: "foo",
@@ -141,6 +148,7 @@ describe("getColumns", () => {
       ])
 
       expect(ctx.columns.primary).toEqual({
+        _id: "four",
         _instanceName: "four",
         active: true,
         columnType: "foo",
@@ -173,6 +181,7 @@ describe("getColumns", () => {
     it("returns all columns, including those missing from the initial data", ctx => {
       expect(ctx.columns.sortable).toEqual([
         {
+          _id: "three",
           _instanceName: "three",
           active: true,
           columnType: "foo",
@@ -181,6 +190,7 @@ describe("getColumns", () => {
           label: "three label",
         },
         {
+          _id: "two",
           _instanceName: "two",
           active: false,
           columnType: "foo",
@@ -189,6 +199,7 @@ describe("getColumns", () => {
           label: "two",
         },
         {
+          _id: "one",
           _instanceName: "one",
           active: false,
           columnType: "foo",
@@ -199,6 +210,7 @@ describe("getColumns", () => {
       ])
 
       expect(ctx.columns.primary).toEqual({
+        _id: "four",
         _instanceName: "four",
         active: true,
         columnType: "foo",
@@ -228,6 +240,7 @@ describe("getColumns", () => {
     it("returns all valid columns, excluding those that aren't valid for the schema", ctx => {
       expect(ctx.columns.sortable).toEqual([
         {
+          _id: "three",
           _instanceName: "three",
           active: true,
           columnType: "foo",
@@ -236,6 +249,7 @@ describe("getColumns", () => {
           label: "three label",
         },
         {
+          _id: "two",
           _instanceName: "two",
           active: false,
           columnType: "foo",
@@ -244,6 +258,7 @@ describe("getColumns", () => {
           label: "two",
         },
         {
+          _id: "one",
           _instanceName: "one",
           active: false,
           columnType: "foo",
@@ -254,6 +269,7 @@ describe("getColumns", () => {
       ])
 
       expect(ctx.columns.primary).toEqual({
+        _id: "four",
         _instanceName: "four",
         active: true,
         columnType: "foo",
@@ -318,6 +334,7 @@ describe("getColumns", () => {
       beforeEach(ctx => {
         ctx.updateSortable([
           {
+            _id: "three",
             _instanceName: "three",
             active: true,
             columnType: "foo",
@@ -326,6 +343,7 @@ describe("getColumns", () => {
             label: "three",
           },
           {
+            _id: "one",
             _instanceName: "one",
             active: true,
             columnType: "foo",
@@ -334,6 +352,7 @@ describe("getColumns", () => {
             label: "one",
           },
           {
+            _id: "two",
             _instanceName: "two",
             active: false,
             columnType: "foo",

packages/builder/src/dataBinding.js

@@ -1106,50 +1106,51 @@ export const getAllStateVariables = () => {
   getAllAssets().forEach(asset => {
     findAllMatchingComponents(asset.props, component => {
       const settings = componentStore.getComponentSettings(component._component)
+      const nestedTypes = [
+        "buttonConfiguration",
+        "fieldConfiguration",
+        "stepConfiguration",
+      ]
 
+      // Extracts all event settings from a component instance.
+      // Recurses into nested types to find all event-like settings at any
+      // depth.
       const parseEventSettings = (settings, comp) => {
+        if (!settings?.length) {
+          return
+        }
+
+        // Extract top level event settings
         settings
           .filter(setting => setting.type === "event")
           .forEach(setting => {
             eventSettings.push(comp[setting.key])
           })
-      }
 
-      const parseComponentSettings = (settings, component) => {
+        // Recurse into any nested instance types
-        // Parse the nested button configurations
         settings
-          .filter(setting => setting.type === "buttonConfiguration")
+          .filter(setting => nestedTypes.includes(setting.type))
           .forEach(setting => {
-            const buttonConfig = component[setting.key]
+            const instances = comp[setting.key]
+            if (Array.isArray(instances) && instances.length) {
+              instances.forEach(instance => {
+                let type = instance?._component
 
-            if (Array.isArray(buttonConfig)) {
+                // Backwards compatibility for multi-step from blocks which
-              buttonConfig.forEach(button => {
+                // didn't set a proper component type previously.
-                const nestedSettings = componentStore.getComponentSettings(
+                if (setting.type === "stepConfiguration" && !type) {
-                  button._component
+                  type = "@budibase/standard-components/multistepformblockstep"
-                )
+                }
-                parseEventSettings(nestedSettings, button)
+
+                // Parsed nested component instances inside this setting
+                const nestedSettings = componentStore.getComponentSettings(type)
+                parseEventSettings(nestedSettings, instance)
               })
             }
           })
+      }
 
       parseEventSettings(settings, component)
-      }
-
-      // Parse the base component settings
-      parseComponentSettings(settings, component)
-
-      // Parse step configuration
-      const stepSetting = settings.find(
-        setting => setting.type === "stepConfiguration"
-      )
-      const steps = stepSetting ? component[stepSetting.key] : []
-      const stepDefinition = componentStore.getComponentSettings(
-        "@budibase/standard-components/multistepformblockstep"
-      )
-
-      steps?.forEach(step => {
-        parseComponentSettings(stepDefinition, step)
-      })
     })
   })
 

packages/client/src/components/app/Layout.svelte

@@ -324,10 +324,7 @@
   <div
     id="side-panel-container"
     class:open={$sidePanelStore.open}
-    use:clickOutside={{
+    use:clickOutside={autoCloseSidePanel ? sidePanelStore.actions.close : null}
-      callback: autoCloseSidePanel ? sidePanelStore.actions.close : null,
-      allowedType: "mousedown",
-    }}
     class:builder={$builderStore.inBuilder}
   >
     <div class="side-panel-header">

packages/client/src/utils/buttonActions.js

@@ -542,16 +542,22 @@ export const enrichButtonActions = (actions, context) => {
                 // then execute the rest of the actions in the chain
                 const result = await callback()
                 if (result !== false) {
-                  // Generate a new total context to pass into the next enrichment
+                  // Generate a new total context for the next enrichment
                   buttonContext.push(result)
                   const newContext = { ...context, actions: buttonContext }
 
-                  // Enrich and call the next button action if there is more than one action remaining
+                  // Enrich and call the next button action if there is more
+                  // than one action remaining
                   const next = enrichButtonActions(
                     actions.slice(i + 1),
                     newContext
                   )
-                  resolve(typeof next === "function" ? await next() : true)
+                  if (typeof next === "function") {
+                    // Pass the event context back into the new action chain
+                    resolve(await next(eventContext))
+                  } else {
+                    resolve(true)
+                  }
                 } else {
                   resolve(false)
                 }

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -124,6 +124,7 @@
     const fieldSchema = schemaFields.find(x => x.name === filter.field)
     filter.type = fieldSchema?.type
     filter.subtype = fieldSchema?.subtype
+    filter.formulaType = fieldSchema?.formulaType
 
     // Update external type based on field
     filter.externalType = getSchema(filter)?.externalType

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -121,8 +121,14 @@
 
   const onContextMenu = e => {
     e.preventDefault()
+
+    // The timeout allows time for clickoutside to close other open popvers
+    // before we show this one. Without the timeout, this popover closes again
+    // before it's even visible as clickoutside closes it.
+    setTimeout(() => {
       ui.actions.blur()
       open = !open
+    }, 10)
   }
 
   const sortAscending = () => {

packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte

@@ -18,7 +18,7 @@
     focusedCellAPI,
     focusedRowId,
     notifications,
-    isDatasourcePlus,
+    hasBudibaseIdentifiers,
   } = getContext("grid")
 
   let anchor
@@ -82,7 +82,7 @@
         </MenuItem>
         <MenuItem
           icon="Copy"
-          disabled={isNewRow || !$focusedRow?._id || !$isDatasourcePlus}
+          disabled={isNewRow || !$focusedRow?._id || !$hasBudibaseIdentifiers}
           on:click={() => copyToClipboard($focusedRow?._id)}
           on:click={menu.actions.close}
         >
@@ -90,7 +90,7 @@
         </MenuItem>
         <MenuItem
           icon="Copy"
-          disabled={isNewRow || !$focusedRow?._rev}
+          disabled={isNewRow || !$focusedRow?._rev || !$hasBudibaseIdentifiers}
           on:click={() => copyToClipboard($focusedRow?._rev)}
           on:click={menu.actions.close}
         >

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -75,14 +75,18 @@ export const deriveStores = context => {
     }
   )
 
-  const isDatasourcePlus = derived(datasource, $datasource => {
+  const hasBudibaseIdentifiers = derived(datasource, $datasource => {
-    return ["table", "viewV2"].includes($datasource?.type)
+    let type = $datasource?.type
+    if (type === "provider") {
+      type = $datasource.value?.datasource?.type
+    }
+    return ["table", "viewV2", "link"].includes(type)
   })
 
   return {
     schema,
     enrichedSchema,
-    isDatasourcePlus,
+    hasBudibaseIdentifiers,
   }
 }
 

packages/frontend-core/src/components/grid/stores/menu.js

@@ -17,6 +17,7 @@ export const createActions = context => {
 
   const open = (cellId, e) => {
     e.preventDefault()
+    e.stopPropagation()
 
     // Get DOM node for grid data wrapper to compute relative position to
     const gridNode = document.getElementById(gridID)

packages/frontend-core/src/components/grid/stores/rows.js

@@ -83,7 +83,7 @@ export const createActions = context => {
     error,
     notifications,
     fetch,
-    isDatasourcePlus,
+    hasBudibaseIdentifiers,
     refreshing,
   } = context
   const instanceLoaded = writable(false)
@@ -196,9 +196,16 @@ export const createActions = context => {
   // Handles validation errors from the rows API and updates local validation
   // state, storing error messages against relevant cells
   const handleValidationError = (rowId, error) => {
+    let errorString
+    if (typeof error === "string") {
+      errorString = error
+    } else if (typeof error?.message === "string") {
+      errorString = error.message
+    }
+
     // If the server doesn't reply with a valid error, assume that the source
     // of the error is the focused cell's column
-    if (!error?.json?.validationErrors && error?.message) {
+    if (!error?.json?.validationErrors && errorString) {
       const focusedColumn = get(focusedCellId)?.split("-")[1]
       if (focusedColumn) {
         error = {
@@ -261,7 +268,7 @@ export const createActions = context => {
         focusedCellId.set(`${rowId}-${erroredColumns[0]}`)
       }
     } else {
-      get(notifications).error(error?.message || "An unknown error occurred")
+      get(notifications).error(errorString || "An unknown error occurred")
     }
   }
 
@@ -458,14 +465,14 @@ export const createActions = context => {
     }
     let rowsToAppend = []
     let newRow
-    const $isDatasourcePlus = get(isDatasourcePlus)
+    const $hasBudibaseIdentifiers = get(hasBudibaseIdentifiers)
     for (let i = 0; i < newRows.length; i++) {
       newRow = newRows[i]
 
       // Ensure we have a unique _id.
       // This means generating one for non DS+, overwriting any that may already
       // exist as we cannot allow duplicates.
-      if (!$isDatasourcePlus) {
+      if (!$hasBudibaseIdentifiers) {
         newRow._id = Helpers.uuid()
       }
 
@@ -510,7 +517,7 @@ export const createActions = context => {
   const cleanRow = row => {
     let clone = { ...row }
     delete clone.__idx
-    if (!get(isDatasourcePlus)) {
+    if (!get(hasBudibaseIdentifiers)) {
       delete clone._id
     }
     return clone

packages/server/src/api/controllers/application.ts

@@ -52,6 +52,8 @@ import {
   FetchAppPackageResponse,
   DuplicateAppRequest,
   DuplicateAppResponse,
+  UpdateAppRequest,
+  UpdateAppResponse,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -450,7 +452,7 @@ export async function create(ctx: UserCtx<CreateAppRequest, App>) {
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
 export async function update(
-  ctx: UserCtx<{ name?: string; url?: string }, App>
+  ctx: UserCtx<UpdateAppRequest, UpdateAppResponse>
 ) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   // validation

packages/server/src/api/controllers/row/staticFormula.ts

@@ -1,20 +1,11 @@
 import { getRowParams } from "../../../db/utils"
 import {
   outputProcessing,
-  processAutoColumn,
   processFormulas,
 } from "../../../utilities/rowProcessor"
-import { context, locks } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
-import {
+import { Table, Row, FormulaType, FieldType } from "@budibase/types"
-  Table,
-  Row,
-  LockType,
-  LockName,
-  FormulaType,
-  FieldType,
-} from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
-import sdk from "../../../sdk"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
 
@@ -151,30 +142,7 @@ export async function finaliseRow(
   // if another row has been written since processing this will
   // handle the auto ID clash
   if (oldTable && !isEqual(oldTable, table)) {
-    try {
     await db.put(table)
-    } catch (err: any) {
-      if (err.status === 409) {
-        // Some conflicts with the autocolumns occurred, we need to refetch the table and recalculate
-        await locks.doWithLock(
-          {
-            type: LockType.AUTO_EXTEND,
-            name: LockName.PROCESS_AUTO_COLUMNS,
-            resource: table._id,
-          },
-          async () => {
-            const latestTable = await sdk.tables.getTable(table._id!)
-            let response = processAutoColumn(null, latestTable, row, {
-              reprocessing: true,
-            })
-            await db.put(response.table)
-            row = response.row
-          }
-        )
-      } else {
-        throw err
-      }
-    }
   }
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row

packages/server/src/api/routes/tests/search.spec.ts

@@ -156,7 +156,7 @@ describe.each([
     return expectSearch({ query })
   }
 
-  describe("strings", () => {
+  describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
     beforeAll(async () => {
       await createTable({
         name: { name: "name", type: FieldType.STRING },
@@ -508,7 +508,7 @@ describe.each([
     })
   })
 
-  describe("array of strings", () => {
+  describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
     beforeAll(async () => {
       await createTable({
         numbers: {

packages/server/src/sdk/app/rows/tests/internal.spec.ts

@@ -145,6 +145,7 @@ describe("sdk >> rows >> internal", () => {
                 lastID: 1,
               },
             },
+            _rev: expect.stringMatching("2-.*"),
           },
           row: {
             ...row,
@@ -189,7 +190,6 @@ describe("sdk >> rows >> internal", () => {
             type: FieldType.AUTO,
             subtype: AutoFieldSubType.AUTO_ID,
             autocolumn: true,
-            lastID: 0,
           },
         },
       })
@@ -199,7 +199,7 @@ describe("sdk >> rows >> internal", () => {
           await internalSdk.save(table._id!, row, config.getUser()._id)
         }
         await Promise.all(
-          makeRows(10).map(row =>
+          makeRows(20).map(row =>
             internalSdk.save(table._id!, row, config.getUser()._id)
           )
         )
@@ -209,19 +209,21 @@ describe("sdk >> rows >> internal", () => {
       })
 
       const persistedRows = await config.getRows(table._id!)
-      expect(persistedRows).toHaveLength(20)
+      expect(persistedRows).toHaveLength(30)
       expect(persistedRows).toEqual(
         expect.arrayContaining(
-          Array.from({ length: 20 }).map((_, i) =>
+          Array.from({ length: 30 }).map((_, i) =>
             expect.objectContaining({ id: i + 1 })
           )
         )
       )
 
       const persistedTable = await config.getTable(table._id)
-      expect((table.schema.id as AutoColumnFieldMetadata).lastID).toBe(0)
+      expect(
+        (table.schema.id as AutoColumnFieldMetadata).lastID
+      ).toBeUndefined()
       expect((persistedTable.schema.id as AutoColumnFieldMetadata).lastID).toBe(
-        20
+        30
       )
     })
   })

packages/server/src/tests/utilities/api/application.ts

@@ -5,6 +5,8 @@ import {
   type FetchAppDefinitionResponse,
   type FetchAppPackageResponse,
   DuplicateAppResponse,
+  UpdateAppRequest,
+  UpdateAppResponse,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 import { AppStatus } from "../../../db/utils"
@@ -109,11 +111,11 @@ export class ApplicationAPI extends TestAPI {
 
   update = async (
     appId: string,
-    app: { name?: string; url?: string },
+    app: UpdateAppRequest,
     expectations?: Expectations
-  ): Promise<App> => {
+  ): Promise<UpdateAppResponse> => {
     return await this._put<App>(`/api/applications/${appId}`, {
-      fields: app,
+      body: app,
       expectations,
     })
   }

packages/server/src/utilities/rowProcessor/index.ts

@@ -1,6 +1,6 @@
 import * as linkRows from "../../db/linkedRows"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
-import { objectStore, utils } from "@budibase/backend-core"
+import { context, objectStore, utils } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
 import {
@@ -25,7 +25,44 @@ type AutoColumnProcessingOpts = {
   noAutoRelationships?: boolean
 }
 
-const BASE_AUTO_ID = 1
+// Returns the next auto ID for a column in a table. On success, the table will
+// be updated which is why it gets returned. The nextID returned is guaranteed
+// to be given only to you, and if you don't use it it's gone forever (a gap
+// will be left in the auto ID sequence).
+//
+// This function can throw if it fails to generate an auto ID after so many
+// attempts.
+async function getNextAutoId(
+  table: Table,
+  column: string
+): Promise<{ table: Table; nextID: number }> {
+  const db = context.getAppDB()
+  for (let attempt = 0; attempt < 5; attempt++) {
+    const schema = table.schema[column]
+    if (schema.type !== FieldType.NUMBER && schema.type !== FieldType.AUTO) {
+      throw new Error(`Column ${column} is not an auto column`)
+    }
+    schema.lastID = (schema.lastID || 0) + 1
+    try {
+      const resp = await db.put(table)
+      table._rev = resp.rev
+      return { table, nextID: schema.lastID }
+    } catch (e: any) {
+      if (e.status !== 409) {
+        throw e
+      }
+      // We wait for a random amount of time before retrying. The randomness
+      // makes it less likely for multiple requests modifying this table to
+      // collide.
+      await new Promise(resolve =>
+        setTimeout(resolve, Math.random() * 1.2 ** attempt * 1000)
+      )
+      table = await db.get(table._id)
+    }
+  }
+
+  throw new Error("Failed to generate an auto ID")
+}
 
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
@@ -37,7 +74,7 @@ const BASE_AUTO_ID = 1
  * @returns The updated row and table, the table may need to be updated
  * for automatic ID purposes.
  */
-export function processAutoColumn(
+export async function processAutoColumn(
   userId: string | null | undefined,
   table: Table,
   row: Row,
@@ -79,8 +116,9 @@ export function processAutoColumn(
         break
       case AutoFieldSubType.AUTO_ID:
         if (creating) {
-          schema.lastID = !schema.lastID ? BASE_AUTO_ID : schema.lastID + 1
+          const { table: newTable, nextID } = await getNextAutoId(table, key)
-          row[key] = schema.lastID
+          table = newTable
+          row[key] = nextID
         }
         break
     }

packages/types/src/api/web/application.ts

@@ -44,3 +44,6 @@ export interface PublishResponse {
   status: string
   appUrl: string
 }
+
+export interface UpdateAppRequest extends Partial<App> {}
+export interface UpdateAppResponse extends App {}

packages/types/src/sdk/locks.ts

@@ -21,7 +21,6 @@ export enum LockName {
   PERSIST_WRITETHROUGH = "persist_writethrough",
   QUOTA_USAGE_EVENT = "quota_usage_event",
   APP_MIGRATION = "app_migrations",
-  PROCESS_AUTO_COLUMNS = "process_auto_columns",
   PROCESS_USER_INVITE = "process_user_invite",
 }