Merge branch 'master' of github.com:Budibase/budibase into feature/sql-query-aliasing

.eslintrc.json

@@ -45,6 +45,16 @@
         "no-prototype-builtins": "off",
         "local-rules/no-budibase-imports": "error"
       }
+    },
+    {
+      "files": [
+        "packages/builder/**/*",
+        "packages/client/**/*",
+        "packages/frontend-core/**/*"
+      ],
+      "rules": {
+        "no-console": ["error", { "allow": ["warn", "error", "debug"] } ]
+      }
     }
   ],
   "rules": {

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.16.0",
+  "version": "2.17.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit 485ec16a9eed48c548a5f1239772139f3319f028
+Subproject commit cc12291732ee902dc832bc7d93cf2086ffdf0cff

packages/backend-core/package.json

@@ -21,7 +21,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/nano": "10.1.4",
+    "@budibase/nano": "10.1.5",
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",

packages/backend-core/src/objectStore/cloudfront.ts

@@ -23,7 +23,7 @@ const getCloudfrontSignParams = () => {
   return {
     keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID!,
     privateKeyString: getPrivateKey(),
-    expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour
+    expireTime: new Date().getTime() + 1000 * 60 * 60 * 24, // 1 day
   }
 }
 

packages/backend-core/src/objectStore/objectStore.ts

@@ -7,7 +7,7 @@ import tar from "tar-fs"
 import zlib from "zlib"
 import { promisify } from "util"
 import { join } from "path"
-import fs from "fs"
+import fs, { ReadStream } from "fs"
 import env from "../environment"
 import { budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
@@ -184,7 +184,7 @@ export async function upload({
 export async function streamUpload(
   bucketName: string,
   filename: string,
-  stream: any,
+  stream: ReadStream | ReadableStream,
   extra = {}
 ) {
   const objectStore = ObjectStore(bucketName)

packages/builder/src/builderStore/dataBinding.js

@@ -364,7 +364,6 @@ const getContextBindings = (asset, componentId) => {
  * Generates a set of bindings for a given component context
  */
 const generateComponentContextBindings = (asset, componentContext) => {
-  console.log("Hello ")
   const { component, definition, contexts } = componentContext
   if (!component || !definition || !contexts?.length) {
     return []

packages/builder/src/builderStore/store/frontend.js

@@ -158,6 +158,7 @@ export const getFrontendStore = () => {
           ...INITIAL_FRONTEND_STATE.features,
           ...application.features,
         },
+        automations: application.automations || {},
         icon: application.icon || {},
         initialised: true,
       }))

packages/builder/src/builderStore/websocket.js

@@ -21,7 +21,7 @@ export const createBuilderWebsocket = appId => {
     })
   })
   socket.on("connect_error", err => {
-    console.log("Failed to connect to builder websocket:", err.message)
+    console.error("Failed to connect to builder websocket:", err.message)
   })
   socket.on("disconnect", () => {
     userStore.actions.reset()

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -15,7 +15,6 @@
     Icon,
     Checkbox,
     DatePicker,
-    Detail,
   } from "@budibase/bbui"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import { automationStore, selectedAutomation } from "builderStore"
@@ -33,6 +32,8 @@
   import Editor from "components/integration/QueryEditor.svelte"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte"
+  import BindingPicker from "components/common/bindings/BindingPicker.svelte"
+  import { BindingHelpers } from "components/common/bindings/utils"
   import {
     bindingsToCompletions,
     hbAutocomplete,
@@ -56,7 +57,7 @@
   let drawer
   let fillWidth = true
   let inputData
-  let codeBindingOpen = false
+  let insertAtPos, getCaretPosition
   $: filters = lookForFilters(schemaProperties) || []
   $: tempFilters = filters
   $: stepId = block.stepId
@@ -75,6 +76,10 @@
   $: isUpdateRow = stepId === ActionStepID.UPDATE_ROW
   $: codeMode =
     stepId === "EXECUTE_BASH" ? EditorModes.Handlebars : EditorModes.JS
+  $: bindingsHelpers = new BindingHelpers(getCaretPosition, insertAtPos, {
+    disableWrapping: true,
+  })
+  $: editingJs = codeMode === EditorModes.JS
 
   $: stepCompletions =
     codeMode === EditorModes.Handlebars
@@ -157,6 +162,7 @@
     let bindings = []
     let loopBlockCount = 0
     const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => {
+      if (!name) return
       const runtimeBinding = determineRuntimeBinding(name, idx, isLoopBlock)
       const categoryName = determineCategoryName(idx, isLoopBlock, bindingName)
 
@@ -291,7 +297,6 @@
         loopBlockCount++
         continue
       }
-
       Object.entries(schema).forEach(([name, value]) =>
         addBinding(name, value, icon, idx, isLoopBlock, bindingName)
       )
@@ -539,39 +544,51 @@
             />
           {:else if value.customType === "code"}
             <CodeEditorModal>
-              {#if codeMode == EditorModes.JS}
+              <div class:js-editor={editingJs}>
-                <ActionButton
+                <div class:js-code={editingJs} style="width: 100%">
-                  on:click={() => (codeBindingOpen = !codeBindingOpen)}
+                  <CodeEditor
-                  quiet
+                    value={inputData[key]}
-                  icon={codeBindingOpen ? "ChevronDown" : "ChevronRight"}
+                    on:change={e => {
-                >
+                      // need to pass without the value inside
-                  <Detail size="S">Bindings</Detail>
+                      onChange({ detail: e.detail }, key)
-                </ActionButton>
+                      inputData[key] = e.detail
-                {#if codeBindingOpen}
+                    }}
-                  <pre>{JSON.stringify(bindings, null, 2)}</pre>
+                    completions={stepCompletions}
-                {/if}
+                    mode={codeMode}
-              {/if}
+                    autocompleteEnabled={codeMode !== EditorModes.JS}
-              <CodeEditor
+                    bind:getCaretPosition
-                value={inputData[key]}
+                    bind:insertAtPos
-                on:change={e => {
+                    height={500}
-                  // need to pass without the value inside
+                  />
-                  onChange({ detail: e.detail }, key)
+                  <div class="messaging">
-                  inputData[key] = e.detail
+                    {#if codeMode === EditorModes.Handlebars}
-                }}
+                      <Icon name="FlashOn" />
-                completions={stepCompletions}
+                      <div class="messaging-wrap">
-                mode={codeMode}
+                        <div>
-                autocompleteEnabled={codeMode != EditorModes.JS}
+                          Add available bindings by typing <strong>
-                height={500}
+                            &#125;&#125;
-              />
+                          </strong>
-              <div class="messaging">
+                        </div>
-                {#if codeMode == EditorModes.Handlebars}
+                      </div>
-                  <Icon name="FlashOn" />
+                    {/if}
-                  <div class="messaging-wrap">
+                  </div>
-                    <div>
+                </div>
-                      Add available bindings by typing <strong>
+                {#if editingJs}
-                        &#125;&#125;
+                  <div class="js-binding-picker">
-                      </strong>
+                    <BindingPicker
-                    </div>
+                      {bindings}
+                      allowHelpers={false}
+                      addBinding={binding =>
+                        bindingsHelpers.onSelectBinding(
+                          inputData[key],
+                          binding,
+                          {
+                            js: true,
+                            dontDecode: true,
+                          }
+                        )}
+                      mode="javascript"
+                    />
                   </div>
                 {/if}
               </div>
@@ -658,4 +675,20 @@
   .test :global(.drawer) {
     width: 10000px !important;
   }
+
+  .js-editor {
+    display: flex;
+    flex-direction: row;
+    flex-grow: 1;
+    width: 100%;
+  }
+
+  .js-code {
+    flex: 7;
+  }
+
+  .js-binding-picker {
+    flex: 3;
+    margin-top: calc((var(--spacing-xl) * -1) + 1px);
+  }
 </style>

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -54,6 +54,7 @@
   export let placeholder = null
   export let autocompleteEnabled = true
   export let autofocus = false
+  export let jsBindingWrapping = true
 
   // Export a function to expose caret position
   export const getCaretPosition = () => {
@@ -187,7 +188,7 @@
       )
       complete.push(
         EditorView.inputHandler.of((view, from, to, insert) => {
-          if (insert === "$") {
+          if (jsBindingWrapping && insert === "$") {
             let { text } = view.state.doc.lineAt(from)
 
             const left = from ? text.substring(0, from) : ""

packages/builder/src/components/common/CodeEditor/index.js

@@ -286,13 +286,14 @@ export const hbInsert = (value, from, to, text) => {
   return parsedInsert
 }
 
-export function jsInsert(value, from, to, text, { helper } = {}) {
+export function jsInsert(value, from, to, text, { helper, disableWrapping }) {
   let parsedInsert = ""
 
   const left = from ? value.substring(0, from) : ""
   const right = to ? value.substring(to) : ""
-
+  if (disableWrapping) {
-  if (helper) {
+    parsedInsert = text
+  } else if (helper) {
     parsedInsert = `helpers.${text}()`
   } else if (!left.includes('$("') || !right.includes('")')) {
     parsedInsert = `$("${text}")`
@@ -312,7 +313,7 @@ export const insertBinding = (view, from, to, text, mode) => {
   } else if (mode.name == "handlebars") {
     parsedInsert = hbInsert(view.state.doc?.toString(), from, to, text)
   } else {
-    console.log("Unsupported")
+    console.warn("Unsupported")
     return
   }
 

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -29,10 +29,9 @@
     hbAutocomplete,
     EditorModes,
     bindingsToCompletions,
-    hbInsert,
-    jsInsert,
   } from "../CodeEditor"
   import BindingPicker from "./BindingPicker.svelte"
+  import { BindingHelpers } from "./utils"
 
   const dispatch = createEventDispatcher()
 
@@ -60,8 +59,10 @@
   let targetMode = null
 
   $: usingJS = mode === "JavaScript"
-  $: editorMode = mode == "JavaScript" ? EditorModes.JS : EditorModes.Handlebars
+  $: editorMode =
+    mode === "JavaScript" ? EditorModes.JS : EditorModes.Handlebars
   $: bindingCompletions = bindingsToCompletions(bindings, editorMode)
+  $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
 
   const updateValue = val => {
     valid = isValid(readableToRuntimeBinding(bindings, val))
@@ -70,31 +71,13 @@
     }
   }
 
-  // Adds a JS/HBS helper to the expression
   const onSelectHelper = (helper, js) => {
-    const pos = getCaretPosition()
+    bindingHelpers.onSelectHelper(js ? jsValue : hbsValue, helper, { js })
-    const { start, end } = pos
-    if (js) {
-      let js = decodeJSBinding(jsValue)
-      const insertVal = jsInsert(js, start, end, helper.text, { helper: true })
-      insertAtPos({ start, end, value: insertVal })
-    } else {
-      const insertVal = hbInsert(hbsValue, start, end, helper.text)
-      insertAtPos({ start, end, value: insertVal })
-    }
   }
 
-  // Adds a data binding to the expression
   const onSelectBinding = (binding, { forceJS } = {}) => {
-    const { start, end } = getCaretPosition()
+    const js = usingJS || forceJS
-    if (usingJS || forceJS) {
+    bindingHelpers.onSelectBinding(js ? jsValue : hbsValue, binding, { js })
-      let js = decodeJSBinding(jsValue)
-      const insertVal = jsInsert(js, start, end, binding.readableBinding)
-      insertAtPos({ start, end, value: insertVal })
-    } else {
-      const insertVal = hbInsert(hbsValue, start, end, binding.readableBinding)
-      insertAtPos({ start, end, value: insertVal })
-    }
   }
 
   const onChangeMode = e => {

packages/builder/src/components/common/bindings/BindingPicker.svelte

@@ -9,6 +9,7 @@
   export let bindings
   export let mode
   export let allowHelpers
+  export let noPaddingTop = false
 
   let search = ""
   let popover
@@ -47,9 +48,10 @@
     })
   $: filteredHelpers = helpers?.filter(helper => {
     return (
-      !search ||
+      (!search ||
-      helper.label.match(searchRgx) ||
+        helper.label.match(searchRgx) ||
-      helper.description.match(searchRgx)
+        helper.description.match(searchRgx)) &&
+      (mode.name !== "javascript" || helper.allowsJs)
     )
   })
 

packages/builder/src/components/common/bindings/utils.js

@@ -1,38 +1,41 @@
-export function addHBSBinding(value, caretPos, binding) {
+import { decodeJSBinding } from "@budibase/string-templates"
-  binding = typeof binding === "string" ? binding : binding.path
+import { hbInsert, jsInsert } from "components/common/CodeEditor"
-  value = value == null ? "" : value
 
-  const left = caretPos?.start ? value.substring(0, caretPos.start) : ""
+export class BindingHelpers {
-  const right = caretPos?.end ? value.substring(caretPos.end) : ""
+  constructor(getCaretPosition, insertAtPos, { disableWrapping } = {}) {
-  if (!left.includes("{{") || !right.includes("}}")) {
+    this.getCaretPosition = getCaretPosition
-    binding = `{{ ${binding} }}`
+    this.insertAtPos = insertAtPos
+    this.disableWrapping = disableWrapping
   }
-  if (caretPos.start) {
-    value =
-      value.substring(0, caretPos.start) +
-      binding +
-      value.substring(caretPos.end, value.length)
-  } else {
-    value += binding
-  }
-  return value
-}
 
-export function addJSBinding(value, caretPos, binding, { helper } = {}) {
+  // Adds a JS/HBS helper to the expression
-  binding = typeof binding === "string" ? binding : binding.path
+  onSelectHelper(value, helper, { js, dontDecode }) {
-  value = value == null ? "" : value
+    const pos = this.getCaretPosition()
-  if (!helper) {
+    const { start, end } = pos
-    binding = `$("${binding}")`
+    if (js) {
-  } else {
+      const jsVal = dontDecode ? value : decodeJSBinding(value)
-    binding = `helpers.${binding}()`
+      const insertVal = jsInsert(jsVal, start, end, helper.text, {
+        helper: true,
+      })
+      this.insertAtPos({ start, end, value: insertVal })
+    } else {
+      const insertVal = hbInsert(value, start, end, helper.text)
+      this.insertAtPos({ start, end, value: insertVal })
+    }
   }
-  if (caretPos.start) {
+
-    value =
+  // Adds a data binding to the expression
-      value.substring(0, caretPos.start) +
+  onSelectBinding(value, binding, { js, dontDecode }) {
-      binding +
+    const { start, end } = this.getCaretPosition()
-      value.substring(caretPos.end, value.length)
+    if (js) {
-  } else {
+      const jsVal = dontDecode ? value : decodeJSBinding(value)
-    value += binding
+      const insertVal = jsInsert(jsVal, start, end, binding.readableBinding, {
+        disableWrapping: this.disableWrapping,
+      })
+      this.insertAtPos({ start, end, value: insertVal })
+    } else {
+      const insertVal = hbInsert(value, start, end, binding.readableBinding)
+      this.insertAtPos({ start, end, value: insertVal })
+    }
   }
-  return value
 }

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -67,7 +67,7 @@
         }))
         navigateStep(target)
       } else {
-        console.log("Could not retrieve step")
+        console.warn("Could not retrieve step")
       }
     } else {
       if (typeof tourStep.onComplete === "function") {

packages/builder/src/components/portal/onboarding/tourHandler.js

@@ -3,11 +3,11 @@ import { get } from "svelte/store"
 
 const registerNode = async (node, tourStepKey) => {
   if (!node) {
-    console.log("Tour Handler - an anchor node is required")
+    console.warn("Tour Handler - an anchor node is required")
   }
 
   if (!get(store).tourKey) {
-    console.log("Tour Handler - No active tour ", tourStepKey, node)
+    console.error("Tour Handler - No active tour ", tourStepKey, node)
     return
   }
 

packages/builder/src/components/portal/onboarding/tours.js

@@ -45,7 +45,7 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
         onboarding: false,
       }))
     } catch (e) {
-      console.log("Onboarding failed", e)
+      console.error("Onboarding failed", e)
       return false
     }
     return true

packages/builder/src/constants/completions.js

@@ -1,4 +1,4 @@
-import { getManifest } from "@budibase/string-templates"
+import { getManifest, helpersToRemoveForJs } from "@budibase/string-templates"
 
 export function handlebarsCompletions() {
   const manifest = getManifest()
@@ -11,6 +11,9 @@ export function handlebarsCompletions() {
       label: helperName,
       displayText: helperName,
       description: helperConfig.description,
+      allowsJs:
+        !helperConfig.requiresBlock &&
+        !helpersToRemoveForJs.includes(helperName),
     }))
   )
 }

packages/builder/src/helpers/urlStateSync.js

@@ -52,7 +52,7 @@ export const syncURLToState = options => {
   let cachedPage = get(routify.page)
   let previousParamsHash = null
   let debug = false
-  const log = (...params) => debug && console.log(`[${urlParam}]`, ...params)
+  const log = (...params) => debug && console.debug(`[${urlParam}]`, ...params)
 
   // Navigate to a certain URL
   const gotoUrl = (url, params) => {

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -107,7 +107,7 @@
       return
     }
     if (!prodAppId) {
-      console.log("Application id required")
+      console.error("Application id required")
       return
     }
     await usersFetch.update({

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/GeneralPanel.svelte

@@ -66,7 +66,7 @@
     try {
       await store.actions.screens.updateSetting(get(selectedScreen), key, value)
     } catch (error) {
-      console.log(error)
+      console.error(error)
       notifications.error("Error saving screen settings")
     }
   }

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte

@@ -71,7 +71,7 @@
       $goto(`./${screenId}`)
       store.actions.screens.select(screenId)
     } catch (error) {
-      console.log(error)
+      console.error(error)
       notifications.error("Error creating screens")
     }
   }

packages/builder/src/pages/builder/app/[application]/settings/automations/index.svelte

@@ -36,15 +36,12 @@
   let status = null
   let timeRange = null
   let loaded = false
-
+  $: app = $apps.find(app => $store.appId?.includes(app.appId))
-  $: app = $apps.find(app => app.devId === $store.appId?.includes(app.appId))
   $: licensePlan = $auth.user?.license?.plan
   $: page = $pageInfo.page
   $: fetchLogs(automationId, status, page, timeRange)
   $: isCloud = $admin.cloud
-
   $: chainAutomations = app?.automations?.chainAutomations ?? !isCloud
-
   const timeOptions = [
     { value: "90-d", label: "Past 90 days" },
     { value: "30-d", label: "Past 30 days" },

packages/builder/src/pages/builder/app/[application]/settings/backups/_components/ActionsRenderer.svelte

@@ -13,6 +13,7 @@
   import CreateRestoreModal from "./CreateRestoreModal.svelte"
   import { createEventDispatcher } from "svelte"
   import { isOnlyUser } from "builderStore"
+  import { BackupType } from "constants/backend/backups"
 
   export let row
 
@@ -42,12 +43,11 @@
 </script>
 
 <div class="cell">
-  <ActionMenu align="right">
+  {#if row.type !== BackupType.RESTORE}
-    <div slot="control">
+    <ActionMenu align="right">
-      <Icon size="M" hoverable name="MoreSmallList" />
+      <div slot="control">
-    </div>
+        <Icon size="M" hoverable name="MoreSmallList" />
-
+      </div>
-    {#if row.type !== "restore"}
       <AbsTooltip
         position={TooltipPosition.Left}
         text="Unavailable - another user is editing this app"
@@ -62,8 +62,8 @@
       </AbsTooltip>
       <MenuItem on:click={deleteDialog.show} icon="Delete">Delete</MenuItem>
       <MenuItem on:click={downloadExport} icon="Download">Download</MenuItem>
-    {/if}
+    </ActionMenu>
-  </ActionMenu>
+  {/if}
 </div>
 
 <Modal bind:this={restoreBackupModal}>

packages/builder/src/pages/builder/auth/login.svelte

@@ -31,7 +31,7 @@
   async function login() {
     form.validate()
     if (Object.keys(errors).length > 0) {
-      console.log("errors", errors)
+      console.error("errors", errors)
       return
     }
     try {

packages/client/manifest.json

@@ -6742,6 +6742,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "select",
         "label": "Layout",

packages/client/src/components/app/embedded-map/EmbeddedMap.svelte

@@ -307,7 +307,7 @@
       // Reset view
       resetView()
     } catch (e) {
-      console.log("There was a problem with the map", e)
+      console.error("There was a problem with the map", e)
     }
   }
 

packages/client/src/components/app/forms/CodeScanner.svelte

@@ -61,7 +61,7 @@
           resolve({ initialised: true })
         })
         .catch(err => {
-          console.log("There was a problem scanning the image", err)
+          console.error("There was a problem scanning the image", err)
           resolve({ initialised: false })
         })
     })

packages/client/src/sdk.js

@@ -28,6 +28,7 @@ import { fetchDatasourceSchema } from "./utils/schema.js"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
 import { processStringSync, makePropSafe } from "@budibase/string-templates"
+import { fetchData, LuceneUtils } from "@budibase/frontend-core"
 
 export default {
   API,
@@ -54,6 +55,8 @@ export default {
   linkable,
   getAction,
   fetchDatasourceSchema,
+  fetchData,
+  LuceneUtils,
   ContextScopes,
   getAPIKey,
   enrichButtonActions,

packages/client/src/stores/org.js

@@ -14,7 +14,7 @@ const createOrgStore = () => {
       const settingsConfigDoc = await API.getTenantConfig(tenantId)
       set({ logoUrl: settingsConfigDoc.config.logoUrl })
     } catch (e) {
-      console.log("Could not init org ", e)
+      console.error("Could not init org ", e)
     }
   }
 

packages/client/src/utils/buttonActions.js

@@ -211,29 +211,27 @@ const deleteRowHandler = async action => {
 
 const triggerAutomationHandler = async action => {
   const { fields, notificationOverride, timeout } = action.parameters
-  if (fields) {
+  try {
-    try {
+    const result = await API.triggerAutomation({
-      const result = await API.triggerAutomation({
+      automationId: action.parameters.automationId,
-        automationId: action.parameters.automationId,
+      fields,
-        fields,
+      timeout,
-        timeout,
+    })
-      })
-
-      // Value will exist if automation is synchronous, so return it.
-      if (result.value) {
-        if (!notificationOverride) {
-          notificationStore.actions.success("Automation ran successfully")
-        }
-        return { result }
-      }
 
+    // Value will exist if automation is synchronous, so return it.
+    if (result.value) {
       if (!notificationOverride) {
-        notificationStore.actions.success("Automation triggered")
+        notificationStore.actions.success("Automation ran successfully")
       }
-    } catch (error) {
+      return { result }
-      // Abort next actions
-      return false
     }
+
+    if (!notificationOverride) {
+      notificationStore.actions.success("Automation triggered")
+    }
+  } catch (error) {
+    // Abort next actions
+    return false
   }
 }
 const navigationHandler = action => {

packages/frontend-core/src/components/grid/lib/websocket.js

@@ -29,7 +29,7 @@ export const createGridWebsocket = context => {
     connectToDatasource(get(datasource))
   })
   socket.on("connect_error", err => {
-    console.log("Failed to connect to grid websocket:", err.message)
+    console.error("Failed to connect to grid websocket:", err.message)
   })
 
   // User events

packages/pro

@@ -1 +1 @@
-Subproject commit eb9565f568cfef14b336b14eee753119acfdd43b
+Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4

packages/server/__mocks__/aws-sdk.ts

@@ -1,7 +1,13 @@
+import fs from "fs"
+import { join } from "path"
+
 module AwsMock {
   const aws: any = {}
 
-  const response = (body: any) => () => ({ promise: () => body })
+  const response = (body: any, extra?: any) => () => ({
+    promise: () => body,
+    ...extra,
+  })
 
   function DocumentClient() {
     // @ts-ignore
@@ -73,9 +79,18 @@ module AwsMock {
 
     // @ts-ignore
     this.getObject = jest.fn(
-      response({
+      response(
-        Body: "",
+        {
-      })
+          Body: "",
+        },
+        {
+          createReadStream: jest
+            .fn()
+            .mockReturnValue(
+              fs.createReadStream(join(__dirname, "aws-sdk.ts"))
+            ),
+        }
+      )
     )
   }
 

packages/server/src/api/controllers/application.ts

@@ -445,6 +445,9 @@ export async function update(ctx: UserCtx) {
     name: app.name,
     url: app.url,
     icon: app.icon,
+    automations: {
+      chainAutomations: app.automations?.chainAutomations,
+    },
   })
 }
 

packages/server/src/sdk/app/backups/imports.ts

@@ -165,8 +165,9 @@ export async function importApp(
   const isTar = template.file && template?.file?.type?.endsWith("gzip")
   const isDirectory =
     template.file && fs.lstatSync(template.file.path).isDirectory()
+  let tmpPath: string | undefined = undefined
   if (template.file && (isTar || isDirectory)) {
-    const tmpPath = isTar ? await untarFile(template.file) : template.file.path
+    tmpPath = isTar ? await untarFile(template.file) : template.file.path
     if (isTar && template.file.password) {
       await decryptFiles(tmpPath, template.file.password)
     }
@@ -208,5 +209,9 @@ export async function importApp(
   }
   await updateAttachmentColumns(prodAppId, db)
   await updateAutomations(prodAppId, db)
+  // clear up afterward
+  if (tmpPath) {
+    fs.rmSync(tmpPath, { recursive: true, force: true })
+  }
   return ok
 }

packages/server/src/utilities/workerRequests.ts

@@ -1,4 +1,4 @@
-import fetch from "node-fetch"
+import { Response, default as fetch } from "node-fetch"
 import env from "../environment"
 import { checkSlashesInUrl } from "./index"
 import {
@@ -40,25 +40,21 @@ export function request(ctx?: Ctx, request?: any) {
 }
 
 async function checkResponse(
-  response: any,
+  response: Response,
   errorMsg: string,
   { ctx }: { ctx?: Ctx } = {}
 ) {
-  if (response.status !== 200) {
+  if (response.status >= 300) {
-    let error
+    let responseErrorMessage
-    try {
+    if (response.headers.get("content-type")?.includes("json")) {
-      error = await response.json()
+      const error = await response.json()
-      if (!error.message) {
+      responseErrorMessage = error.message ?? JSON.stringify(error)
-        error = JSON.stringify(error)
+    } else {
-      }
+      responseErrorMessage = await response.text()
-    } catch (err) {
-      error = await response.text()
     }
-    const msg = `Unable to ${errorMsg} - ${
+    const msg = `Unable to ${errorMsg} - ${responseErrorMessage}`
-      error.message ? error.message : error
-    }`
     if (ctx) {
-      ctx.throw(400, msg)
+      ctx.throw(msg, response.status)
     } else {
       throw msg
     }

packages/string-templates/package.json

@@ -25,7 +25,7 @@
     "manifest": "node ./scripts/gen-collection-info.js"
   },
   "dependencies": {
-    "@budibase/handlebars-helpers": "^0.13.0",
+    "@budibase/handlebars-helpers": "^0.13.1",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
     "lodash.clonedeep": "^4.5.0",

packages/string-templates/scripts/gen-collection-info.js

@@ -10,8 +10,8 @@ const marked = require("marked")
  * https://github.com/budibase/handlebars-helpers
  */
 const { join } = require("path")
+const path = require("path")
 
-const DIRECTORY = join(__dirname, "..", "..", "..")
 const COLLECTIONS = [
   "math",
   "array",
@@ -115,6 +115,8 @@ function getCommentInfo(file, func) {
     docs.example = docs.example.replace("product", "multiply")
   }
   docs.description = blocks[0].trim()
+  docs.acceptsBlock = docs.tags.some(el => el.title === "block")
+  docs.acceptsInline = docs.tags.some(el => el.title === "inline")
   return docs
 }
 
@@ -127,7 +129,7 @@ function run() {
   const foundNames = []
   for (let collection of COLLECTIONS) {
     const collectionFile = fs.readFileSync(
-      `${DIRECTORY}/node_modules/${HELPER_LIBRARY}/lib/${collection}.js`,
+      `${path.dirname(require.resolve(HELPER_LIBRARY))}/lib/${collection}.js`,
       "utf8"
     )
     const collectionInfo = {}
@@ -159,6 +161,7 @@ function run() {
         numArgs: args.length,
         example: jsDocInfo.example || undefined,
         description: jsDocInfo.description,
+        requiresBlock: jsDocInfo.acceptsBlock && !jsDocInfo.acceptsInline,
       })
     }
     outputJSON[collection] = collectionInfo

packages/string-templates/src/conversion/index.js

@@ -1,4 +1,4 @@
-const { getHelperList } = require("../helpers")
+const { getJsHelperList } = require("../helpers")
 
 function getLayers(fullBlock) {
   let layers = []
@@ -109,7 +109,7 @@ module.exports.convertHBSBlock = (block, blockNumber) => {
   const layers = getLayers(block)
 
   let value = null
-  const list = getHelperList()
+  const list = getJsHelperList()
   for (let layer of layers) {
     const parts = splitBySpace(layer)
     if (value || parts.length > 1 || list[parts[0]]) {

packages/string-templates/src/helpers/date.js

@@ -115,7 +115,7 @@ module.exports.duration = (str, pattern, format) => {
   setLocale(config.str, config.pattern)
 
   const duration = dayjs.duration(config.str, config.pattern)
-  if (!isOptions(format)) {
+  if (format && !isOptions(format)) {
     return duration.format(format)
   } else {
     return duration.humanize()

packages/string-templates/src/helpers/index.js

@@ -7,7 +7,7 @@ const {
   HelperFunctionBuiltin,
   LITERAL_MARKER,
 } = require("./constants")
-const { getHelperList } = require("./list")
+const { getJsHelperList } = require("./list")
 
 const HTML_SWAPS = {
   "<": "&lt;",
@@ -97,4 +97,4 @@ module.exports.unregisterAll = handlebars => {
   externalHandlebars.unregisterAll(handlebars)
 }
 
-module.exports.getHelperList = getHelperList
+module.exports.getJsHelperList = getJsHelperList

packages/string-templates/src/helpers/javascript.js

@@ -1,7 +1,7 @@
 const { atob } = require("../utilities")
 const cloneDeep = require("lodash.clonedeep")
 const { LITERAL_MARKER } = require("../helpers/constants")
-const { getHelperList } = require("./list")
+const { getJsHelperList } = require("./list")
 
 // The method of executing JS scripts depends on the bundle being built.
 // This setter is used in the entrypoint (either index.cjs or index.mjs).
@@ -49,7 +49,7 @@ module.exports.processJS = (handlebars, context) => {
     // app context.
     const sandboxContext = {
       $: path => getContextValue(path, cloneDeep(context)),
-      helpers: getHelperList(),
+      helpers: getJsHelperList(),
     }
 
     // Create a sandbox with our context and run the JS

packages/string-templates/src/helpers/list.js

@@ -3,7 +3,10 @@ const helperList = require("@budibase/handlebars-helpers")
 
 let helpers = undefined
 
-module.exports.getHelperList = () => {
+const helpersToRemoveForJs = ["sortBy"]
+module.exports.helpersToRemoveForJs = helpersToRemoveForJs
+
+module.exports.getJsHelperList = () => {
   if (helpers) {
     return helpers
   }
@@ -15,12 +18,17 @@ module.exports.getHelperList = () => {
   }
   for (let collection of constructed) {
     for (let [key, func] of Object.entries(collection)) {
-      helpers[key] = func
+      // Handlebars injects the hbs options to the helpers by default. We are adding an empty {} as a last parameter to simulate it
+      helpers[key] = (...props) => func(...props, {})
     }
   }
   for (let key of Object.keys(externalHandlebars.addedHelpers)) {
     helpers[key] = externalHandlebars.addedHelpers[key]
   }
+
+  for (const toRemove of helpersToRemoveForJs) {
+    delete helpers[toRemove]
+  }
   Object.freeze(helpers)
   return helpers
 }

packages/string-templates/src/index.cjs

@@ -20,6 +20,7 @@ module.exports.findHBSBlocks = templates.findHBSBlocks
 module.exports.convertToJS = templates.convertToJS
 module.exports.setJSRunner = templates.setJSRunner
 module.exports.FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
+module.exports.helpersToRemoveForJs = templates.helpersToRemoveForJs
 
 if (!process.env.NO_JS) {
   const { VM } = require("vm2")

packages/string-templates/src/index.js

@@ -10,6 +10,7 @@ const {
 } = require("./utilities")
 const { convertHBSBlock } = require("./conversion")
 const javascript = require("./helpers/javascript")
+const { helpersToRemoveForJs } = require("./helpers/list")
 
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
@@ -394,3 +395,4 @@ module.exports.convertToJS = hbs => {
 }
 
 module.exports.FIND_ANY_HBS_REGEX = FIND_ANY_HBS_REGEX
+module.exports.helpersToRemoveForJs = helpersToRemoveForJs

packages/string-templates/src/index.mjs

@@ -21,6 +21,7 @@ export const findHBSBlocks = templates.findHBSBlocks
 export const convertToJS = templates.convertToJS
 export const setJSRunner = templates.setJSRunner
 export const FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
+export const helpersToRemoveForJs = templates.helpersToRemoveForJs
 
 if (process && !process.env.NO_JS) {
   /**

packages/string-templates/test/manifest.spec.js

@@ -16,21 +16,55 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
 })
 
 const fs = require("fs")
-const { processString } = require("../src/index.cjs")
+const {
+  processString,
+  convertToJS,
+  processStringSync,
+  encodeJSBinding,
+} = require("../src/index.cjs")
 
 const tk = require("timekeeper")
+const { getJsHelperList } = require("../src/helpers")
+
 tk.freeze("2021-01-21T12:00:00")
 
+const processJS = (js, context) => {
+  return processStringSync(encodeJSBinding(js), context)
+}
+
 const manifest = JSON.parse(
   fs.readFileSync(require.resolve("../manifest.json"), "utf8")
 )
 
 const collections = Object.keys(manifest)
 const examples = collections.reduce((acc, collection) => {
-  const functions = Object.keys(manifest[collection]).filter(
+  const functions = Object.entries(manifest[collection])
-    fnc => manifest[collection][fnc].example
+    .filter(([_, details]) => details.example)
-  )
+    .map(([name, details]) => {
-  if (functions.length) {
+      const example = details.example
+      let [hbs, js] = example.split("->").map(x => x.trim())
+      if (!js) {
+        // The function has no return value
+        return
+      }
+
+      // Trim 's
+      js = js.replace(/^\'|\'$/g, "")
+      if ((parsedExpected = tryParseJson(js))) {
+        if (Array.isArray(parsedExpected)) {
+          if (typeof parsedExpected[0] === "object") {
+            js = JSON.stringify(parsedExpected)
+          } else {
+            js = parsedExpected.join(",")
+          }
+        }
+      }
+      const requiresHbsBody = details.requiresBlock
+      return [name, { hbs, js, requiresHbsBody }]
+    })
+    .filter(x => !!x)
+
+  if (Object.keys(functions).length) {
     acc[collection] = functions
   }
   return acc
@@ -55,11 +89,7 @@ function tryParseJson(str) {
 describe("manifest", () => {
   describe("examples are valid", () => {
     describe.each(Object.keys(examples))("%s", collection => {
-      it.each(examples[collection])("%s", async func => {
+      it.each(examples[collection])("%s", async (_, { hbs, js }) => {
-        const example = manifest[collection][func].example
-
-        let [hbs, js] = example.split("->").map(x => x.trim())
-
         const context = {
           double: i => i * 2,
           isString: x => typeof x === "string",
@@ -71,23 +101,40 @@ describe("manifest", () => {
           context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
         })
 
-        if (js === undefined) {
+        let result = await processString(hbs, context)
-          // The function has no return value
+        result = result.replace(/ /g, " ")
-          return
+        expect(result).toEqual(js)
+      })
+    })
+  })
+
+  describe("can be parsed and run as js", () => {
+    const jsHelpers = getJsHelperList()
+    const jsExamples = Object.keys(examples).reduce((acc, v) => {
+      acc[v] = examples[v].filter(([key]) => jsHelpers[key])
+      return acc
+    }, {})
+
+    describe.each(Object.keys(jsExamples))("%s", collection => {
+      it.each(
+        jsExamples[collection].filter(
+          ([_, { requiresHbsBody }]) => !requiresHbsBody
+        )
+      )("%s", async (_, { hbs, js }) => {
+        const context = {
+          double: i => i * 2,
+          isString: x => typeof x === "string",
         }
 
-        let result = await processString(hbs, context)
+        const arrays = hbs.match(/\[[^/\]]+\]/)
-        // Trim 's
+        arrays?.forEach((arrayString, i) => {
-        js = js.replace(/^\'|\'$/g, "")
+          hbs = hbs.replace(new RegExp(escapeRegExp(arrayString)), `array${i}`)
-        if ((parsedExpected = tryParseJson(js))) {
+          context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
-          if (Array.isArray(parsedExpected)) {
+        })
-            if (typeof parsedExpected[0] === "object") {
+
-              js = JSON.stringify(parsedExpected)
+        let convertedJs = convertToJS(hbs)
-            } else {
+
-              js = parsedExpected.join(",")
+        let result = processJS(convertedJs, context)
-            }
-          }
-        }
         result = result.replace(/ /g, " ")
         expect(result).toEqual(js)
       })

packages/types/package.json

@@ -15,7 +15,7 @@
   },
   "jest": {},
   "devDependencies": {
-    "@budibase/nano": "10.1.4",
+    "@budibase/nano": "10.1.5",
     "@types/koa": "2.13.4",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.3",

packages/worker/package.json

@@ -63,7 +63,7 @@
     "koa-useragent": "^4.1.0",
     "lodash": "4.17.21",
     "node-fetch": "2.6.7",
-    "nodemailer": "6.7.2",
+    "nodemailer": "6.9.9",
     "passport-google-oauth": "2.0.0",
     "passport-local": "1.0.0",
     "pouchdb": "7.3.0",

yarn.lock

@@ -2038,10 +2038,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/handlebars-helpers@^0.13.0":
+"@budibase/handlebars-helpers@^0.13.1":
-  version "0.13.0"
+  version "0.13.1"
-  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.0.tgz#224333d14e3900b7dacf48286af1e624a9fd62ea"
+  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.1.tgz#d02e73c0df8305cd675e70dc37f8427eb0842080"
-  integrity sha512-g8+sFrMNxsIDnK+MmdUICTVGr6ReUFtnPp9hJX0VZwz1pN3Ynolpk/Qbu6rEWAvoU1sEqY1mXr9uo/+kEfeGbQ==
+  integrity sha512-v4RbXhr3igvK3i2pj5cNltu/4NMxdPIzcUt/o0RoInhesNH1VSLRdweSFr6/Y34fsCR5jHZ6vltdcz2RgrTKgw==
   dependencies:
     get-object "^0.2.0"
     get-value "^3.0.1"
@@ -2057,10 +2057,10 @@
     to-gfm-code-block "^0.1.1"
     uuid "^9.0.1"
 
-"@budibase/nano@10.1.4":
+"@budibase/nano@10.1.5":
-  version "10.1.4"
+  version "10.1.5"
-  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.4.tgz#5c2670d0b4c12d736ddd6581c57d47c0aa45efad"
+  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.5.tgz#eeaded7bfc707ecabf8fde604425b865a90c06ec"
-  integrity sha512-J+IVaAljGideDvJss/AUxXA1599HEIUJo5c0LLlmc1KMA3GZWZjyX+w2fxAw3qF7hqFvX+qAStQgdcD3+/GPMA==
+  integrity sha512-q1eKIsYKo+iK17zsJYd3VBl+5ufQMPpHYLec0wVsid8wnJVrTQk7RNpBlBUn/EDgXM7t8XNNHlERqHu+CxJu8Q==
   dependencies:
     "@types/tough-cookie" "^4.0.2"
     axios "^1.1.3"