Updating the string templating system to be capable of async operations.

packages/builder/src/components/automation/AutomationBuilder/FlowChart/AutomationBlockTagline.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { processString } from "@budibase/string-templates"
+  import { processStringSync } from "@budibase/string-templates"
   import { get } from "lodash/fp"
   import { backendUiStore } from "builderStore"
 
@@ -55,7 +55,7 @@
     })
 
     // Fill in bindings with templating library
-    return processString(formattedTagline, { inputs })
+    return processStringSync(formattedTagline, { inputs })
   }
 </script>
 

packages/client/src/utils/buttonActions.js

@@ -5,26 +5,31 @@ import { saveRow, deleteRow, triggerAutomation } from "../api"
 const saveRowHandler = async (action, context) => {
   let draft = context[`${action.parameters.contextPath}_draft`]
   if (action.parameters.fields) {
-    Object.entries(action.parameters.fields).forEach(([key, entry]) => {
+    for (let [key, entry] of Object.entries(action.parameters.fields)) {
-      draft[key] = enrichDataBinding(entry.value, context)
+      draft[key] = await enrichDataBinding(entry.value, context)
-    })
+    }
   }
   await saveRow(draft)
 }
 
 const deleteRowHandler = async (action, context) => {
   const { tableId, revId, rowId } = action.parameters
+  const [ enrichTable, enrichRow, enrichRev ] = await Promise.all([
+    enrichDataBinding(tableId, context),
+    enrichDataBinding(rowId, context),
+    enrichDataBinding(revId, context)
+  ])
   await deleteRow({
-    tableId: enrichDataBinding(tableId, context),
+    tableId: enrichTable,
-    rowId: enrichDataBinding(rowId, context),
+    rowId: enrichRow,
-    revId: enrichDataBinding(revId, context),
+    revId: enrichRev,
   })
 }
 
 const triggerAutomationHandler = async (action, context) => {
   const params = {}
   for (let field in action.parameters.fields) {
-    params[field] = enrichDataBinding(
+    params[field] = await enrichDataBinding(
       action.parameters.fields[field].value,
       context
     )

packages/client/src/utils/componentProps.js

@@ -5,7 +5,7 @@ import { enrichButtonActions } from "./buttonActions"
  * Enriches component props.
  * Data bindings are enriched, and button actions are enriched.
  */
-export const enrichProps = (props, dataContexts, dataBindings) => {
+export const enrichProps = async (props, dataContexts, dataBindings) => {
   // Exclude all private props that start with an underscore
   let validProps = {}
   Object.entries(props)
@@ -24,7 +24,7 @@ export const enrichProps = (props, dataContexts, dataBindings) => {
   }
 
   // Enrich all data bindings in top level props
-  let enrichedProps = enrichDataBindings(validProps, context)
+  let enrichedProps = await enrichDataBindings(validProps, context)
 
   // Enrich button actions if they exist
   if (props._component.endsWith("/button") && enrichedProps.onClick) {

packages/client/src/utils/enrichDataBinding.js

@@ -6,7 +6,7 @@ const looksLikeTemplate = /{{.*}}/
 /**
  * Enriches a given input with a row from the database.
  */
-export const enrichDataBinding = (input, context) => {
+export const enrichDataBinding = async (input, context) => {
   // Only accept string inputs
   if (!input || typeof input !== "string") {
     return input
@@ -21,10 +21,10 @@ export const enrichDataBinding = (input, context) => {
 /**
  * Enriches each prop in a props object
  */
-export const enrichDataBindings = (props, context) => {
+export const enrichDataBindings = async (props, context) => {
   let enrichedProps = {}
-  Object.entries(props).forEach(([key, value]) => {
+  for (let [key, value] of Object.entries(props)) {
-    enrichedProps[key] = enrichDataBinding(value, context)
+    enrichedProps[key] = await enrichDataBinding(value, context)
-  })
+  }
   return enrichedProps
 }

packages/server/src/api/controllers/application.js

@@ -31,7 +31,7 @@ const {
   createLoginScreen,
 } = require("../../constants/screens")
 const { cloneDeep } = require("lodash/fp")
-const { objectTemplate } = require("../../utilities/stringTemplating")
+const { processObject } = require("@budibase/string-templates")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
@@ -213,7 +213,7 @@ const createEmptyAppPackage = async (ctx, app) => {
   let screensAndLayouts = []
   for (let layout of BASE_LAYOUTS) {
     const cloned = cloneDeep(layout)
-    screensAndLayouts.push(objectTemplate(cloned, app))
+    screensAndLayouts.push(await processObject(cloned, app))
   }
 
   const homeScreen = createHomeScreen(app)

packages/server/src/api/controllers/static/index.js

@@ -7,7 +7,7 @@ const fs = require("fs-extra")
 const uuid = require("uuid")
 const AWS = require("aws-sdk")
 const { prepareUpload } = require("../deploy/utils")
-const { stringTemplate } = require("../../../utilities/stringTemplating")
+const { processString } = require("@budibase/string-templates")
 const {
   budibaseAppsDir,
   budibaseTempDir,
@@ -161,7 +161,7 @@ exports.serveApp = async function(ctx) {
   })
 
   const appHbs = fs.readFileSync(`${__dirname}/templates/app.hbs`, "utf8")
-  ctx.body = stringTemplate(appHbs, {
+  ctx.body = await processString(appHbs, {
     head,
     body: html,
     style: css.code,

packages/server/src/automations/thread.js

@@ -2,7 +2,7 @@ const actions = require("./actions")
 const logic = require("./logic")
 const automationUtils = require("./automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
-const { objectTemplate } = require("../utilities/stringTemplating")
+const { processObject } = require("@budibase/string-templates")
 
 const FILTER_STEP_ID = logic.BUILTIN_DEFINITIONS.FILTER.stepId
 
@@ -44,7 +44,7 @@ class Orchestrator {
     let automation = this._automation
     for (let step of automation.definition.steps) {
       let stepFn = await this.getStepFunctionality(step.type, step.stepId)
-      step.inputs = objectTemplate(step.inputs, this._context)
+      step.inputs = await processObject(step.inputs, this._context)
       step.inputs = automationUtils.cleanInputValues(
         step.inputs,
         step.schema.inputs

packages/server/src/utilities/initialiseBudibase.js

@@ -1,6 +1,6 @@
 const { existsSync, readFile, writeFile, ensureDir } = require("fs-extra")
 const { join, resolve } = require("./centralPath")
-const { stringTemplate } = require("./stringTemplating")
+const { processString } = require("@budibase/string-templates")
 const uuid = require("uuid")
 
 module.exports = async opts => {
@@ -31,7 +31,7 @@ const createDevEnvFile = async opts => {
       }
     )
     opts.cookieKey1 = opts.cookieKey1 || uuid.v4()
-    const config = stringTemplate(template, opts)
+    const config = await processString(template, opts)
     await writeFile(destConfigFile, config, { flag: "w+" })
   }
 }

packages/server/src/utilities/stringTemplating.js

@@ -1,4 +0,0 @@
-const stringTemplates = require("@budibase/string-templates")
-
-exports.objectTemplate = stringTemplates.processObject
-exports.stringTemplate = stringTemplates.processString

packages/string-templates/src/index.js

@@ -7,10 +7,6 @@ const FIND_HBS_REGEX = /{{.*}}/
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
 
-function attemptToCorrectError(string) {
-  return string
-}
-
 /**
  * When running handlebars statements to execute on the context of the automation it possible user's may input handlebars
  * in a few different forms, some of which are invalid but are logically valid. An example of this would be the handlebars
@@ -54,28 +50,34 @@ function cleanHandlebars(string) {
 }
 
 /**
- * Given an input object this will recurse through all props to try and update
+ * utility function to check if the object is valid
- * any handlebars statements within.
- * @param {object|array} object The input structure which is to be recursed, it is important to note that
- * if the structure contains any cycles then this will fail.
- * @param {object} context The context that handlebars should fill data from.
- * @returns {object|array} The structure input, as fully updated as possible.
  */
-module.exports.processObject = (object, context) => {
+function testObject(object) {
   // JSON stringify will fail if there are any cycles, stops infinite recursion
   try {
     JSON.stringify(object)
   } catch (err) {
     throw "Unable to process inputs to JSON, cannot recurse"
   }
+}
+
+/**
+ * Given an input object this will recurse through all props to try and update any handlebars statements within.
+ * @param {object|array} object The input structure which is to be recursed, it is important to note that
+ * if the structure contains any cycles then this will fail.
+ * @param {object} context The context that handlebars should fill data from.
+ * @returns {Promise<object|array>} The structure input, as fully updated as possible.
+ */
+module.exports.processObject = async (object, context) => {
+  testObject(object)
+  // TODO: carry out any async calls before carrying out async call
   for (let key of Object.keys(object)) {
     let val = object[key]
     if (typeof val === "string") {
-      object[key] = module.exports.processString(object[key], context)
+      object[key] = await module.exports.processString(object[key], context)
     }
-    // this covers objects and arrays
     else if (typeof val === "object") {
-      object[key] = module.exports.processObject(object[key], context)
+      object[key] = await module.exports.processObject(object[key], context)
     }
   }
   return object
@@ -86,20 +88,50 @@ module.exports.processObject = (object, context) => {
  * then nothing will occur.
  * @param {string} string The template string which is the filled from the context object.
  * @param {object} context An object of information which will be used to enrich the string.
+ * @returns {Promise<string>} The enriched string, all templates should have been replaced if they can be.
+ */
+module.exports.processString = async (string, context) => {
+  // TODO: carry out any async calls before carrying out async call
+  return module.exports.processStringSync(string, context)
+}
+
+/**
+ * Given an input object this will recurse through all props to try and update any handlebars statements within. This is
+ * a pure sync call and therefore does not have the full functionality of the async call.
+ * @param {object|array} object The input structure which is to be recursed, it is important to note that
+ * if the structure contains any cycles then this will fail.
+ * @param {object} context The context that handlebars should fill data from.
+ * @returns {object|array} The structure input, as fully updated as possible.
+ */
+module.exports.processObjectSync = (object, context) => {
+  testObject(object)
+  for (let key of Object.keys(object)) {
+    let val = object[key]
+    if (typeof val === "string") {
+      object[key] = module.exports.processStringSync(object[key], context)
+    }
+    else if (typeof val === "object") {
+      object[key] = module.exports.processObjectSync(object[key], context)
+    }
+  }
+  return object
+}
+
+/**
+ * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
+ * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call.
+ * @param {string} string The template string which is the filled from the context object.
+ * @param {object} context An object of information which will be used to enrich the string.
  * @returns {string} The enriched string, all templates should have been replaced if they can be.
  */
-module.exports.processString = (string, context) => {
+module.exports.processStringSync = (string, context) => {
   if (typeof string !== "string") {
     throw "Cannot process non-string types."
   }
   let template
-  try {
+  string = cleanHandlebars(string)
-    string = cleanHandlebars(string)
+  // this does not throw an error when template can't be fulfilled, have to try correct beforehand
-    template = hbsInstance.compile(string)
+  template = hbsInstance.compile(string)
-  } catch (err) {
-    string = attemptToCorrectError(string)
-    template = hbsInstance.compile(string)
-  }
   return template(context)
 }
 
@@ -107,7 +139,7 @@ module.exports.processString = (string, context) => {
  * Errors can occur if a user of this library attempts to use a helper that has not been added to the system, these errors
  * can be captured to alert the user of the mistake.
  * @param {function} handler a function which will be called every time an error occurs when processing a handlebars
- *         statement.
+ * statement.
  */
 module.exports.errorEvents = handler => {
   hbsInstance.registerHelper("helperMissing", handler)

packages/string-templates/test/basic.spec.js

@@ -4,17 +4,17 @@ const {
 } = require("../src/index")
 
 describe("Test that the string processing works correctly", () => {
-  it("should process a basic template string", () => {
+  it("should process a basic template string", async () => {
-    const output = processString("templating is {{ adjective }}", {
+    const output = await processString("templating is {{ adjective }}", {
       adjective: "easy"
     })
     expect(output).toBe("templating is easy")
   })
 
-  it("should fail gracefully when wrong type passed in", () => {
+  it("should fail gracefully when wrong type passed in", async () => {
     let error = null
     try {
-      processString(null, null)
+      await processString(null, null)
     } catch (err) {
       error = err
     }
@@ -23,8 +23,8 @@ describe("Test that the string processing works correctly", () => {
 })
 
 describe("Test that the object processing works correctly", () => {
-  it("should be able to process an object with some template strings", () => {
+  it("should be able to process an object with some template strings", async () => {
-    const output = processObject({
+    const output = await processObject({
       first: "thing is {{ adjective }}",
       second: "thing is bad",
       third: "we are {{ adjective }} {{ noun }}",
@@ -37,30 +37,30 @@ describe("Test that the object processing works correctly", () => {
     expect(output.third).toBe("we are easy people")
   })
 
-  it("should be able to handle arrays of string templates", () => {
+  it("should be able to handle arrays of string templates", async () => {
-    const output = processObject(["first {{ noun }}", "second {{ noun }}"], {
+    const output = await processObject(["first {{ noun }}", "second {{ noun }}"], {
       noun: "person"
     })
     expect(output[0]).toBe("first person")
     expect(output[1]).toBe("second person")
   })
 
-  it("should fail gracefully when object passed in has cycles", () => {
+  it("should fail gracefully when object passed in has cycles", async () => {
     let error = null
     try {
       const innerObj = { a: "thing {{ a }}" }
       innerObj.b = innerObj
-      processObject(innerObj, { a: 1 })
+      await processObject(innerObj, { a: 1 })
     } catch (err) {
       error = err
     }
     expect(error).not.toBeNull()
   })
 
-  it("should fail gracefully when wrong type is passed in", () => {
+  it("should fail gracefully when wrong type is passed in", async () => {
     let error = null
     try {
-      processObject(null, null)
+      await processObject(null, null)
     } catch (err) {
       error = err
     }

packages/string-templates/test/escapes.spec.js

@@ -3,15 +3,15 @@ const {
 } = require("../src/index")
 
 describe("Handling context properties with spaces in their name", () => {
-  it("should be able to handle a property with a space in its name", () => {
+  it("should be able to handle a property with a space in its name", async () => {
-    const output = processString("hello my name is {{ person name }}", {
+    const output = await processString("hello my name is {{ person name }}", {
       "person name": "Mike",
     })
     expect(output).toBe("hello my name is Mike")
   })
 
-  it("should be able to handle an object with layers that requires escaping", () => {
+  it("should be able to handle an object with layers that requires escaping", async () => {
-    const output = processString("testcase {{ testing.test case }}", {
+    const output = await processString("testcase {{ testing.test case }}", {
       testing: {
         "test case": 1
       }

packages/string-templates/test/helpers.spec.js

@@ -3,8 +3,8 @@ const {
 } = require("../src/index")
 
 describe("test the custom helpers we have applied", () => {
-  it("should be able to use the object helper", () => {
+  it("should be able to use the object helper", async () => {
-    const output = processString("object is {{ object obj }}", {
+    const output = await processString("object is {{ object obj }}", {
       obj: { a: 1 },
     })
     expect(output).toBe("object is {\"a\":1}")