From a254567bec5410556b2f22f013aba083d01a867f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 25 Jan 2021 17:08:21 +0000
Subject: [PATCH] Updating string templates to have test cases for all the main
 helpers we want to make use of and adding a readme.

---
 .../PropertiesPanel/BindingPanel.svelte       |   2 +-
 packages/string-templates/README.md           |  73 +++++
 .../string-templates/src/helpers/external.js  |   2 +-
 packages/string-templates/src/index.js        |   9 +-
 .../src/processors/preprocessor.js            |  20 +-
 packages/string-templates/src/utilities.js    |  24 +-
 .../string-templates/test/helpers.spec.js     | 250 +++++++++++++++++-
 7 files changed, 362 insertions(+), 18 deletions(-)
 create mode 100644 packages/string-templates/README.md

diff --git a/packages/builder/src/components/design/PropertiesPanel/BindingPanel.svelte b/packages/builder/src/components/design/PropertiesPanel/BindingPanel.svelte
index bbc86d2f78..0798275e7a 100644
--- a/packages/builder/src/components/design/PropertiesPanel/BindingPanel.svelte
+++ b/packages/builder/src/components/design/PropertiesPanel/BindingPanel.svelte
@@ -1,6 +1,6 @@
 <script>
   import groupBy from "lodash/fp/groupBy"
-  import { Button, TextArea, Drawer, Heading, Spacer } from "@budibase/bbui"
+  import { TextArea, Heading, Spacer } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { isValid } from "@budibase/string-templates"
   import {
diff --git a/packages/string-templates/README.md b/packages/string-templates/README.md
new file mode 100644
index 0000000000..b9f6a6f557
--- /dev/null
+++ b/packages/string-templates/README.md
@@ -0,0 +1,73 @@
+# String templating
+This package provides a common system for string templating across the Budibase Builder, client and server.
+The templating is provided through the use of [Handlebars](https://handlebarsjs.com/) an extension of Mustache
+which is capable of carrying out logic. We have also extended the base Handlebars functionality through the use 
+of a set of helpers provided through the [handlebars-helpers](https://github.com/helpers/handlebars-helpers) package.
+
+We have not implemented all the helpers provided by the helpers package as some of them provide functionality
+we felt would not be beneficial. The following collections of helpers have been implemented:
+1. [Math](https://github.com/helpers/handlebars-helpers/tree/master#math) - a set of useful helpers for 
+carrying out logic pertaining to numbers e.g. `avg`, `add`, `abs` and so on.
+2. [Array](https://github.com/helpers/handlebars-helpers/tree/master#array) - some very specific helpers
+for use with arrays, useful for example in automations. Helpers like `first`, `last`, `after` and `join` 
+can be useful for getting particular portions of arrays or turning them into strings.
+3. [Number](https://github.com/helpers/handlebars-helpers/tree/master#number) - unlike the math helpers these
+are useful for converting numbers into useful formats for display, e.g. `bytes`, `addCommas` and `toPrecision`.
+4. [URL](https://github.com/helpers/handlebars-helpers/tree/master#url) - very specific helpers for dealing with URLs,
+such as `encodeURI`, `escape`, `stripQueryString` and `stripProtocol`. These are primarily useful
+for building up particular URLs to hit as say part of an automation.
+5. [String](https://github.com/helpers/handlebars-helpers/tree/master#string) - these helpers are useful for building 
+strings and preparing them for display, e.g. `append`, `camelcase`, `capitalize` and `ellipsis`.
+6. [Comparison](https://github.com/helpers/handlebars-helpers/tree/master#comparison) - these helpers are mainly for 
+building strings when particular conditions are met, for example `and`, `or`, `gt`, `lt`, `not` and so on. This is a very
+extensive set of helpers but is mostly as would be expected from a set of logical operators.
+
+## Format
+There are two main ways that the templating system can be used, the first is very similar to that which
+would be produced by Mustache - a single statement:
+```
+Hello I'm building a {{uppercase adjective}} string with Handlebars!
+```
+In the statement above provided a context of `{adjective: "cool"}` will produce a string of `Hello I'm building a COOL string with Handlebars!`.
+Here we can see an example of how string helpers can be used to make a string exactly as we need it. These statements are relatively
+simple; we can also stack helpers as such: `{{ uppercase (remove string "bad") }}` with the use of parenthesis.
+
+The other type of statement that can be made with the templating system is conditional ones, that appear as such:
+```
+Hello I'm building a {{ #gte score "50" }}Great{{ else }}Bad{{ /gte }} string with Handlebars!
+``` 
+In this string we can see that the string `Great` or `Bad` will be inserted depending on the state of the 
+`score` context variable. The comparison, string and array helpers all provide some conditional operators which can be used
+in this way. There will also be some operators which will be built with a very similar syntax but will produce an 
+iterative operation, like a for each - an example of this would be the `forEach` array helper.
+
+## Usage
+Usage of this templating package is through one of the primary functions provided by the package - these functions are 
+as follows:
+1. `processString` - `async (string, object)` - given a template string and a context object this will build a string 
+using our pre-processors, post-processors and handlebars.
+2. `processObject` - `async (object, object)` - carries out the functionality provided by `processString` for any string 
+inside the given object. This will recurse deeply into the provided object so for very large objects this could be slow.
+3. `processStringSync` - `(string, object)` - a reduced functionality processing of strings which is synchronous, like 
+functions provided by Node (e.g. `readdirSync`)
+4. `processObjectSync` - `(object, object)` - as with the sync'd string, recurses an object to process it synchronously.
+5. `makePropSafe` - `(string)` - some properties cannot be handled by Handlebars, for example `Table 1` is not valid due 
+to spaces found in the property name. This will update the property name to `[Table 1]` wrapping it in literal
+specifiers so that it is safe for use in Handlebars. Ideally this function should be called for every level of an object
+being accessed, for example `[Table 1].[property name]` is the syntax that is required for Handlebars. 
+6. `isValid` - `(string)` - checks the given string for any templates and provides a boolean stating whether it is a valid
+template.
+
+## Development
+This library is built with [Rollup](https://rollupjs.org/guide/en/) as many of the packages built by Budibase are. We have
+built the string templating package as a UMD so that it can be used by Node and Browser based applications. This package also 
+builds Typescript stubs which when making use of the library will be used by your IDE to provide code completion. The following 
+commands are provided for development purposes:
+1. `yarn build` - will build the Typescript stubs and the bundle into the `dist` directory.
+2. `yarn test` - runs the test suite which will check various helpers are still functioning as
+expected and a few expected use cases.
+3. `yarn dev:builder` - an internal command which is used by lerna to watch and build any changes
+to the package as part of the main `yarn dev` of the repo.
+
+It is also important to note this package is managed in the same manner as all other in the mono-repo,
+through lerna.
\ No newline at end of file
diff --git a/packages/string-templates/src/helpers/external.js b/packages/string-templates/src/helpers/external.js
index 7c767db5f4..a137c25cef 100644
--- a/packages/string-templates/src/helpers/external.js
+++ b/packages/string-templates/src/helpers/external.js
@@ -13,7 +13,7 @@ const EXTERNAL_FUNCTION_COLLECTIONS = [
   "number",
   "url",
   "string",
-  "markdown",
+  "comparison",
 ]
 
 const DATE_NAME = "date"
diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js
index 0bb3abf806..529bb35cfd 100644
--- a/packages/string-templates/src/index.js
+++ b/packages/string-templates/src/index.js
@@ -2,7 +2,7 @@ const handlebars = require("handlebars")
 const { registerAll } = require("./helpers/index")
 const processors = require("./processors")
 const { cloneDeep } = require("lodash/fp")
-const { removeNull } = require("./utilities")
+const { removeNull, addConstants } = require("./utilities")
 
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
@@ -82,16 +82,15 @@ module.exports.processObjectSync = (object, context) => {
  * @returns {string} The enriched string, all templates should have been replaced if they can be.
  */
 module.exports.processStringSync = (string, context) => {
-  const clonedContext = removeNull(cloneDeep(context))
+  let clonedContext = removeNull(cloneDeep(context))
+  clonedContext = addConstants(clonedContext)
   // remove any null/undefined properties
   if (typeof string !== "string") {
     throw "Cannot process non-string types."
   }
-
-  let template
   string = processors.preprocess(string)
   // this does not throw an error when template can't be fulfilled, have to try correct beforehand
-  template = hbsInstance.compile(string)
+  const template = hbsInstance.compile(string)
   return processors.postprocess(template(clonedContext))
 }
 
diff --git a/packages/string-templates/src/processors/preprocessor.js b/packages/string-templates/src/processors/preprocessor.js
index a7c5891f58..8c2f9f7c56 100644
--- a/packages/string-templates/src/processors/preprocessor.js
+++ b/packages/string-templates/src/processors/preprocessor.js
@@ -1,9 +1,11 @@
 const { HelperNames } = require("../helpers")
 const { swapStrings, isAlphaNumeric } = require("../utilities")
 
+const FUNCTION_CASES = ["#", "else", "/"]
+
 const PreprocessorNames = {
   SWAP_TO_DOT: "swap-to-dot-notation",
-  HANDLE_SPACES: "handle-spaces-in-properties",
+  FIX_FUNCTIONS: "fix-functions",
   FINALISE: "finalise",
 }
 
@@ -37,6 +39,14 @@ module.exports.processors = [
     return statement
   }),
 
+  new Preprocessor(PreprocessorNames.FIX_FUNCTIONS, statement => {
+    for (let specialCase of FUNCTION_CASES) {
+      const toFind = `{ ${specialCase}`, replacement = `{${specialCase}`
+      statement = statement.replace(new RegExp(toFind, "g"), replacement)
+    }
+    return statement
+  }),
+
   new Preprocessor(PreprocessorNames.FINALISE, statement => {
     let insideStatement = statement.slice(2, statement.length - 2)
     if (insideStatement.charAt(0) === " ") {
@@ -46,7 +56,13 @@ module.exports.processors = [
       insideStatement = insideStatement.slice(0, insideStatement.length - 1)
     }
     const possibleHelper = insideStatement.split(" ")[0]
-    if (HelperNames().some(option => possibleHelper === option)) {
+    // function helpers can't be wrapped
+    for (let specialCase of FUNCTION_CASES) {
+      if (possibleHelper.includes(specialCase)) {
+        return statement
+      }
+    }
+    if (HelperNames().some(option => possibleHelper.includes(option))) {
       insideStatement = `(${insideStatement})`
     }
     return `{{ all ${insideStatement} }}`
diff --git a/packages/string-templates/src/utilities.js b/packages/string-templates/src/utilities.js
index 71b5aa1875..45064b9a8b 100644
--- a/packages/string-templates/src/utilities.js
+++ b/packages/string-templates/src/utilities.js
@@ -1,3 +1,4 @@
+const _ = require("lodash")
 const ALPHA_NUMERIC_REGEX = /^[A-Za-z0-9]+$/g
 
 module.exports.FIND_HBS_REGEX = /{{([^{}])+}}/g
@@ -12,12 +13,19 @@ module.exports.swapStrings = (string, start, length, swap) => {
 
 // removes null and undefined
 module.exports.removeNull = obj => {
-  return Object.fromEntries(
-    Object.entries(obj)
-      .filter(entry => entry[1] != null)
-      .map(([key, value]) => [
-        key,
-        value === Object(value) ? module.exports.removeNull(value) : value,
-      ])
-  )
+  obj = _(obj).omitBy(_.isUndefined).omitBy(_.isNull).value()
+  for (let [key, value] of Object.entries(obj)) {
+    // only objects
+    if (typeof value === "object" && !Array.isArray(value)) {
+      obj[key] = module.exports.removeNull(value)
+    }
+  }
+  return obj
+}
+
+module.exports.addConstants = obj => {
+  if (obj.now == null) {
+    obj.now = (new Date()).toISOString()
+  }
+  return obj
 }
diff --git a/packages/string-templates/test/helpers.spec.js b/packages/string-templates/test/helpers.spec.js
index b1a9748409..c85cd63f58 100644
--- a/packages/string-templates/test/helpers.spec.js
+++ b/packages/string-templates/test/helpers.spec.js
@@ -9,5 +9,253 @@ describe("test the custom helpers we have applied", () => {
     })
     expect(output).toBe("object is {\"a\":1}")
   })
-
 })
+
+describe("test the math helpers", () => {
+  it("should be able to produce an absolute", async () => {
+    const output = await processString("{{abs a}}", {
+      a: -10,
+    })
+    expect(parseInt(output)).toBe(10)
+  })
+
+  it("should be able to add", async () => {
+    const output = await processString("{{add a b}}", {
+      a: 10,
+      b: 10,
+    })
+    expect(parseInt(output)).toBe(20)
+  })
+
+  it("should be able to average", async () => {
+    const output = await processString("{{avg a b c}}", {
+      a: 1,
+      b: 2,
+      c: 3,
+    })
+    expect(parseInt(output)).toBe(2)
+  })
+})
+
+describe("test the array helpers", () => {
+  const array = ["hi", "person", "how", "are", "you"]
+  it("should allow use of the after helper", async () => {
+    const output = await processString("{{after array 1}}", {
+      array,
+    })
+    expect(output).toBe("person,how,are,you")
+  })
+
+  it("should allow use of the before helper", async () => {
+    const output = await processString("{{before array 2}}", {
+      array,
+    })
+    expect(output).toBe("hi,person,how")
+  })
+
+  it("should allow use of the filter helper", async () => {
+    const output = await processString("{{#filter array \"person\"}}THING{{else}}OTHER{{/filter}}", {
+      array,
+    })
+    expect(output).toBe("THING")
+  })
+
+  it("should allow use of the itemAt helper", async () => {
+    const output = await processString("{{itemAt array 1}}", {
+      array,
+    })
+    expect(output).toBe("person")
+  })
+
+  it("should allow use of the join helper", async () => {
+    const output = await processString("{{join array \"-\"}}", {
+      array,
+    })
+    expect(output).toBe("hi-person-how-are-you")
+  })
+
+  it("should allow use of the sort helper", async () => {
+    const output = await processString("{{sort array}}", {
+      array: ["d", "a", "c", "e"]
+    })
+    expect(output).toBe("a,c,d,e")
+  })
+
+  it("should allow use of the unique helper", async () => {
+    const output = await processString("{{unique array}}", {
+      array: ["a", "a", "b"]
+    })
+    expect(output).toBe("a,b")
+  })
+
+  it("should allow a complex case", async () => {
+    const output = await processString("{{ last ( sort ( unique array ) ) }}", {
+      array: ["a", "a", "d", "c", "e"]
+    })
+    expect(output).toBe("e")
+  })
+})
+
+describe("test the number helpers", () => {
+  it("should allow use of the addCommas helper", async () => {
+    const output = await processString("{{ addCommas number }}", {
+      number: 10000000
+    })
+    expect(output).toBe("10,000,000")
+  })
+
+  it("should allow use of the phoneNumber helper", async () => {
+    const output = await processString("{{ phoneNumber number }}", {
+      number: 4490102030,
+    })
+    expect(output).toBe("(449) 010-2030")
+  })
+
+  it("should allow use of the toPrecision helper", async () => {
+    const output = await processString("{{ toPrecision number 2 }}", {
+      number: 1.222222222,
+    })
+    expect(output).toBe("1.2")
+  })
+
+  it("should allow use of the bytes helper", async () => {
+    const output = await processString("{{ bytes number }}", {
+      number: 1000000,
+    })
+    expect(output).toBe("1 MB")
+  })
+})
+
+describe("test the url helpers", () => {
+  const url = "http://example.com?query=1"
+  it("should allow use of the stripQueryString helper", async () => {
+    const output = await processString('{{stripQuerystring url }}', {
+      url,
+    })
+    expect(output).toBe("http://example.com")
+  })
+
+  it("should allow use of the stripProtocol helper", async () => {
+    const output = await processString("{{ stripProtocol url }}", {
+      url,
+    })
+    expect(output).toBe("//example.com/?query=1")
+  })
+
+  it("should allow use of the urlParse helper", async () => {
+    const output = await processString("{{ object ( urlParse url ) }}", {
+      url,
+    })
+    expect(output).toBe("{\"protocol\":\"http:\",\"slashes\":true,\"auth\":null,\"host\":\"example.com\"," +
+      "\"port\":null,\"hostname\":\"example.com\",\"hash\":null,\"search\":\"?query=1\"," +
+      "\"query\":\"query=1\",\"pathname\":\"/\",\"path\":\"/?query=1\"," +
+      "\"href\":\"http://example.com/?query=1\"}")
+  })
+})
+
+describe("test the date helpers", () => {
+  it("should allow use of the date helper", async () => {
+    const date = new Date(1611577535000)
+    const output = await processString("{{ date time 'YYYY-MM-DD' }}", {
+      time: date.toISOString(),
+    })
+    expect(output).toBe("2021-01-25")
+  })
+
+  it("should allow use of the date helper with now time", async () => {
+    const date = new Date()
+    const output = await processString("{{ date now 'DD' }}", {})
+    expect(output).toBe(date.getDate().toString())
+  })
+})
+
+describe("test the string helpers", () => {
+  it("should allow use of the append helper", async () => {
+    const output = await processString("{{ append filename '.txt' }}", {
+      filename: "yummy",
+    })
+    expect(output).toBe("yummy.txt")
+  })
+
+  it("should allow use of the camelcase helper", async () => {
+    const output = await processString("{{ camelcase camel }}", {
+      camel: "testing this thing",
+    })
+    expect(output).toBe("testingThisThing")
+  })
+
+  it("should allow use of the capitalize helper", async () => {
+    const output = await processString("{{ capitalize string }}", {
+      string: "this is a string",
+    })
+    expect(output).toBe("This is a string")
+  })
+
+  it("should allow use of the capitalizeAll helper", async () => {
+    const output = await processString("{{ capitalizeAll string }}", {
+      string: "this is a string",
+    })
+    expect(output).toBe("This Is A String")
+  })
+
+  it("should allow use of the replace helper", async () => {
+    const output = await processString("{{ replace string 'Mike' name }}", {
+      string: "Hello my name is Mike",
+      name: "David",
+    })
+    expect(output).toBe("Hello my name is David")
+  })
+
+  it("should allow use of the split helper", async () => {
+    const output = await processString("{{ first ( split string ' ' ) }}", {
+      string: "this is a string",
+    })
+    expect(output).toBe("this")
+  })
+
+  it("should allow use of the remove helper", async () => {
+    const output = await processString("{{ remove string 'string' }}", {
+      string: "this is a string",
+    })
+    expect(output).toBe("this is a ")
+  })
+
+  it("should allow use of the startsWith helper", async () => {
+    const output = await processString("{{ #startsWith 'Hello' string }}Hi!{{ else }}Goodbye!{{ /startsWith }}", {
+      string: "Hello my name is Mike",
+    })
+    expect(output).toBe("Hi!")
+  })
+})
+
+describe("test the comparison helpers", () => {
+  async function compare(func, a, b) {
+    const output = await processString(`{{ #${func} a b }}Success{{ else }}Fail{{ /${func} }}`, {
+      a,
+      b,
+    })
+    expect(output).toBe("Success")
+  }
+  it("should allow use of the lt helper", async () => {
+    await compare("lt", 10, 15)
+  })
+
+  it("should allow use of the gt helper", async () => {
+    await compare("gt", 15, 10)
+  })
+
+  it("should allow use of the and helper", async () => {
+    await compare("and", true, true)
+  })
+
+  it("should allow use of the or helper", async () => {
+    await compare("or", false, true)
+  })
+
+  it("should allow use of gte with a literal value", async () => {
+    const output = await processString(`{{ #gte a "50" }}s{{ else }}f{{ /gte }}`, {
+      a: 51,
+    })
+    expect(output).toBe("s")
+  })
+})
\ No newline at end of file