From aaa5c1841f1c0fd7008184fe356513c717fbd935 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 14 Dec 2021 12:30:26 +0000
Subject: [PATCH 01/11] Variables UI WIP

---
 .../rest/RestExtraConfigForm.svelte           | 37 +++++++++-
 .../variables/ViewDynamicVariables.svelte     | 40 +++++++++++
 .../integration/KeyValueBuilder.svelte        | 28 +++++++-
 .../[selectedDatasource]/index.svelte         |  6 +-
 .../rest/[query]/index.svelte                 | 69 ++++++++++++++++++-
 5 files changed, 175 insertions(+), 5 deletions(-)
 create mode 100644 packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
index 0272463d92..8a4f49523f 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
@@ -1,9 +1,18 @@
 <script>
-  import { Divider, Heading, ActionButton, Badge, Body } from "@budibase/bbui"
+  import {
+    Divider,
+    Heading,
+    ActionButton,
+    Badge,
+    Body,
+    Layout,
+  } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import RestAuthenticationBuilder from "./auth/RestAuthenticationBuilder.svelte"
+  import ViewDynamicVariables from "./variables/ViewDynamicVariables.svelte"
 
   export let datasource
+  export let queries
 
   let addHeader
 </script>
@@ -43,6 +52,32 @@
 </Body>
 <RestAuthenticationBuilder bind:configs={datasource.config.authConfigs} />
 
+<Divider size="S" />
+<div class="section-header">
+  <div class="badge">
+    <Heading size="S">Variables</Heading>
+    <Badge quiet grey>Optional</Badge>
+  </div>
+</div>
+<Body size="S"
+  >Variables enabled you to store and reuse values in queries. Static variables
+  use constant values while dynamic values can be bound to the response headers
+  or body of a query</Body
+>
+<Heading size="XS">Static</Heading>
+<Layout noPadding gap="XS">
+  <KeyValueBuilder
+    name="Variable"
+    keyPlaceholder="Name"
+    headings
+    bind:object={datasource.config.staticVariables}
+    on:change
+  />
+</Layout>
+<div />
+<Heading size="XS">Dynamic</Heading>
+<ViewDynamicVariables {queries} {datasource} />
+
 <style>
   .section-header {
     display: flex;
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte
new file mode 100644
index 0000000000..150e59ff64
--- /dev/null
+++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte
@@ -0,0 +1,40 @@
+<script>
+  import { Body, Table } from "@budibase/bbui"
+  import { onMount } from "svelte"
+
+  export let datasource
+  export let queries
+
+  let dynamicVariables = []
+
+  const dynamicVariableSchema = {
+    name: "",
+    value: "",
+    query: "",
+  }
+
+  /**
+   * Add the query name to the dynamic variables
+   */
+  const enrichDynamicVariables = () => {
+    datasource.config.dynamicVariables?.forEach(dv => {
+      const query = queries.find(query => query._id === dv.queryId)
+      if (query) {
+        dynamicVariables.push({ ...dv, query: query.name })
+      }
+    })
+  }
+
+  onMount(() => {
+    enrichDynamicVariables()
+  })
+</script>
+
+<Table
+  schema={dynamicVariableSchema}
+  data={dynamicVariables}
+  allowEditColumns={false}
+  allowEditRows={false}
+  allowSelectRows={false}
+/>
+<Body size="S" />
diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte
index bde1dc256a..7baa2724b5 100644
--- a/packages/builder/src/components/integration/KeyValueBuilder.svelte
+++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte
@@ -6,6 +6,8 @@
     Label,
     Toggle,
     Select,
+    ActionMenu,
+    MenuItem,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { lowercase } from "helpers"
@@ -23,7 +25,9 @@
   export let toggle
   export let keyPlaceholder = "Key"
   export let valuePlaceholder = "Value"
+  export let valueHeading
   export let tooltip
+  export let menuItems
 
   let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
   let fieldActivity = []
@@ -80,13 +84,18 @@
   {#if headings}
     <div class="container" class:container-active={toggle}>
       <Label {tooltip}>{keyPlaceholder}</Label>
-      <Label>{valuePlaceholder}</Label>
+      <Label>{valueHeading || valuePlaceholder}</Label>
       {#if toggle}
         <Label>Active</Label>
       {/if}
     </div>
   {/if}
-  <div class="container" class:container-active={toggle} class:readOnly>
+  <div
+    class="container"
+    class:container-active={toggle}
+    class:container-menu={menuItems}
+    class:readOnly
+  >
     {#each fields as field, idx}
       <Input
         placeholder={keyPlaceholder}
@@ -112,6 +121,18 @@
       {#if !readOnly}
         <Icon hoverable name="Close" on:click={() => deleteEntry(idx)} />
       {/if}
+      {#if menuItems?.length > 0}
+        <ActionMenu>
+          <div slot="control" class="icon">
+            <Icon size="S" hoverable name="MoreSmallList" />
+          </div>
+          {#each menuItems as item}
+            <MenuItem on:click={item.onClick}>
+              {item.text}
+            </MenuItem>
+          {/each}
+        </ActionMenu>
+      {/if}
     {/each}
   </div>
 {/if}
@@ -134,6 +155,9 @@
   .container-active {
     grid-template-columns: 1fr 1fr 50px 20px;
   }
+  .container-menu {
+    grid-template-columns: 1fr 1fr 20px 20px;
+  }
   .readOnly {
     grid-template-columns: 1fr 1fr;
   }
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
index 87ff9691f8..697b7eb4a6 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
@@ -142,7 +142,11 @@
         </div>
       {/if}
       {#if datasource?.source === IntegrationTypes.REST}
-        <RestExtraConfigForm bind:datasource on:change={hasChanged} />
+        <RestExtraConfigForm
+          queries={queryList}
+          bind:datasource
+          on:change={hasChanged}
+        />
       {/if}
     </Layout>
   </section>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
index 56b51e5158..b506450101 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
@@ -51,6 +51,7 @@
   let response, schema, isGet
   let datasourceType, integrationInfo, queryConfig, responseSuccess
   let authConfigId
+  let dynamicVariables
 
   $: datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
   $: datasourceType = datasource?.source
@@ -62,6 +63,7 @@
   $: responseSuccess =
     response?.info?.code >= 200 && response?.info?.code <= 206
   $: authConfigs = buildAuthConfigs(datasource)
+  $: dynamicVariables = buildDynamicVariables(datasource)
 
   function getSelectedQuery() {
     return cloneDeep(
@@ -125,8 +127,14 @@
       saveId = _id
       query = getSelectedQuery()
       notifications.success(`Request saved successfully.`)
+
+      if (dynamicVariables) {
+        const dynamicVars = mapDynamicVariables(saveId)
+        datasource.config.dynamicVariables = dynamicVars
+        await datasources.save(datasource)
+      }
     } catch (err) {
-      notifications.error(`Error creating query. ${err.message}`)
+      notifications.error(`Error saving query. ${err.message}`)
     }
   }
 
@@ -181,6 +189,45 @@
       query.fields.bodyType = "none"
     }
   })
+
+  /**
+   * Convert the dynamic variables list to a simple name / value object
+   */
+  const buildDynamicVariables = datasource => {
+    const variablesList = datasource?.config?.dynamicVariables
+    let variables = {}
+    if (variablesList) {
+      variables = variablesList.reduce(
+        (acc, next) => ({ ...acc, [next.name]: next.value }),
+        {}
+      )
+    }
+    return variables
+  }
+
+  /**
+   * Convert the dynamic variables object back to a list
+   */
+  const mapDynamicVariables = queryId => {
+    let variables = []
+    if (dynamicVariables) {
+      variables = Object.values(dynamicVariables).map((name, value) => ({
+        name,
+        value,
+        queryId,
+      }))
+    }
+    return variables
+  }
+
+  const schemaMenuItems = [
+    {
+      text: "Create dynamic variable",
+      onClick: () => {
+        console.log("create variable")
+      },
+    },
+  ]
 </script>
 
 {#if query}
@@ -318,6 +365,7 @@
                   name="schema"
                   headings
                   options={SchemaTypeOptions}
+                  menuItems={schemaMenuItems}
                 />
               </Tab>
             {/if}
@@ -341,6 +389,25 @@
                   {/if}
                 </div>
               </Tab>
+            {/if}
+            {#if dynamicVariables || response}
+              <Tab title="Dynamic Variables">
+                <Layout noPadding gap="S">
+                  <Body size="S"
+                    >{"Create dynamic variables to use body and headers results in other queries"}</Body
+                  >
+                  <KeyValueBuilder
+                    bind:object={dynamicVariables}
+                    name="Variable"
+                    headings
+                    keyPlaceholder="Name"
+                    valuePlaceholder={`e.g. {{ headers.cookie }}`}
+                    valueHeading={`Value`}
+                  />
+                </Layout>
+              </Tab>
+            {/if}
+            {#if response}
               <div class="stats">
                 <Label size="L">
                   Status: <span class={responseSuccess ? "green" : "red"}

From 4c649e60c750a52d84ccd15de66a9f4db1d0dd4b Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 14 Dec 2021 16:46:31 +0000
Subject: [PATCH 02/11] Read / Write finished

---
 .../integration/KeyValueBuilder.svelte        |   4 +-
 .../rest/[query]/index.svelte                 | 100 ++++++++++--------
 2 files changed, 59 insertions(+), 45 deletions(-)

diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte
index 67e52393f0..700e3cd8cb 100644
--- a/packages/builder/src/components/integration/KeyValueBuilder.svelte
+++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte
@@ -96,6 +96,7 @@
     {#each fields as field, idx}
       <Input
         placeholder={keyPlaceholder}
+        readonly={readOnly}
         bind:value={field.name}
         on:change={changed}
       />
@@ -104,6 +105,7 @@
       {:else}
         <Input
           placeholder={valuePlaceholder}
+          readonly={readOnly}
           bind:value={field.value}
           on:change={changed}
         />
@@ -114,7 +116,7 @@
       {#if !readOnly}
         <Icon hoverable name="Close" on:click={() => deleteEntry(idx)} />
       {/if}
-      {#if menuItems?.length > 0}
+      {#if menuItems?.length > 0 && !readOnly}
         <ActionMenu>
           <div slot="control" class="icon">
             <Icon size="S" hoverable name="MoreSmallList" />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
index 1523a356e8..af1f629abf 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
@@ -59,10 +59,11 @@
   $: queryConfig = integrationInfo?.query
   $: url = buildUrl(url, breakQs)
   $: checkQueryName(url)
-  $: responseSuccess =
-    response?.info?.code >= 200 && response?.info?.code <= 206
+  $: responseSuccess = response?.info?.code >= 200 && response?.info?.code < 400
   $: authConfigs = buildAuthConfigs(datasource)
-  $: dynamicVariables = buildDynamicVariables(datasource)
+  $: schemaReadOnly = !responseSuccess
+  $: variablesReadOnly = !responseSuccess
+  $: showVariablesTab = shouldShowVariables(dynamicVariables, variablesReadOnly)
 
   function getSelectedQuery() {
     return cloneDeep(
@@ -133,7 +134,7 @@
       notifications.success(`Request saved successfully.`)
 
       if (dynamicVariables) {
-        const dynamicVars = mapDynamicVariables(saveId)
+        const dynamicVars = rebuildVariables(saveId)
         datasource.config.dynamicVariables = dynamicVars
         await datasources.save(datasource)
       }
@@ -173,6 +174,53 @@
     return id
   }
 
+  // convert dynamic variables list to simple key/val object
+  const variablesToObject = datasource => {
+    const variablesList = datasource?.config?.dynamicVariables
+    if (variablesList && variablesList.length > 0) {
+      return variablesList.reduce(
+        (acc, next) => ({ ...acc, [next.name]: next.value }),
+        {}
+      )
+    }
+    return {}
+  }
+
+  // convert dynamic variables object back to a list, enrich with query id
+  const rebuildVariables = queryId => {
+    let variables = []
+    if (dynamicVariables) {
+      variables = Object.entries(dynamicVariables).map(entry => {
+        return {
+          name: entry[0],
+          value: entry[1],
+          queryId,
+        }
+      })
+    }
+    return variables
+  }
+
+  const shouldShowVariables = (dynamicVariables, variablesReadOnly) => {
+    if (
+      dynamicVariables &&
+      // show when editable or when read only and not empty
+      (!variablesReadOnly || Object.keys(dynamicVariables).length > 0)
+    ) {
+      return true
+    }
+    return false
+  }
+
+  const schemaMenuItems = [
+    {
+      text: "Create dynamic variable",
+      onClick: () => {
+        console.log("create variable")
+      },
+    },
+  ]
+
   onMount(async () => {
     query = getSelectedQuery()
     // clear any unsaved changes to the datasource
@@ -210,46 +258,8 @@
     if (query && !query.fields.bodyType) {
       query.fields.bodyType = "none"
     }
+    dynamicVariables = variablesToObject(datasource)
   })
-
-  /**
-   * Convert the dynamic variables list to a simple name / value object
-   */
-  const buildDynamicVariables = datasource => {
-    const variablesList = datasource?.config?.dynamicVariables
-    let variables = {}
-    if (variablesList) {
-      variables = variablesList.reduce(
-        (acc, next) => ({ ...acc, [next.name]: next.value }),
-        {}
-      )
-    }
-    return variables
-  }
-
-  /**
-   * Convert the dynamic variables object back to a list
-   */
-  const mapDynamicVariables = queryId => {
-    let variables = []
-    if (dynamicVariables) {
-      variables = Object.values(dynamicVariables).map((name, value) => ({
-        name,
-        value,
-        queryId,
-      }))
-    }
-    return variables
-  }
-
-  const schemaMenuItems = [
-    {
-      text: "Create dynamic variable",
-      onClick: () => {
-        console.log("create variable")
-      },
-    },
-  ]
 </script>
 
 {#if query && queryConfig}
@@ -388,6 +398,7 @@
                   headings
                   options={SchemaTypeOptions}
                   menuItems={schemaMenuItems}
+                  readOnly={schemaReadOnly}
                 />
               </Tab>
             {/if}
@@ -412,7 +423,7 @@
                 </div>
               </Tab>
             {/if}
-            {#if dynamicVariables || response}
+            {#if showVariablesTab}
               <Tab title="Dynamic Variables">
                 <Layout noPadding gap="S">
                   <Body size="S"
@@ -425,6 +436,7 @@
                     keyPlaceholder="Name"
                     valuePlaceholder={`e.g. {{ headers.cookie }}`}
                     valueHeading={`Value`}
+                    readOnly={variablesReadOnly}
                   />
                 </Layout>
               </Tab>

From 242fe21429766c0496ebff76a9d9238835b9933b Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 14 Dec 2021 16:58:17 +0000
Subject: [PATCH 03/11] Add menu to response heades

---
 .../integration/KeyValueBuilder.svelte           |  9 +++++++--
 .../rest/[query]/index.svelte                    | 16 +++++++++++++++-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte
index 700e3cd8cb..96024782eb 100644
--- a/packages/builder/src/components/integration/KeyValueBuilder.svelte
+++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte
@@ -28,6 +28,7 @@
   export let valueHeading
   export let tooltip
   export let menuItems
+  export let showMenu = false
 
   let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
   let fieldActivity = buildFieldActivity(activity)
@@ -90,8 +91,9 @@
   <div
     class="container"
     class:container-active={toggle}
-    class:container-menu={menuItems}
+    class:container-menu={showMenu}
     class:readOnly
+    class:readOnly-menu={readOnly && showMenu}
   >
     {#each fields as field, idx}
       <Input
@@ -116,7 +118,7 @@
       {#if !readOnly}
         <Icon hoverable name="Close" on:click={() => deleteEntry(idx)} />
       {/if}
-      {#if menuItems?.length > 0 && !readOnly}
+      {#if menuItems?.length > 0 && showMenu}
         <ActionMenu>
           <div slot="control" class="icon">
             <Icon size="S" hoverable name="MoreSmallList" />
@@ -156,4 +158,7 @@
   .readOnly {
     grid-template-columns: 1fr 1fr;
   }
+  .readOnly-menu {
+    grid-template-columns: 1fr 1fr 20px;
+  }
 </style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
index af1f629abf..735c98d567 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
@@ -220,6 +220,14 @@
       },
     },
   ]
+  const responseHeadersMenuItems = [
+    {
+      text: "Create dynamic variable",
+      onClick: () => {
+        console.log("create variable")
+      },
+    },
+  ]
 
   onMount(async () => {
     query = getSelectedQuery()
@@ -398,6 +406,7 @@
                   headings
                   options={SchemaTypeOptions}
                   menuItems={schemaMenuItems}
+                  showMenu={!schemaReadOnly}
                   readOnly={schemaReadOnly}
                 />
               </Tab>
@@ -407,7 +416,12 @@
                 <TextArea disabled value={response.extra?.raw} height="300" />
               </Tab>
               <Tab title="Headers">
-                <KeyValueBuilder object={response.extra?.headers} readOnly />
+                <KeyValueBuilder
+                  object={response.extra?.headers}
+                  readOnly
+                  menuItems={responseHeadersMenuItems}
+                  showMenu={true}
+                />
               </Tab>
               <Tab title="Preview">
                 <div class="table">

From 91cce4cb0fbc27493e72c750950feb906cf13d9a Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 14 Dec 2021 17:05:17 +0000
Subject: [PATCH 04/11] Link datasource dynamic variables table to query

---
 .../rest/variables/ViewDynamicVariables.svelte           | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte
index 150e59ff64..b778b950be 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte
@@ -1,6 +1,8 @@
 <script>
   import { Body, Table } from "@budibase/bbui"
   import { onMount } from "svelte"
+  import { queries as queriesStore } from "stores/backend"
+  import { goto } from "@roxi/routify"
 
   export let datasource
   export let queries
@@ -13,6 +15,12 @@
     query: "",
   }
 
+  const onClick = dynamicVariable => {
+    const queryId = dynamicVariable.queryId
+    queriesStore.select({ _id: queryId })
+    $goto(`./${queryId}`)
+  }
+
   /**
    * Add the query name to the dynamic variables
    */
@@ -31,6 +39,7 @@
 </script>
 
 <Table
+  on:click={({ detail }) => onClick(detail)}
   schema={dynamicVariableSchema}
   data={dynamicVariables}
   allowEditColumns={false}

From 41edc72f0ffb59b8321acd9fa9688eac7d2de77c Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Wed, 15 Dec 2021 14:37:03 +0000
Subject: [PATCH 05/11] Copy changes

---
 .../TableIntegrationMenu/rest/RestExtraConfigForm.svelte     | 4 ++++
 .../src/components/integration/KeyValueBuilder.svelte        | 3 ++-
 .../[selectedDatasource]/rest/[query]/index.svelte           | 5 +++--
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
index 8a4f49523f..4a104a1987 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
@@ -76,6 +76,10 @@
 </Layout>
 <div />
 <Heading size="XS">Dynamic</Heading>
+<Body size="S">
+  Dynamic variables are evaluated when a dependant query is executed. The value
+  is cached for 24 hours and will re-evaluate if the dependendent query fails.
+</Body>
 <ViewDynamicVariables {queries} {datasource} />
 
 <style>
diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte
index 96024782eb..b8d0e0a467 100644
--- a/packages/builder/src/components/integration/KeyValueBuilder.svelte
+++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte
@@ -26,6 +26,7 @@
   export let keyPlaceholder = "Key"
   export let valuePlaceholder = "Value"
   export let valueHeading
+  export let keyHeading
   export let tooltip
   export let menuItems
   export let showMenu = false
@@ -81,7 +82,7 @@
 {#if Object.keys(object || {}).length > 0}
   {#if headings}
     <div class="container" class:container-active={toggle}>
-      <Label {tooltip}>{keyPlaceholder}</Label>
+      <Label {tooltip}>{keyHeading || keyPlaceholder}</Label>
       <Label>{valueHeading || valuePlaceholder}</Label>
       {#if toggle}
         <Label>Active</Label>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
index 735c98d567..54adacbe36 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
@@ -447,9 +447,10 @@
                     bind:object={dynamicVariables}
                     name="Variable"
                     headings
-                    keyPlaceholder="Name"
-                    valuePlaceholder={`e.g. {{ headers.cookie }}`}
+                    keyHeading="Name"
+                    keyPlaceholder="e.g. cookie"
                     valueHeading={`Value`}
+                    valuePlaceholder={`e.g. {{ headers.set-cookie }}`}
                     readOnly={variablesReadOnly}
                   />
                 </Layout>

From 0a6754b13c7a86e0867cc4695a8abc147fe19dd7 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 15 Dec 2021 19:20:19 +0000
Subject: [PATCH 06/11] Finishing up front-end, getting variable creation and
 management up and working from within the query schema/header menus.

---
 .../integration/KeyValueBuilder.svelte        |   5 +-
 packages/builder/src/helpers/data/utils.js    |  59 +++++++
 .../_components/DynamicVariableModal.svelte   |  46 ++++++
 .../rest/[query]/index.svelte                 | 153 +++++++-----------
 packages/client/manifest.json                 |   8 +
 packages/client/src/api/datasources.js        |  12 +-
 .../src/components/app/DataProvider.svelte    |   6 +-
 .../server/src/api/controllers/query/index.js |   4 +-
 .../src/automations/steps/executeQuery.js     |  10 +-
 9 files changed, 196 insertions(+), 107 deletions(-)
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte

diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte
index b8d0e0a467..7229712a65 100644
--- a/packages/builder/src/components/integration/KeyValueBuilder.svelte
+++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte
@@ -125,7 +125,7 @@
             <Icon size="S" hoverable name="MoreSmallList" />
           </div>
           {#each menuItems as item}
-            <MenuItem on:click={item.onClick}>
+            <MenuItem on:click={() => item.onClick(field)}>
               {item.text}
             </MenuItem>
           {/each}
@@ -162,4 +162,7 @@
   .readOnly-menu {
     grid-template-columns: 1fr 1fr 20px;
   }
+  .control {
+    margin-top: 4px;
+  }
 </style>
diff --git a/packages/builder/src/helpers/data/utils.js b/packages/builder/src/helpers/data/utils.js
index 613f533043..e163043470 100644
--- a/packages/builder/src/helpers/data/utils.js
+++ b/packages/builder/src/helpers/data/utils.js
@@ -119,3 +119,62 @@ export function flipHeaderState(headersActivity) {
   })
   return enabled
 }
+
+// convert dynamic variables list to simple key/val object
+export function variablesToObject(datasource) {
+  const variablesList = datasource?.config?.dynamicVariables
+  if (variablesList && variablesList.length > 0) {
+    return variablesList.reduce(
+      (acc, next) => ({ ...acc, [next.name]: next.value }),
+      {}
+    )
+  }
+  return {}
+}
+
+// convert dynamic variables object back to a list, enrich with query id
+export function rebuildVariables(queryId, variables) {
+  let vars = []
+  if (variables) {
+    vars = Object.entries(variables).map(entry => {
+      return {
+        name: entry[0],
+        value: entry[1],
+        queryId,
+      }
+    })
+  }
+  return vars
+}
+
+export function shouldShowVariables(dynamicVariables, variablesReadOnly) {
+  return !!(
+    dynamicVariables &&
+    // show when editable or when read only and not empty
+    (!variablesReadOnly || Object.keys(dynamicVariables).length > 0)
+  )
+}
+
+export function buildAuthConfigs(datasource) {
+  if (datasource?.config?.authConfigs) {
+    return datasource.config.authConfigs.map(c => ({
+      label: c.name,
+      value: c._id,
+    }))
+  }
+  return []
+}
+
+export default {
+  breakQueryString,
+  buildQueryString,
+  fieldsToSchema,
+  flipHeaderState,
+  keyValueToQueryParameters,
+  queryParametersToKeyValue,
+  schemaToFields,
+  variablesToObject,
+  rebuildVariables,
+  shouldShowVariables,
+  buildAuthConfigs,
+}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte
new file mode 100644
index 0000000000..dae6804e25
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte
@@ -0,0 +1,46 @@
+<script>
+  import { Input, ModalContent, Modal } from "@budibase/bbui"
+
+  export let datasource
+  export let dynamicVariables
+  export let binding
+
+  let name, modal
+
+  export const show = () => {
+    modal.show()
+  }
+  export const hide = () => {
+    modal.hide()
+  }
+
+  function checkValid(vars, name) {
+    if (!name) {
+      return false
+    }
+    const varKeys = Object.keys(vars || {})
+    return varKeys.find(key => key.toLowerCase() === name.toLowerCase()) == null
+  }
+
+  $: valid = checkValid(dynamicVariables, name)
+  $: error = name && !valid ? "Variable name is already in use." : null
+
+  async function saveVariable() {
+    const copiedName = name,
+      copiedBinding = binding
+    name = null
+    binding = null
+    dynamicVariables[copiedName] = copiedBinding
+  }
+</script>
+
+<Modal bind:this={modal}>
+  <ModalContent
+    title="Add dynamic variable"
+    confirmText="Save"
+    onConfirm={saveVariable}
+    disabled={!valid}
+  >
+    <Input label="Variable name" bind:value={name} on:input {error} />
+  </ModalContent>
+</Modal>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
index 62a418a958..88a460bc8f 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
@@ -1,22 +1,22 @@
 <script>
   import { params } from "@roxi/routify"
-  import { datasources, integrations, queries, flags } from "stores/backend"
+  import { datasources, flags, integrations, queries } from "stores/backend"
   import {
-    Layout,
-    Input,
-    Select,
-    Tabs,
-    Tab,
     Banner,
-    Divider,
-    Button,
-    Heading,
-    RadioGroup,
-    Label,
     Body,
-    TextArea,
-    Table,
+    Button,
+    Divider,
+    Heading,
+    Input,
+    Label,
+    Layout,
     notifications,
+    RadioGroup,
+    Select,
+    Tab,
+    Table,
+    Tabs,
+    TextArea,
   } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import EditableLabel from "components/common/inputs/EditableLabel.svelte"
@@ -26,33 +26,24 @@
   import RestBodyInput from "../../_components/RestBodyInput.svelte"
   import { capitalise } from "helpers"
   import { onMount } from "svelte"
-  import {
-    fieldsToSchema,
-    schemaToFields,
-    breakQueryString,
-    buildQueryString,
-    keyValueToQueryParameters,
-    queryParametersToKeyValue,
-    flipHeaderState,
-  } from "helpers/data/utils"
+  import restUtils from "helpers/data/utils"
   import {
     RestBodyTypes as bodyTypes,
     SchemaTypeOptions,
   } from "constants/backend"
   import JSONPreview from "components/integration/JSONPreview.svelte"
   import AccessLevelSelect from "components/integration/AccessLevelSelect.svelte"
+  import DynamicVariableModal from "../../_components/DynamicVariableModal.svelte"
   import Placeholder from "assets/bb-spaceship.svg"
   import { cloneDeep } from "lodash/fp"
 
   let query, datasource
   let breakQs = {},
     bindings = {}
-  let url = ""
-  let saveId, isGet
+  let saveId, url
   let response, schema, enabledHeaders
-  let datasourceType, integrationInfo, queryConfig, responseSuccess
   let authConfigId
-  let dynamicVariables
+  let dynamicVariables, addVariableModal, varBinding
 
   $: datasourceType = datasource?.source
   $: integrationInfo = $integrations[datasourceType]
@@ -61,10 +52,13 @@
   $: checkQueryName(url)
   $: responseSuccess = response?.info?.code >= 200 && response?.info?.code < 400
   $: isGet = query?.queryVerb === "read"
-  $: authConfigs = buildAuthConfigs(datasource)
+  $: authConfigs = restUtils.buildAuthConfigs(datasource)
   $: schemaReadOnly = !responseSuccess
   $: variablesReadOnly = !responseSuccess
-  $: showVariablesTab = shouldShowVariables(dynamicVariables, variablesReadOnly)
+  $: showVariablesTab = restUtils.shouldShowVariables(
+    dynamicVariables,
+    variablesReadOnly
+  )
 
   function getSelectedQuery() {
     return cloneDeep(
@@ -92,7 +86,7 @@
     if (!base) {
       return base
     }
-    const qs = buildQueryString(qsObj)
+    const qs = restUtils.buildQueryString(qsObj)
     let newUrl = base
     if (base.includes("?")) {
       newUrl = base.split("?")[0]
@@ -100,29 +94,15 @@
     return qs.length > 0 ? `${newUrl}?${qs}` : newUrl
   }
 
-  const buildAuthConfigs = datasource => {
-    if (datasource?.config?.authConfigs) {
-      return datasource.config.authConfigs.map(c => ({
-        label: c.name,
-        value: c._id,
-      }))
-    }
-    return []
-  }
-
-  function learnMoreBanner() {
-    window.open("https://docs.budibase.com/building-apps/data/transformers")
-  }
-
   function buildQuery() {
     const newQuery = { ...query }
-    const queryString = buildQueryString(breakQs)
+    const queryString = restUtils.buildQueryString(breakQs)
     newQuery.fields.path = url.split("?")[0]
     newQuery.fields.queryString = queryString
     newQuery.fields.authConfigId = authConfigId
-    newQuery.fields.disabledHeaders = flipHeaderState(enabledHeaders)
-    newQuery.schema = fieldsToSchema(schema)
-    newQuery.parameters = keyValueToQueryParameters(bindings)
+    newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
+    newQuery.schema = restUtils.fieldsToSchema(schema)
+    newQuery.parameters = restUtils.keyValueToQueryParameters(bindings)
     return newQuery
   }
 
@@ -135,8 +115,10 @@
       notifications.success(`Request saved successfully.`)
 
       if (dynamicVariables) {
-        const dynamicVars = rebuildVariables(saveId)
-        datasource.config.dynamicVariables = dynamicVars
+        datasource.config.dynamicVariables = restUtils.rebuildVariables(
+          saveId,
+          dynamicVariables
+        )
         await datasources.save(datasource)
       }
     } catch (err) {
@@ -175,57 +157,21 @@
     return id
   }
 
-  // convert dynamic variables list to simple key/val object
-  const variablesToObject = datasource => {
-    const variablesList = datasource?.config?.dynamicVariables
-    if (variablesList && variablesList.length > 0) {
-      return variablesList.reduce(
-        (acc, next) => ({ ...acc, [next.name]: next.value }),
-        {}
-      )
-    }
-    return {}
-  }
-
-  // convert dynamic variables object back to a list, enrich with query id
-  const rebuildVariables = queryId => {
-    let variables = []
-    if (dynamicVariables) {
-      variables = Object.entries(dynamicVariables).map(entry => {
-        return {
-          name: entry[0],
-          value: entry[1],
-          queryId,
-        }
-      })
-    }
-    return variables
-  }
-
-  const shouldShowVariables = (dynamicVariables, variablesReadOnly) => {
-    if (
-      dynamicVariables &&
-      // show when editable or when read only and not empty
-      (!variablesReadOnly || Object.keys(dynamicVariables).length > 0)
-    ) {
-      return true
-    }
-    return false
-  }
-
   const schemaMenuItems = [
     {
       text: "Create dynamic variable",
-      onClick: () => {
-        console.log("create variable")
+      onClick: input => {
+        varBinding = `{{ data.0.[${input.name}] }}`
+        addVariableModal.show()
       },
     },
   ]
   const responseHeadersMenuItems = [
     {
       text: "Create dynamic variable",
-      onClick: () => {
-        console.log("create variable")
+      onClick: input => {
+        varBinding = `{{ info.headers.[${input.name}] }}`
+        addVariableModal.show()
       },
     },
   ]
@@ -237,14 +183,14 @@
     datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
     const datasourceUrl = datasource?.config.url
     const qs = query?.fields.queryString
-    breakQs = breakQueryString(qs)
+    breakQs = restUtils.breakQueryString(qs)
     if (datasourceUrl && !query.fields.path?.startsWith(datasourceUrl)) {
       const path = query.fields.path
       query.fields.path = `${datasource.config.url}/${path ? path : ""}`
     }
     url = buildUrl(query.fields.path, breakQs)
-    schema = schemaToFields(query.schema)
-    bindings = queryParametersToKeyValue(query.parameters)
+    schema = restUtils.schemaToFields(query.schema)
+    bindings = restUtils.queryParametersToKeyValue(query.parameters)
     authConfigId = getAuthConfigId()
     if (!query.fields.disabledHeaders) {
       query.fields.disabledHeaders = {}
@@ -255,7 +201,7 @@
         query.fields.disabledHeaders[header] = false
       }
     }
-    enabledHeaders = flipHeaderState(query.fields.disabledHeaders)
+    enabledHeaders = restUtils.flipHeaderState(query.fields.disabledHeaders)
     if (query && !query.transformer) {
       query.transformer = "return data"
     }
@@ -267,10 +213,16 @@
     if (query && !query.fields.bodyType) {
       query.fields.bodyType = "none"
     }
-    dynamicVariables = variablesToObject(datasource)
+    dynamicVariables = restUtils.variablesToObject(datasource)
   })
 </script>
 
+<DynamicVariableModal
+  {datasource}
+  {dynamicVariables}
+  bind:binding={varBinding}
+  bind:this={addVariableModal}
+/>
 {#if query && queryConfig}
   <div class="inner">
     <div class="top">
@@ -340,7 +292,10 @@
               {#if !$flags.queryTransformerBanner}
                 <Banner
                   extraButtonText="Learn more"
-                  extraButtonAction={learnMoreBanner}
+                  extraButtonAction={() =>
+                    window.open(
+                      "https://docs.budibase.com/building-apps/data/transformers"
+                    )}
                   on:change={() =>
                     flags.updateFlag("queryTransformerBanner", true)}
                 >
@@ -449,9 +404,9 @@
                     name="Variable"
                     headings
                     keyHeading="Name"
-                    keyPlaceholder="e.g. cookie"
+                    keyPlaceholder="Variable name"
                     valueHeading={`Value`}
-                    valuePlaceholder={`e.g. {{ headers.set-cookie }}`}
+                    valuePlaceholder={`{{ value }}`}
                     readOnly={variablesReadOnly}
                   />
                 </Layout>
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index de2ebc51e0..b794df814f 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -2457,6 +2457,10 @@
           "label": "Rows",
           "key": "rows"
         },
+	{
+	  "label": "Extra Info",
+	  "key": "info"
+	},
         {
           "label": "Rows Length",
           "key": "rowsLength"
@@ -3178,6 +3182,10 @@
             "label": "Rows",
             "key": "rows"
           },
+	  {
+            "label": "Extra Info",
+            "key": "info"
+          },
           {
             "label": "Rows Length",
             "key": "rowsLength"
diff --git a/packages/client/src/api/datasources.js b/packages/client/src/api/datasources.js
index 981d8301ca..a1d6b8dfd8 100644
--- a/packages/client/src/api/datasources.js
+++ b/packages/client/src/api/datasources.js
@@ -15,7 +15,8 @@ export const fetchDatasource = async dataSource => {
 
   // Fetch all rows in data source
   const { type, tableId, fieldName } = dataSource
-  let rows = []
+  let rows = [],
+    info = {}
   if (type === "table") {
     rows = await fetchTableData(tableId)
   } else if (type === "view") {
@@ -28,7 +29,12 @@ export const fetchDatasource = async dataSource => {
         parameters[param.name] = param.default
       }
     }
-    rows = await executeQuery({ queryId: dataSource._id, parameters })
+    const { data, ...rest } = await executeQuery({
+      queryId: dataSource._id,
+      parameters,
+    })
+    info = rest
+    rows = data
   } else if (type === FieldTypes.LINK) {
     rows = await fetchRelationshipData({
       rowId: dataSource.rowId,
@@ -38,7 +44,7 @@ export const fetchDatasource = async dataSource => {
   }
 
   // Enrich the result is always an array
-  return Array.isArray(rows) ? rows : []
+  return { rows: Array.isArray(rows) ? rows : [], info }
 }
 
 /**
diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte
index e9d306cc3b..9c79ef8e9a 100644
--- a/packages/client/src/components/app/DataProvider.svelte
+++ b/packages/client/src/components/app/DataProvider.svelte
@@ -30,6 +30,7 @@
   // Provider state
   let rows = []
   let allRows = []
+  let info = {}
   let schema = {}
   let bookmarks = [null]
   let pageNumber = 0
@@ -120,6 +121,7 @@
   // Build our data context
   $: dataContext = {
     rows,
+    info,
     schema,
     rowsLength: rows.length,
 
@@ -206,7 +208,9 @@
     } else {
       // For other data sources like queries or views, fetch all rows from the
       // server
-      allRows = await API.fetchDatasource(dataSource)
+      const data = await API.fetchDatasource(dataSource)
+      allRows = data.rows
+      info = data.info
     }
     loading = false
     loaded = true
diff --git a/packages/server/src/api/controllers/query/index.js b/packages/server/src/api/controllers/query/index.js
index e12b0039a5..ff9d814bc9 100644
--- a/packages/server/src/api/controllers/query/index.js
+++ b/packages/server/src/api/controllers/query/index.js
@@ -182,13 +182,13 @@ exports.execute = async function (ctx) {
 
   // call the relevant CRUD method on the integration class
   try {
-    const { rows } = await Runner.run({
+    const { rows, extra } = await Runner.run({
       datasource,
       queryVerb: query.queryVerb,
       query: enrichedQuery,
       transformer: query.transformer,
     })
-    ctx.body = rows
+    ctx.body = { data: rows, ...extra }
   } catch (err) {
     ctx.throw(400, err)
   }
diff --git a/packages/server/src/automations/steps/executeQuery.js b/packages/server/src/automations/steps/executeQuery.js
index 43534209cc..0fa8661561 100644
--- a/packages/server/src/automations/steps/executeQuery.js
+++ b/packages/server/src/automations/steps/executeQuery.js
@@ -35,6 +35,11 @@ exports.definition = {
           type: "object",
           description: "The response from the datasource execution",
         },
+        info: {
+          type: "object",
+          description:
+            "Some query types may return extra data, like headers from a REST query",
+        },
         success: {
           type: "boolean",
           description: "Whether the action was successful",
@@ -68,13 +73,16 @@ exports.run = async function ({ inputs, appId, emitter }) {
 
   try {
     await queryController.execute(ctx)
+    const { data, ...rest } = ctx.body
     return {
-      response: ctx.body,
+      response: data,
+      info: rest,
       success: true,
     }
   } catch (err) {
     return {
       success: false,
+      info: {},
       response: automationUtils.getError(err),
     }
   }

From c55cc0f56abbbd7127076d085eac3bae1c9c02a0 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 16 Dec 2021 11:41:28 +0000
Subject: [PATCH 07/11] Fixing issue with switching between datasources and
 making sure old client apps can still use the deprecated API.

---
 .../datasource/[selectedDatasource]/index.svelte | 16 ++++++++++------
 packages/client/src/api/queries.js               |  2 +-
 .../server/src/api/controllers/query/index.js    | 16 ++++++++++++++--
 packages/server/src/api/routes/query.js          |  9 ++++++++-
 4 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
index ab3ac18dc7..98b7859305 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
@@ -21,11 +21,11 @@
   import { cloneDeep } from "lodash/fp"
 
   import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte"
-  import { onMount } from "svelte"
   let importQueriesModal
 
   let changed
-  let datasource
+  let integration, baseDatasource, datasource
+  let queryList
   const querySchema = {
     name: {},
     queryVerb: { displayName: "Method" },
@@ -34,11 +34,12 @@
   $: baseDatasource = $datasources.list.find(
     ds => ds._id === $datasources.selected
   )
-  $: integration = datasource && $integrations[datasource.source]
+
   $: queryList = $queries.list.filter(
     query => query.datasourceId === datasource?._id
   )
   $: hasChanged(baseDatasource, datasource)
+  $: updateDatasource(baseDatasource)
 
   function hasChanged(base, ds) {
     if (base && ds) {
@@ -66,9 +67,12 @@
     $goto(`./${query._id}`)
   }
 
-  onMount(() => {
-    datasource = cloneDeep(baseDatasource)
-  })
+  function updateDatasource(base) {
+    if (base) {
+      datasource = cloneDeep(base)
+      integration = $integrations[datasource.source]
+    }
+  }
 </script>
 
 <Modal bind:this={importQueriesModal}>
diff --git a/packages/client/src/api/queries.js b/packages/client/src/api/queries.js
index 257d07f8ef..834282fe73 100644
--- a/packages/client/src/api/queries.js
+++ b/packages/client/src/api/queries.js
@@ -11,7 +11,7 @@ export const executeQuery = async ({ queryId, parameters }) => {
     return
   }
   const res = await API.post({
-    url: `/api/queries/${queryId}`,
+    url: `/api/v2/queries/${queryId}`,
     body: {
       parameters,
     },
diff --git a/packages/server/src/api/controllers/query/index.js b/packages/server/src/api/controllers/query/index.js
index ff9d814bc9..198d68d081 100644
--- a/packages/server/src/api/controllers/query/index.js
+++ b/packages/server/src/api/controllers/query/index.js
@@ -169,7 +169,7 @@ exports.preview = async function (ctx) {
   }
 }
 
-exports.execute = async function (ctx) {
+async function execute(ctx, opts = { rowsOnly: false }) {
   const db = new CouchDB(ctx.appId)
 
   const query = await db.get(ctx.params.queryId)
@@ -188,12 +188,24 @@ exports.execute = async function (ctx) {
       query: enrichedQuery,
       transformer: query.transformer,
     })
-    ctx.body = { data: rows, ...extra }
+    if (opts && opts.rowsOnly) {
+      ctx.body = rows
+    } else {
+      ctx.body = { data: rows, ...extra }
+    }
   } catch (err) {
     ctx.throw(400, err)
   }
 }
 
+exports.executeV1 = async function (ctx) {
+  return execute(ctx, { rowsOnly: true })
+}
+
+exports.executeV2 = async function (ctx) {
+  return execute(ctx, { rowsOnly: false })
+}
+
 exports.destroy = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
diff --git a/packages/server/src/api/routes/query.js b/packages/server/src/api/routes/query.js
index 4a3cfeeb97..466fb1efaf 100644
--- a/packages/server/src/api/routes/query.js
+++ b/packages/server/src/api/routes/query.js
@@ -41,11 +41,18 @@ router
     authorized(PermissionTypes.QUERY, PermissionLevels.READ),
     queryController.find
   )
+  // DEPRECATED - use new query endpoint for future work
   .post(
     "/api/queries/:queryId",
     paramResource("queryId"),
     authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
-    queryController.execute
+    queryController.executeV1
+  )
+  .post(
+    "/api/v2/queries/:queryId",
+    paramResource("queryId"),
+    authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
+    queryController.executeV2
   )
   .delete(
     "/api/queries/:queryId/:revId",

From 4db85b52cbfd4d788619b8ac1c6914c817083229 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 16 Dec 2021 11:44:17 +0000
Subject: [PATCH 08/11] Fixing lint issues.

---
 .../builder/src/components/integration/KeyValueBuilder.svelte  | 2 +-
 .../_components/DynamicVariableModal.svelte                    | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte
index 7229712a65..94f3cb7f1f 100644
--- a/packages/builder/src/components/integration/KeyValueBuilder.svelte
+++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte
@@ -121,7 +121,7 @@
       {/if}
       {#if menuItems?.length > 0 && showMenu}
         <ActionMenu>
-          <div slot="control" class="icon">
+          <div slot="control" class="control icon">
             <Icon size="S" hoverable name="MoreSmallList" />
           </div>
           {#each menuItems as item}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte
index dae6804e25..1fdbc98483 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte
@@ -1,11 +1,10 @@
 <script>
   import { Input, ModalContent, Modal } from "@budibase/bbui"
 
-  export let datasource
   export let dynamicVariables
   export let binding
 
-  let name, modal
+  let name, modal, valid
 
   export const show = () => {
     modal.show()

From 53186abe20e0a937e07383cbe48957cca15bff09 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 16 Dec 2021 16:58:15 +0000
Subject: [PATCH 09/11] Adding start to redis caching.

---
 packages/auth/src/redis/utils.js              |  1 +
 .../server/src/api/controllers/query/index.js | 54 +-------------
 packages/server/src/threads/query.js          | 74 ++++++++++++++++++-
 packages/server/src/threads/utils.js          | 20 +++++
 4 files changed, 97 insertions(+), 52 deletions(-)

diff --git a/packages/auth/src/redis/utils.js b/packages/auth/src/redis/utils.js
index 466b117e96..3461d2a511 100644
--- a/packages/auth/src/redis/utils.js
+++ b/packages/auth/src/redis/utils.js
@@ -16,6 +16,7 @@ exports.Databases = {
   USER_CACHE: "users",
   FLAGS: "flags",
   APP_METADATA: "appMetadata",
+  QUERY_VARS: "queryVars",
 }
 
 exports.SEPARATOR = SEPARATOR
diff --git a/packages/server/src/api/controllers/query/index.js b/packages/server/src/api/controllers/query/index.js
index 198d68d081..ddb49b7b3d 100644
--- a/packages/server/src/api/controllers/query/index.js
+++ b/packages/server/src/api/controllers/query/index.js
@@ -1,4 +1,3 @@
-const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../../db")
 const {
   generateQueryID,
@@ -90,47 +89,6 @@ exports.save = async function (ctx) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-async function enrichQueryFields(fields, parameters = {}) {
-  const enrichedQuery = {}
-
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      enrichedQuery[key] = await processString(fields[key], parameters, {
-        noHelpers: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-
-  return enrichedQuery
-}
-
 exports.find = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const query = enrichQueries(await db.get(ctx.params.queryId))
@@ -148,13 +106,13 @@ exports.preview = async function (ctx) {
   const datasource = await db.get(ctx.request.body.datasourceId)
 
   const { fields, parameters, queryVerb, transformer } = ctx.request.body
-  const enrichedQuery = await enrichQueryFields(fields, parameters)
 
   try {
     const { rows, keys, info, extra } = await Runner.run({
       datasource,
       queryVerb,
-      query: enrichedQuery,
+      fields,
+      parameters,
       transformer,
     })
 
@@ -175,17 +133,13 @@ async function execute(ctx, opts = { rowsOnly: false }) {
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
 
-  const enrichedQuery = await enrichQueryFields(
-    query.fields,
-    ctx.request.body.parameters
-  )
-
   // call the relevant CRUD method on the integration class
   try {
     const { rows, extra } = await Runner.run({
       datasource,
       queryVerb: query.queryVerb,
-      query: enrichedQuery,
+      fields: query.fields,
+      parameters: ctx.request.body.parameter,
       transformer: query.transformer,
     })
     if (opts && opts.rowsOnly) {
diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js
index f8d713e6a2..d60812f5fd 100644
--- a/packages/server/src/threads/query.js
+++ b/packages/server/src/threads/query.js
@@ -1,6 +1,66 @@
 require("./utils").threadSetup()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
+const { processStringSync } = require("@budibase/string-templates")
+
+async function addDatasourceVariables(datasource, parameters) {
+  if (!datasource || !datasource.config) {
+    return parameters
+  }
+  const staticVars = datasource.config.staticVariables || {}
+  const dynamicVars = datasource.config.dynamicVariables || []
+  for (let [key, value] of Object.entries(staticVars)) {
+    if (!parameters[key]) {
+      parameters[key] = value
+    }
+  }
+  for (let variable of dynamicVars) {
+    console.log(variable)
+    // TODO: get the variable from query
+  }
+  return parameters
+}
+
+function enrichQueryFields(fields, parameters = {}) {
+  const enrichedQuery = {}
+
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = enrichQueryFields(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noHelpers: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+
+  return enrichedQuery
+}
 
 function formatResponse(resp) {
   if (typeof resp === "string") {
@@ -22,7 +82,16 @@ function hasExtraData(response) {
   )
 }
 
-async function runAndTransform(datasource, queryVerb, query, transformer) {
+async function runAndTransform(
+  datasource,
+  queryVerb,
+  fields,
+  parameters,
+  transformer
+) {
+  // pre-query, make sure datasource variables are added to parameters
+  parameters = await addDatasourceVariables(datasource, parameters)
+  const query = enrichQueryFields(fields, parameters)
   const Integration = integrations[datasource.source]
   if (!Integration) {
     throw "Integration type does not exist."
@@ -69,7 +138,8 @@ module.exports = (input, callback) => {
   runAndTransform(
     input.datasource,
     input.queryVerb,
-    input.query,
+    input.fields,
+    input.parameters,
     input.transformer
   )
     .then(response => {
diff --git a/packages/server/src/threads/utils.js b/packages/server/src/threads/utils.js
index 48b40d7002..a0fb9170a0 100644
--- a/packages/server/src/threads/utils.js
+++ b/packages/server/src/threads/utils.js
@@ -1,6 +1,18 @@
 const env = require("../environment")
 const CouchDB = require("../db")
 const { init } = require("@budibase/auth")
+const redis = require("@budibase/auth/redis")
+const { SEPARATOR } = require("@budibase/auth/db")
+
+let client
+
+if (!client) {
+  client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
+}
+
+process.on("exit", async () => {
+  if (client) await client.finish()
+})
 
 exports.threadSetup = () => {
   // don't run this if not threading
@@ -11,3 +23,11 @@ exports.threadSetup = () => {
   env.setInThread()
   init(CouchDB)
 }
+
+function makeVariableKey(queryId, variable) {
+  return `${queryId}${SEPARATOR}${variable}`
+}
+
+exports.checkCacheForDynamicVariable = async (queryId, variable) => {
+  await client.get(makeVariableKey(queryId, variable))
+}

From f2bb4a2162ae3a57f83bc9700ac05a644cbfdae5 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 16 Dec 2021 19:33:47 +0000
Subject: [PATCH 10/11] Fixing some issues discovered with saving queries and
 dynamic variables, also displaying dynamic variables in a reactive way.

---
 .../variables/ViewDynamicVariables.svelte     | 14 +++---
 .../rest/[query]/index.svelte                 |  2 +-
 packages/server/package.json                  |  1 +
 packages/server/src/threads/query.js          | 11 +++--
 packages/server/src/threads/utils.js          | 11 +++--
 packages/server/yarn.lock                     | 45 +++++++++++--------
 6 files changed, 50 insertions(+), 34 deletions(-)

diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte
index b778b950be..b000258399 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte
@@ -1,6 +1,5 @@
 <script>
   import { Body, Table } from "@budibase/bbui"
-  import { onMount } from "svelte"
   import { queries as queriesStore } from "stores/backend"
   import { goto } from "@roxi/routify"
 
@@ -9,6 +8,8 @@
 
   let dynamicVariables = []
 
+  $: enrichDynamicVariables(datasource, queries)
+
   const dynamicVariableSchema = {
     name: "",
     value: "",
@@ -24,18 +25,15 @@
   /**
    * Add the query name to the dynamic variables
    */
-  const enrichDynamicVariables = () => {
-    datasource.config.dynamicVariables?.forEach(dv => {
-      const query = queries.find(query => query._id === dv.queryId)
+  function enrichDynamicVariables(ds, possibleQueries) {
+    dynamicVariables = []
+    ds.config.dynamicVariables?.forEach(dv => {
+      const query = possibleQueries.find(query => query._id === dv.queryId)
       if (query) {
         dynamicVariables.push({ ...dv, query: query.name })
       }
     })
   }
-
-  onMount(() => {
-    enrichDynamicVariables()
-  })
 </script>
 
 <Table
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
index 88a460bc8f..62a16e9316 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
@@ -119,7 +119,7 @@
           saveId,
           dynamicVariables
         )
-        await datasources.save(datasource)
+        datasource = await datasources.save(datasource)
       }
     } catch (err) {
       notifications.error(`Error saving query. ${err.message}`)
diff --git a/packages/server/package.json b/packages/server/package.json
index 68b464d452..0d429fa3a6 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -121,6 +121,7 @@
     "pouchdb-replication-stream": "1.2.9",
     "server-destroy": "1.0.1",
     "svelte": "^3.38.2",
+    "swagger-parser": "^10.0.3",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js
index d60812f5fd..f0ca114e94 100644
--- a/packages/server/src/threads/query.js
+++ b/packages/server/src/threads/query.js
@@ -3,6 +3,8 @@ const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
 
+async function getDynamicVariable() {}
+
 async function addDatasourceVariables(datasource, parameters) {
   if (!datasource || !datasource.config) {
     return parameters
@@ -14,9 +16,12 @@ async function addDatasourceVariables(datasource, parameters) {
       parameters[key] = value
     }
   }
-  for (let variable of dynamicVars) {
-    console.log(variable)
-    // TODO: get the variable from query
+  const dynamics = dynamicVars.map(dynVar =>
+    getDynamicVariable(datasource, dynVar, parameters)
+  )
+  for (let i = 0; i < dynamicVars.length; i++) {
+    const variable = dynamicVars[i]
+    parameters[variable.name] = dynamics[i]
   }
   return parameters
 }
diff --git a/packages/server/src/threads/utils.js b/packages/server/src/threads/utils.js
index a0fb9170a0..824d7e3f16 100644
--- a/packages/server/src/threads/utils.js
+++ b/packages/server/src/threads/utils.js
@@ -6,15 +6,17 @@ const { SEPARATOR } = require("@budibase/auth/db")
 
 let client
 
-if (!client) {
-  client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
+async function startup() {
+  if (!client) {
+    client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
+  }
 }
 
 process.on("exit", async () => {
   if (client) await client.finish()
 })
 
-exports.threadSetup = () => {
+exports.threadSetup = async () => {
   // don't run this if not threading
   if (env.isTest() || env.DISABLE_THREADING) {
     return
@@ -22,6 +24,9 @@ exports.threadSetup = () => {
   // when thread starts, make sure it is recorded
   env.setInThread()
   init(CouchDB)
+  startup().catch(err => {
+    console.error("Redis connection failed for thread - " + err)
+  })
 }
 
 function makeVariableKey(queryId, variable) {
diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock
index 8e2c9db352..0bf8969806 100644
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@@ -27,7 +27,7 @@
   resolved "https://registry.yarnpkg.com/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz#b789a362e055b0340d04712eafe7027ddc1ac267"
   integrity sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==
 
-"@apidevtools/swagger-parser@^10.0.3":
+"@apidevtools/swagger-parser@10.0.3":
   version "10.0.3"
   resolved "https://registry.yarnpkg.com/@apidevtools/swagger-parser/-/swagger-parser-10.0.3.tgz#32057ae99487872c4dd96b314a1ab4b95d89eaf5"
   integrity sha512-sNiLY51vZOmSPFZA5TF35KZ2HbgYklQnTSDnkghamzLb3EkNtcQnrBQEj5AOCxHpTtXpqMCRM1CrmV2rG6nw4g==
@@ -983,10 +983,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/auth@^1.0.19-alpha.1":
-  version "1.0.22"
-  resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.22.tgz#a93ea2fea46e00138ad3fa129c9ea19b056654e2"
-  integrity sha512-eHCNEzGl6HxYlMpfRTXBokq2ALTK5f+CDSgJmGaL/jfPc2NlzCI5NoigZUkSrdwDiYZnnWLfDDR4dArYyLlFuA==
+"@budibase/auth@^1.0.23":
+  version "1.0.24"
+  resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.24.tgz#69a015a7fe9dcc9ea9e471a7d76d2c0544de0b16"
+  integrity sha512-pX+x+ib5X8s9ek4TZVmddRXlCE3kFmK5EH7Qk0Wmcn9tZwbaZy1XXd4qlJ7aFGSmvy1VNbz5AXIyRtliescaEQ==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -1056,10 +1056,10 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/bbui@^1.0.22":
-  version "1.0.22"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.22.tgz#ac3bd3a8699bd0be84aac3c5dff9d093e5b08462"
-  integrity sha512-8/5rXEOwkr0OcQD1fn5GpmI3d5dS1cIJBAODjTVtlZrTdacwlz5W2j3zIh+CBG0X7zhVxEze3zs2b1vDNTvK6A==
+"@budibase/bbui@^1.0.24":
+  version "1.0.24"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.24.tgz#15a976b500c4103bc3e94ed665c6c557bb6271bf"
+  integrity sha512-4zr+fSChOelHDqNNM2OiWWbWOL57JGBVBx333GiMr2P7N0MoepjwHEp9MlITWl5mUfS8rxgWTC9mSSMpRdpSUQ==
   dependencies:
     "@adobe/spectrum-css-workflow-icons" "^1.2.1"
     "@spectrum-css/actionbutton" "^1.0.1"
@@ -1106,14 +1106,14 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/client@^1.0.19-alpha.1":
-  version "1.0.22"
-  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.22.tgz#80d6c3fb2b57a050199dde4a4b3e82b221601c25"
-  integrity sha512-Cpao7l2lIWyJZJs8+zq1wFnQGaWRTDiRG+HkkjvqQZDkZexlo89zWPkY56NBbMT1qAXd6K3zAdRNNKVCBCtOaA==
+"@budibase/client@^1.0.23":
+  version "1.0.24"
+  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.24.tgz#69be5b66cc701c6018c816728975a076230c052a"
+  integrity sha512-ppb+exifHCfuDGqs1cQ0kzc3wWaOGkVgq/8Rdb7R7BLIOKYkB67L3FvkqTl+39wrc32CpmzOXXQ73keE4l6P8Q==
   dependencies:
-    "@budibase/bbui" "^1.0.22"
+    "@budibase/bbui" "^1.0.24"
     "@budibase/standard-components" "^0.9.139"
-    "@budibase/string-templates" "^1.0.22"
+    "@budibase/string-templates" "^1.0.24"
     regexparam "^1.3.0"
     shortid "^2.2.15"
     svelte-spa-router "^3.0.5"
@@ -1163,10 +1163,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/string-templates@^1.0.19-alpha.1", "@budibase/string-templates@^1.0.22":
-  version "1.0.22"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.22.tgz#b795c61e53d541c0aa346a90d04b50dcca6ae117"
-  integrity sha512-1ZhxzL75kVhP44fJlCWwqmGIPjZol1eB/xi3O11xJPYQ7lfzeJcGUpksvlgbLgBlw+MKkgppK7gEoMP247E0Qw==
+"@budibase/string-templates@^1.0.23", "@budibase/string-templates@^1.0.24":
+  version "1.0.24"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.24.tgz#8f317ff18159d8f8e36179de33252e019c85c267"
+  integrity sha512-KAwyYxtm3Mrsm44MhchSxOtB8ucZ2JI93VPBQn0hjdzHtMuAkadtFNPZFeD3pKpCjDmI13XuW+BmJqCF3B0tYg==
   dependencies:
     "@budibase/handlebars-helpers" "^0.11.7"
     dayjs "^1.10.4"
@@ -11927,6 +11927,13 @@ svg.select.js@^3.0.1:
   dependencies:
     svg.js "^2.6.5"
 
+swagger-parser@^10.0.3:
+  version "10.0.3"
+  resolved "https://registry.yarnpkg.com/swagger-parser/-/swagger-parser-10.0.3.tgz#04cb01c18c3ac192b41161c77f81e79309135d03"
+  integrity sha512-nF7oMeL4KypldrQhac8RyHerJeGPD1p2xDh900GPvc+Nk7nWP6jX2FcC7WmkinMoAmoO774+AFXcWsW8gMWEIg==
+  dependencies:
+    "@apidevtools/swagger-parser" "10.0.3"
+
 symbol-tree@^3.2.2, symbol-tree@^3.2.4:
   version "3.2.4"
   resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2"

From 4ceed6c340461a7565b1cf5f429fc135ef87a259 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 16 Dec 2021 22:43:14 +0000
Subject: [PATCH 11/11] Adding in dynamic variable processing - finding which
 variables need to be used and then calling them from within the query thread.

---
 .../server/src/api/controllers/query/index.js |   2 +
 packages/server/src/threads/query.js          | 260 ++++++++----------
 packages/server/src/threads/utils.js          |  83 +++++-
 3 files changed, 200 insertions(+), 145 deletions(-)

diff --git a/packages/server/src/api/controllers/query/index.js b/packages/server/src/api/controllers/query/index.js
index ddb49b7b3d..d0992d037a 100644
--- a/packages/server/src/api/controllers/query/index.js
+++ b/packages/server/src/api/controllers/query/index.js
@@ -109,6 +109,7 @@ exports.preview = async function (ctx) {
 
   try {
     const { rows, keys, info, extra } = await Runner.run({
+      appId: ctx.appId,
       datasource,
       queryVerb,
       fields,
@@ -136,6 +137,7 @@ async function execute(ctx, opts = { rowsOnly: false }) {
   // call the relevant CRUD method on the integration class
   try {
     const { rows, extra } = await Runner.run({
+      appId: ctx.appId,
       datasource,
       queryVerb: query.queryVerb,
       fields: query.fields,
diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js
index f0ca114e94..b86c1a49fd 100644
--- a/packages/server/src/threads/query.js
+++ b/packages/server/src/threads/query.js
@@ -1,152 +1,134 @@
-require("./utils").threadSetup()
+const threadUtils = require("./utils")
+threadUtils.threadSetup()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
+const CouchDB = require("../db")
 
-async function getDynamicVariable() {}
+class QueryRunner {
+  constructor(input, flags = { noRecursiveQuery: false }) {
+    this.appId = input.appId
+    this.datasource = input.datasource
+    this.queryVerb = input.queryVerb
+    this.fields = input.fields
+    this.parameters = input.parameters
+    this.transformer = input.transformer
+    this.noRecursiveQuery = flags.noRecursiveQuery
+  }
 
-async function addDatasourceVariables(datasource, parameters) {
-  if (!datasource || !datasource.config) {
+  async execute() {
+    let { datasource, fields, queryVerb, transformer } = this
+    // pre-query, make sure datasource variables are added to parameters
+    const parameters = await this.addDatasourceVariables()
+    const query = threadUtils.enrichQueryFields(fields, parameters)
+    const Integration = integrations[datasource.source]
+    if (!Integration) {
+      throw "Integration type does not exist."
+    }
+    const integration = new Integration(datasource.config)
+
+    let output = threadUtils.formatResponse(await integration[queryVerb](query))
+    let rows = output,
+      info = undefined,
+      extra = undefined
+    if (threadUtils.hasExtraData(output)) {
+      rows = output.data
+      info = output.info
+      extra = output.extra
+    }
+
+    // transform as required
+    if (transformer) {
+      const runner = new ScriptRunner(transformer, { data: rows })
+      rows = runner.execute()
+    }
+
+    // needs to an array for next step
+    if (!Array.isArray(rows)) {
+      rows = [rows]
+    }
+
+    // map into JSON if just raw primitive here
+    if (rows.find(row => typeof row !== "object")) {
+      rows = rows.map(value => ({ value }))
+    }
+
+    // get all the potential fields in the schema
+    let keys = rows.flatMap(Object.keys)
+
+    if (integration.end) {
+      integration.end()
+    }
+
+    return { rows, keys, info, extra }
+  }
+
+  async runAnotherQuery(queryId, parameters) {
+    const db = new CouchDB(this.appId)
+    const query = await db.get(queryId)
+    const datasource = await db.get(query.datasourceId)
+    return new QueryRunner(
+      {
+        appId: this.appId,
+        datasource,
+        queryVerb: query.queryVerb,
+        fields: query.fields,
+        parameters,
+        transformer: query.transformer,
+      },
+      { noRecursiveQuery: true }
+    ).execute()
+  }
+
+  async getDynamicVariable(variable) {
+    let { parameters } = this
+    const queryId = variable.queryId,
+      name = variable.name
+    let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
+    if (!value) {
+      value = await this.runAnotherQuery(queryId, parameters)
+      await threadUtils.storeDynamicVariable(queryId, name, value)
+    }
+    return value
+  }
+
+  async addDatasourceVariables() {
+    let { datasource, parameters, fields } = this
+    if (!datasource || !datasource.config) {
+      return parameters
+    }
+    const staticVars = datasource.config.staticVariables || {}
+    const dynamicVars = datasource.config.dynamicVariables || []
+    for (let [key, value] of Object.entries(staticVars)) {
+      if (!parameters[key]) {
+        parameters[key] = value
+      }
+    }
+    if (!this.noRecursiveQuery) {
+      // need to see if this uses any variables
+      const stringFields = JSON.stringify(fields)
+      const foundVars = dynamicVars.filter(variable => {
+        // look for {{ variable }} but allow spaces between handlebars
+        const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
+        return regex.test(stringFields)
+      })
+      const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
+      const responses = await Promise.all(dynamics)
+      for (let i = 0; i < foundVars.length; i++) {
+        const variable = foundVars[i]
+        parameters[variable.name] = processStringSync(variable.value, {
+          data: responses[i].rows,
+          info: responses[i].extra,
+        })
+      }
+    }
     return parameters
   }
-  const staticVars = datasource.config.staticVariables || {}
-  const dynamicVars = datasource.config.dynamicVariables || []
-  for (let [key, value] of Object.entries(staticVars)) {
-    if (!parameters[key]) {
-      parameters[key] = value
-    }
-  }
-  const dynamics = dynamicVars.map(dynVar =>
-    getDynamicVariable(datasource, dynVar, parameters)
-  )
-  for (let i = 0; i < dynamicVars.length; i++) {
-    const variable = dynamicVars[i]
-    parameters[variable.name] = dynamics[i]
-  }
-  return parameters
-}
-
-function enrichQueryFields(fields, parameters = {}) {
-  const enrichedQuery = {}
-
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      enrichedQuery[key] = processStringSync(fields[key], parameters, {
-        noHelpers: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-
-  return enrichedQuery
-}
-
-function formatResponse(resp) {
-  if (typeof resp === "string") {
-    try {
-      resp = JSON.parse(resp)
-    } catch (err) {
-      resp = { response: resp }
-    }
-  }
-  return resp
-}
-
-function hasExtraData(response) {
-  return (
-    typeof response === "object" &&
-    !Array.isArray(response) &&
-    response.data != null &&
-    response.info != null
-  )
-}
-
-async function runAndTransform(
-  datasource,
-  queryVerb,
-  fields,
-  parameters,
-  transformer
-) {
-  // pre-query, make sure datasource variables are added to parameters
-  parameters = await addDatasourceVariables(datasource, parameters)
-  const query = enrichQueryFields(fields, parameters)
-  const Integration = integrations[datasource.source]
-  if (!Integration) {
-    throw "Integration type does not exist."
-  }
-  const integration = new Integration(datasource.config)
-
-  let output = formatResponse(await integration[queryVerb](query))
-  let rows = output,
-    info = undefined,
-    extra = undefined
-  if (hasExtraData(output)) {
-    rows = output.data
-    info = output.info
-    extra = output.extra
-  }
-
-  // transform as required
-  if (transformer) {
-    const runner = new ScriptRunner(transformer, { data: rows })
-    rows = runner.execute()
-  }
-
-  // needs to an array for next step
-  if (!Array.isArray(rows)) {
-    rows = [rows]
-  }
-
-  // map into JSON if just raw primitive here
-  if (rows.find(row => typeof row !== "object")) {
-    rows = rows.map(value => ({ value }))
-  }
-
-  // get all the potential fields in the schema
-  let keys = rows.flatMap(Object.keys)
-
-  if (integration.end) {
-    integration.end()
-  }
-
-  return { rows, keys, info, extra }
 }
 
 module.exports = (input, callback) => {
-  runAndTransform(
-    input.datasource,
-    input.queryVerb,
-    input.fields,
-    input.parameters,
-    input.transformer
-  )
+  const Runner = new QueryRunner(input)
+  Runner.execute()
     .then(response => {
       callback(null, response)
     })
diff --git a/packages/server/src/threads/utils.js b/packages/server/src/threads/utils.js
index 824d7e3f16..34ae4f0477 100644
--- a/packages/server/src/threads/utils.js
+++ b/packages/server/src/threads/utils.js
@@ -3,20 +3,23 @@ const CouchDB = require("../db")
 const { init } = require("@budibase/auth")
 const redis = require("@budibase/auth/redis")
 const { SEPARATOR } = require("@budibase/auth/db")
+const { processStringSync } = require("@budibase/string-templates")
 
+const VARIABLE_TTL_SECONDS = 3600
 let client
 
-async function startup() {
+async function getClient() {
   if (!client) {
     client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
   }
+  return client
 }
 
 process.on("exit", async () => {
   if (client) await client.finish()
 })
 
-exports.threadSetup = async () => {
+exports.threadSetup = () => {
   // don't run this if not threading
   if (env.isTest() || env.DISABLE_THREADING) {
     return
@@ -24,9 +27,6 @@ exports.threadSetup = async () => {
   // when thread starts, make sure it is recorded
   env.setInThread()
   init(CouchDB)
-  startup().catch(err => {
-    console.error("Redis connection failed for thread - " + err)
-  })
 }
 
 function makeVariableKey(queryId, variable) {
@@ -34,5 +34,76 @@ function makeVariableKey(queryId, variable) {
 }
 
 exports.checkCacheForDynamicVariable = async (queryId, variable) => {
-  await client.get(makeVariableKey(queryId, variable))
+  const cache = await getClient()
+  return cache.get(makeVariableKey(queryId, variable))
+}
+
+exports.storeDynamicVariable = async (queryId, variable, value) => {
+  const cache = await getClient()
+  await cache.store(
+    makeVariableKey(queryId, variable),
+    value,
+    VARIABLE_TTL_SECONDS
+  )
+}
+
+exports.formatResponse = resp => {
+  if (typeof resp === "string") {
+    try {
+      resp = JSON.parse(resp)
+    } catch (err) {
+      resp = { response: resp }
+    }
+  }
+  return resp
+}
+
+exports.hasExtraData = response => {
+  return (
+    typeof response === "object" &&
+    !Array.isArray(response) &&
+    response.data != null &&
+    response.info != null
+  )
+}
+
+exports.enrichQueryFields = (fields, parameters = {}) => {
+  const enrichedQuery = {}
+
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noHelpers: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+
+  return enrichedQuery
 }
