Finishing up front-end, getting variable creation and management up and working from within the query schema/header menus.

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -125,7 +125,7 @@
             <Icon size="S" hoverable name="MoreSmallList" />
           </div>
           {#each menuItems as item}
-            <MenuItem on:click={item.onClick}>
+            <MenuItem on:click={() => item.onClick(field)}>
               {item.text}
             </MenuItem>
           {/each}
@@ -162,4 +162,7 @@
   .readOnly-menu {
     grid-template-columns: 1fr 1fr 20px;
   }
+  .control {
+    margin-top: 4px;
+  }
 </style>

packages/builder/src/helpers/data/utils.js

@@ -119,3 +119,62 @@ export function flipHeaderState(headersActivity) {
   })
   return enabled
 }
+
+// convert dynamic variables list to simple key/val object
+export function variablesToObject(datasource) {
+  const variablesList = datasource?.config?.dynamicVariables
+  if (variablesList && variablesList.length > 0) {
+    return variablesList.reduce(
+      (acc, next) => ({ ...acc, [next.name]: next.value }),
+      {}
+    )
+  }
+  return {}
+}
+
+// convert dynamic variables object back to a list, enrich with query id
+export function rebuildVariables(queryId, variables) {
+  let vars = []
+  if (variables) {
+    vars = Object.entries(variables).map(entry => {
+      return {
+        name: entry[0],
+        value: entry[1],
+        queryId,
+      }
+    })
+  }
+  return vars
+}
+
+export function shouldShowVariables(dynamicVariables, variablesReadOnly) {
+  return !!(
+    dynamicVariables &&
+    // show when editable or when read only and not empty
+    (!variablesReadOnly || Object.keys(dynamicVariables).length > 0)
+  )
+}
+
+export function buildAuthConfigs(datasource) {
+  if (datasource?.config?.authConfigs) {
+    return datasource.config.authConfigs.map(c => ({
+      label: c.name,
+      value: c._id,
+    }))
+  }
+  return []
+}
+
+export default {
+  breakQueryString,
+  buildQueryString,
+  fieldsToSchema,
+  flipHeaderState,
+  keyValueToQueryParameters,
+  queryParametersToKeyValue,
+  schemaToFields,
+  variablesToObject,
+  rebuildVariables,
+  shouldShowVariables,
+  buildAuthConfigs,
+}

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte

@@ -0,0 +1,46 @@
+<script>
+  import { Input, ModalContent, Modal } from "@budibase/bbui"
+
+  export let datasource
+  export let dynamicVariables
+  export let binding
+
+  let name, modal
+
+  export const show = () => {
+    modal.show()
+  }
+  export const hide = () => {
+    modal.hide()
+  }
+
+  function checkValid(vars, name) {
+    if (!name) {
+      return false
+    }
+    const varKeys = Object.keys(vars || {})
+    return varKeys.find(key => key.toLowerCase() === name.toLowerCase()) == null
+  }
+
+  $: valid = checkValid(dynamicVariables, name)
+  $: error = name && !valid ? "Variable name is already in use." : null
+
+  async function saveVariable() {
+    const copiedName = name,
+      copiedBinding = binding
+    name = null
+    binding = null
+    dynamicVariables[copiedName] = copiedBinding
+  }
+</script>
+
+<Modal bind:this={modal}>
+  <ModalContent
+    title="Add dynamic variable"
+    confirmText="Save"
+    onConfirm={saveVariable}
+    disabled={!valid}
+  >
+    <Input label="Variable name" bind:value={name} on:input {error} />
+  </ModalContent>
+</Modal>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte

@@ -1,22 +1,22 @@
 <script>
   import { params } from "@roxi/routify"
-  import { datasources, integrations, queries, flags } from "stores/backend"
+  import { datasources, flags, integrations, queries } from "stores/backend"
   import {
-    Layout,
-    Input,
-    Select,
-    Tabs,
-    Tab,
     Banner,
-    Divider,
-    Button,
-    Heading,
-    RadioGroup,
-    Label,
     Body,
-    TextArea,
-    Table,
+    Button,
+    Divider,
+    Heading,
+    Input,
+    Label,
+    Layout,
     notifications,
+    RadioGroup,
+    Select,
+    Tab,
+    Table,
+    Tabs,
+    TextArea,
   } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import EditableLabel from "components/common/inputs/EditableLabel.svelte"
@@ -26,33 +26,24 @@
   import RestBodyInput from "../../_components/RestBodyInput.svelte"
   import { capitalise } from "helpers"
   import { onMount } from "svelte"
-  import {
-    fieldsToSchema,
-    schemaToFields,
-    breakQueryString,
-    buildQueryString,
-    keyValueToQueryParameters,
-    queryParametersToKeyValue,
-    flipHeaderState,
-  } from "helpers/data/utils"
+  import restUtils from "helpers/data/utils"
   import {
     RestBodyTypes as bodyTypes,
     SchemaTypeOptions,
   } from "constants/backend"
   import JSONPreview from "components/integration/JSONPreview.svelte"
   import AccessLevelSelect from "components/integration/AccessLevelSelect.svelte"
+  import DynamicVariableModal from "../../_components/DynamicVariableModal.svelte"
   import Placeholder from "assets/bb-spaceship.svg"
   import { cloneDeep } from "lodash/fp"
 
   let query, datasource
   let breakQs = {},
     bindings = {}
-  let url = ""
-  let saveId, isGet
+  let saveId, url
   let response, schema, enabledHeaders
-  let datasourceType, integrationInfo, queryConfig, responseSuccess
   let authConfigId
-  let dynamicVariables
+  let dynamicVariables, addVariableModal, varBinding
 
   $: datasourceType = datasource?.source
   $: integrationInfo = $integrations[datasourceType]
@@ -61,10 +52,13 @@
   $: checkQueryName(url)
   $: responseSuccess = response?.info?.code >= 200 && response?.info?.code < 400
   $: isGet = query?.queryVerb === "read"
-  $: authConfigs = buildAuthConfigs(datasource)
+  $: authConfigs = restUtils.buildAuthConfigs(datasource)
   $: schemaReadOnly = !responseSuccess
   $: variablesReadOnly = !responseSuccess
-  $: showVariablesTab = shouldShowVariables(dynamicVariables, variablesReadOnly)
+  $: showVariablesTab = restUtils.shouldShowVariables(
+    dynamicVariables,
+    variablesReadOnly
+  )
 
   function getSelectedQuery() {
     return cloneDeep(
@@ -92,7 +86,7 @@
     if (!base) {
       return base
     }
-    const qs = buildQueryString(qsObj)
+    const qs = restUtils.buildQueryString(qsObj)
     let newUrl = base
     if (base.includes("?")) {
       newUrl = base.split("?")[0]
@@ -100,29 +94,15 @@
     return qs.length > 0 ? `${newUrl}?${qs}` : newUrl
   }
 
-  const buildAuthConfigs = datasource => {
-    if (datasource?.config?.authConfigs) {
-      return datasource.config.authConfigs.map(c => ({
-        label: c.name,
-        value: c._id,
-      }))
-    }
-    return []
-  }
-
-  function learnMoreBanner() {
-    window.open("https://docs.budibase.com/building-apps/data/transformers")
-  }
-
   function buildQuery() {
     const newQuery = { ...query }
-    const queryString = buildQueryString(breakQs)
+    const queryString = restUtils.buildQueryString(breakQs)
     newQuery.fields.path = url.split("?")[0]
     newQuery.fields.queryString = queryString
     newQuery.fields.authConfigId = authConfigId
-    newQuery.fields.disabledHeaders = flipHeaderState(enabledHeaders)
-    newQuery.schema = fieldsToSchema(schema)
-    newQuery.parameters = keyValueToQueryParameters(bindings)
+    newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
+    newQuery.schema = restUtils.fieldsToSchema(schema)
+    newQuery.parameters = restUtils.keyValueToQueryParameters(bindings)
     return newQuery
   }
 
@@ -135,8 +115,10 @@
       notifications.success(`Request saved successfully.`)
 
       if (dynamicVariables) {
-        const dynamicVars = rebuildVariables(saveId)
-        datasource.config.dynamicVariables = dynamicVars
+        datasource.config.dynamicVariables = restUtils.rebuildVariables(
+          saveId,
+          dynamicVariables
+        )
         await datasources.save(datasource)
       }
     } catch (err) {
@@ -175,57 +157,21 @@
     return id
   }
 
-  // convert dynamic variables list to simple key/val object
-  const variablesToObject = datasource => {
-    const variablesList = datasource?.config?.dynamicVariables
-    if (variablesList && variablesList.length > 0) {
-      return variablesList.reduce(
-        (acc, next) => ({ ...acc, [next.name]: next.value }),
-        {}
-      )
-    }
-    return {}
-  }
-
-  // convert dynamic variables object back to a list, enrich with query id
-  const rebuildVariables = queryId => {
-    let variables = []
-    if (dynamicVariables) {
-      variables = Object.entries(dynamicVariables).map(entry => {
-        return {
-          name: entry[0],
-          value: entry[1],
-          queryId,
-        }
-      })
-    }
-    return variables
-  }
-
-  const shouldShowVariables = (dynamicVariables, variablesReadOnly) => {
-    if (
-      dynamicVariables &&
-      // show when editable or when read only and not empty
-      (!variablesReadOnly || Object.keys(dynamicVariables).length > 0)
-    ) {
-      return true
-    }
-    return false
-  }
-
   const schemaMenuItems = [
     {
       text: "Create dynamic variable",
-      onClick: () => {
-        console.log("create variable")
+      onClick: input => {
+        varBinding = `{{ data.0.[${input.name}] }}`
+        addVariableModal.show()
       },
     },
   ]
   const responseHeadersMenuItems = [
     {
       text: "Create dynamic variable",
-      onClick: () => {
-        console.log("create variable")
+      onClick: input => {
+        varBinding = `{{ info.headers.[${input.name}] }}`
+        addVariableModal.show()
       },
     },
   ]
@@ -237,14 +183,14 @@
     datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
     const datasourceUrl = datasource?.config.url
     const qs = query?.fields.queryString
-    breakQs = breakQueryString(qs)
+    breakQs = restUtils.breakQueryString(qs)
     if (datasourceUrl && !query.fields.path?.startsWith(datasourceUrl)) {
       const path = query.fields.path
       query.fields.path = `${datasource.config.url}/${path ? path : ""}`
     }
     url = buildUrl(query.fields.path, breakQs)
-    schema = schemaToFields(query.schema)
-    bindings = queryParametersToKeyValue(query.parameters)
+    schema = restUtils.schemaToFields(query.schema)
+    bindings = restUtils.queryParametersToKeyValue(query.parameters)
     authConfigId = getAuthConfigId()
     if (!query.fields.disabledHeaders) {
       query.fields.disabledHeaders = {}
@@ -255,7 +201,7 @@
         query.fields.disabledHeaders[header] = false
       }
     }
-    enabledHeaders = flipHeaderState(query.fields.disabledHeaders)
+    enabledHeaders = restUtils.flipHeaderState(query.fields.disabledHeaders)
     if (query && !query.transformer) {
       query.transformer = "return data"
     }
@@ -267,10 +213,16 @@
     if (query && !query.fields.bodyType) {
       query.fields.bodyType = "none"
     }
-    dynamicVariables = variablesToObject(datasource)
+    dynamicVariables = restUtils.variablesToObject(datasource)
   })
 </script>
 
+<DynamicVariableModal
+  {datasource}
+  {dynamicVariables}
+  bind:binding={varBinding}
+  bind:this={addVariableModal}
+/>
 {#if query && queryConfig}
   <div class="inner">
     <div class="top">
@@ -340,7 +292,10 @@
               {#if !$flags.queryTransformerBanner}
                 <Banner
                   extraButtonText="Learn more"
-                  extraButtonAction={learnMoreBanner}
+                  extraButtonAction={() =>
+                    window.open(
+                      "https://docs.budibase.com/building-apps/data/transformers"
+                    )}
                   on:change={() =>
                     flags.updateFlag("queryTransformerBanner", true)}
                 >
@@ -449,9 +404,9 @@
                     name="Variable"
                     headings
                     keyHeading="Name"
-                    keyPlaceholder="e.g. cookie"
+                    keyPlaceholder="Variable name"
                     valueHeading={`Value`}
-                    valuePlaceholder={`e.g. {{ headers.set-cookie }}`}
+                    valuePlaceholder={`{{ value }}`}
                     readOnly={variablesReadOnly}
                   />
                 </Layout>

packages/client/manifest.json

@@ -2457,6 +2457,10 @@
           "label": "Rows",
           "key": "rows"
         },
+	{
+	  "label": "Extra Info",
+	  "key": "info"
+	},
         {
           "label": "Rows Length",
           "key": "rowsLength"
@@ -3178,6 +3182,10 @@
             "label": "Rows",
             "key": "rows"
           },
+	  {
+            "label": "Extra Info",
+            "key": "info"
+          },
           {
             "label": "Rows Length",
             "key": "rowsLength"

packages/client/src/api/datasources.js

@@ -15,7 +15,8 @@ export const fetchDatasource = async dataSource => {
 
   // Fetch all rows in data source
   const { type, tableId, fieldName } = dataSource
-  let rows = []
+  let rows = [],
+    info = {}
   if (type === "table") {
     rows = await fetchTableData(tableId)
   } else if (type === "view") {
@@ -28,7 +29,12 @@ export const fetchDatasource = async dataSource => {
         parameters[param.name] = param.default
       }
     }
-    rows = await executeQuery({ queryId: dataSource._id, parameters })
+    const { data, ...rest } = await executeQuery({
+      queryId: dataSource._id,
+      parameters,
+    })
+    info = rest
+    rows = data
   } else if (type === FieldTypes.LINK) {
     rows = await fetchRelationshipData({
       rowId: dataSource.rowId,
@@ -38,7 +44,7 @@ export const fetchDatasource = async dataSource => {
   }
 
   // Enrich the result is always an array
-  return Array.isArray(rows) ? rows : []
+  return { rows: Array.isArray(rows) ? rows : [], info }
 }
 
 /**

packages/client/src/components/app/DataProvider.svelte

@@ -30,6 +30,7 @@
   // Provider state
   let rows = []
   let allRows = []
+  let info = {}
   let schema = {}
   let bookmarks = [null]
   let pageNumber = 0
@@ -120,6 +121,7 @@
   // Build our data context
   $: dataContext = {
     rows,
+    info,
     schema,
     rowsLength: rows.length,
 
@@ -206,7 +208,9 @@
     } else {
       // For other data sources like queries or views, fetch all rows from the
       // server
-      allRows = await API.fetchDatasource(dataSource)
+      const data = await API.fetchDatasource(dataSource)
+      allRows = data.rows
+      info = data.info
     }
     loading = false
     loaded = true

packages/server/src/api/controllers/query/index.js

@@ -182,13 +182,13 @@ exports.execute = async function (ctx) {
 
   // call the relevant CRUD method on the integration class
   try {
-    const { rows } = await Runner.run({
+    const { rows, extra } = await Runner.run({
       datasource,
       queryVerb: query.queryVerb,
       query: enrichedQuery,
       transformer: query.transformer,
     })
-    ctx.body = rows
+    ctx.body = { data: rows, ...extra }
   } catch (err) {
     ctx.throw(400, err)
   }

packages/server/src/automations/steps/executeQuery.js

@@ -35,6 +35,11 @@ exports.definition = {
           type: "object",
           description: "The response from the datasource execution",
         },
+        info: {
+          type: "object",
+          description:
+            "Some query types may return extra data, like headers from a REST query",
+        },
         success: {
           type: "boolean",
           description: "Whether the action was successful",
@@ -68,13 +73,16 @@ exports.run = async function ({ inputs, appId, emitter }) {
 
   try {
     await queryController.execute(ctx)
+    const { data, ...rest } = ctx.body
     return {
-      response: ctx.body,
+      response: data,
+      info: rest,
       success: true,
     }
   } catch (err) {
     return {
       success: false,
+      info: {},
       response: automationUtils.getError(err),
     }
   }