CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-10-03 10:36:59 +13:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3830 from Budibase/feature/query-variables

Browse source 
Datasource static and dynamic variables
This commit is contained in:
Michael Drury 2022-01-05 15:22:30 +00:00 committed by GitHub
commit fe40e3e85d
36 changed files with 1828 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
packages/auth/src/constants.js
View file

@ -16,6 +16,7 @@ exports.Headers = {
APP_ID: "x-budibase-app-id",
TYPE: "x-budibase-type",
TENANT_ID: "x-budibase-tenant-id",
TOKEN: "x-budibase-token",
}
exports.GlobalRoles = {

7
packages/auth/src/middleware/authenticated.js
View file

@ -1,5 +1,5 @@
const { Cookies, Headers } = require("../constants")
const { getCookie, clearCookie } = require("../utils")
const { getCookie, clearCookie, openJwt } = require("../utils")
const { getUser } = require("../cache/user")
const { getSession, updateSessionTTL } = require("../security/sessions")
const { buildMatcherRegex, matches } = require("./matchers")
@ -35,8 +35,9 @@ module.exports = (
publicEndpoint = true
}
try {
// check the actual user is authenticated first
const authCookie = getCookie(ctx, Cookies.Auth)
// check the actual user is authenticated first, try header or cookie
const headerToken = ctx.request.headers[Headers.TOKEN]
const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
let authenticated = false,
user = null,
internal = false

1
packages/auth/src/redis/utils.js
View file

@ -16,6 +16,7 @@ exports.Databases = {
USER_CACHE: "users",
FLAGS: "flags",
APP_METADATA: "appMetadata",
QUERY_VARS: "queryVars",
}
exports.SEPARATOR = SEPARATOR

13
packages/auth/src/utils.js
View file

@ -63,6 +63,17 @@ exports.getAppId = ctx => {
return appId
}
/**
* opens the contents of the specified encrypted JWT.
* @return {object} the contents of the token.
*/
exports.openJwt = token => {
if (!token) {
return token
}
return jwt.verify(token, options.secretOrKey)
}
/**
* Get a cookie from context, and decrypt if necessary.
* @param {object} ctx The request which is to be manipulated.
@ -75,7 +86,7 @@ exports.getCookie = (ctx, name) => {
return cookie
}
return jwt.verify(cookie, options.secretOrKey)
return exports.openJwt(cookie)
}
/**

11
packages/bbui/src/Table/BoldRenderer.svelte Normal file
View file

@ -0,0 +1,11 @@
<script>
export let value
</script>
<div class="bold">{value}</div>
<style>
.bold {
font-weight: bold;
}
</style>

5
packages/bbui/src/Table/CodeRenderer.svelte Normal file
View file

@ -0,0 +1,5 @@
<script>
export let value
</script>
<code>{value}</code>

4
packages/bbui/src/index.js
View file

@ -61,6 +61,10 @@ export { default as ColorPicker } from "./ColorPicker/ColorPicker.svelte"
export { default as InlineAlert } from "./InlineAlert/InlineAlert.svelte"
export { default as Banner } from "./Banner/Banner.svelte"
// Renderers
export { default as BoldRenderer } from "./Table/BoldRenderer.svelte"
export { default as CodeRenderer } from "./Table/CodeRenderer.svelte"
// Typography
export { default as Body } from "./Typography/Body.svelte"
export { default as Heading } from "./Typography/Heading.svelte"

42
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte
View file

@ -175,15 +175,19 @@
onConfirm={datasources.removeSchemaError}
/>
{/if}
<Table
on:click={({ detail }) => onClickTable(detail)}
schema={tableSchema}
data={Object.values(plusTables)}
allowEditColumns={false}
allowEditRows={false}
allowSelectRows={false}
customRenderers={[{ column: "primary", component: ArrayRenderer }]}
/>
{#if plusTables && Object.values(plusTables).length > 0}
<Table
on:click={({ detail }) => onClickTable(detail)}
schema={tableSchema}
data={Object.values(plusTables)}
allowEditColumns={false}
allowEditRows={false}
allowSelectRows={false}
customRenderers={[{ column: "primary", component: ArrayRenderer }]}
/>
{:else}
<Body size="S"><i>No tables found.</i></Body>
{/if}
{#if plusTables?.length !== 0}
<Divider size="S" />
<div class="query-header">
@ -196,14 +200,18 @@
Tell budibase how your tables are related to get even more smart features.
</Body>
{/if}
<Table
on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
schema={relationshipSchema}
data={relationshipInfo}
allowEditColumns={false}
allowEditRows={false}
allowSelectRows={false}
/>
{#if relationshipInfo && relationshipInfo.length > 0}
<Table
on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
schema={relationshipSchema}
data={relationshipInfo}
allowEditColumns={false}
allowEditRows={false}
allowSelectRows={false}
/>
{:else}
<Body size="S"><i>No relationships configured.</i></Body>
{/if}
<style>
.query-header {

41
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
View file

@ -1,9 +1,18 @@
<script>
import { Divider, Heading, ActionButton, Badge, Body } from "@budibase/bbui"
import {
Divider,
Heading,
ActionButton,
Badge,
Body,
Layout,
} from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import RestAuthenticationBuilder from "./auth/RestAuthenticationBuilder.svelte"
import ViewDynamicVariables from "./variables/ViewDynamicVariables.svelte"
export let datasource
export let queries
let addHeader
</script>
@ -43,6 +52,36 @@
</Body>
<RestAuthenticationBuilder bind:configs={datasource.config.authConfigs} />
<Divider size="S" />
<div class="section-header">
<div class="badge">
<Heading size="S">Variables</Heading>
<Badge quiet grey>Optional</Badge>
</div>
</div>
<Body size="S"
>Variables enable you to store and re-use values in queries, with the choice
of a static value such as a token using static variables, or a value from a
query response using dynamic variables.</Body
>
<Heading size="XS">Static</Heading>
<Layout noPadding gap="XS">
<KeyValueBuilder
name="Variable"
keyPlaceholder="Name"
headings
bind:object={datasource.config.staticVariables}
on:change
/>
</Layout>
<div />
<Heading size="XS">Dynamic</Heading>
<Body size="S">
Dynamic variables are evaluated when a dependant query is executed. The value
is cached for a period of time and will be refreshed if a query fails.
</Body>
<ViewDynamicVariables {queries} {datasource} />
<style>
.section-header {
display: flex;

54
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte Normal file
View file

@ -0,0 +1,54 @@
<script>
import { Body, Table, BoldRenderer, CodeRenderer } from "@budibase/bbui"
import { queries as queriesStore } from "stores/backend"
import { goto } from "@roxi/routify"
export let datasource
export let queries
let dynamicVariables = []
$: enrichDynamicVariables(datasource, queries)
const dynamicVariableSchema = {
name: "",
query: "",
value: "",
}
const onClick = dynamicVariable => {
const queryId = dynamicVariable.queryId
queriesStore.select({ _id: queryId })
$goto(`./${queryId}`)
}
/**
* Add the query name to the dynamic variables
*/
function enrichDynamicVariables(ds, possibleQueries) {
dynamicVariables = []
ds.config.dynamicVariables?.forEach(dv => {
const query = possibleQueries.find(query => query._id === dv.queryId)
if (query) {
dynamicVariables.push({ ...dv, query: query.name })
}
})
}
</script>
{#if dynamicVariables && dynamicVariables.length > 0}
<Table
on:click={({ detail }) => onClick(detail)}
schema={dynamicVariableSchema}
data={dynamicVariables}
allowEditColumns={false}
allowEditRows={false}
allowSelectRows={false}
customRenderers={[
{ column: "name", component: BoldRenderer },
{ column: "value", component: CodeRenderer },
]}
/>
{:else}
<Body size="S"><i>No dynamic variables specified.</i></Body>
{/if}

41
packages/builder/src/components/integration/KeyValueBuilder.svelte
View file

@ -6,6 +6,8 @@
Label,
Toggle,
Select,
ActionMenu,
MenuItem,
} from "@budibase/bbui"
import { createEventDispatcher } from "svelte"
import { lowercase } from "helpers"
@ -23,7 +25,11 @@
export let toggle
export let keyPlaceholder = "Key"
export let valuePlaceholder = "Value"
export let valueHeading
export let keyHeading
export let tooltip
export let menuItems
export let showMenu = false
let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
let fieldActivity = buildFieldActivity(activity)
@ -76,17 +82,24 @@
{#if Object.keys(object || {}).length > 0}
{#if headings}
<div class="container" class:container-active={toggle}>
<Label {tooltip}>{keyPlaceholder}</Label>
<Label>{valuePlaceholder}</Label>
<Label {tooltip}>{keyHeading || keyPlaceholder}</Label>
<Label>{valueHeading || valuePlaceholder}</Label>
{#if toggle}
<Label>Active</Label>
{/if}
</div>
{/if}
<div class="container" class:container-active={toggle} class:readOnly>
<div
class="container"
class:container-active={toggle}
class:container-menu={showMenu}
class:readOnly
class:readOnly-menu={readOnly && showMenu}
>
{#each fields as field, idx}
<Input
placeholder={keyPlaceholder}
readonly={readOnly}
bind:value={field.name}
on:change={changed}
/>
@ -95,6 +108,7 @@
{:else}
<Input
placeholder={valuePlaceholder}
readonly={readOnly}
bind:value={field.value}
on:change={changed}
/>
@ -105,6 +119,18 @@
{#if !readOnly}
<Icon hoverable name="Close" on:click={() => deleteEntry(idx)} />
{/if}
{#if menuItems?.length > 0 && showMenu}
<ActionMenu>
<div slot="control" class="control icon">
<Icon size="S" hoverable name="MoreSmallList" />
</div>
{#each menuItems as item}
<MenuItem on:click={() => item.onClick(field)}>
{item.text}
</MenuItem>
{/each}
</ActionMenu>
{/if}
{/each}
</div>
{/if}
@ -127,7 +153,16 @@
.container-active {
grid-template-columns: 1fr 1fr 50px 20px;
}
.container-menu {
grid-template-columns: 1fr 1fr 20px 20px;
}
.readOnly {
grid-template-columns: 1fr 1fr;
}
.readOnly-menu {
grid-template-columns: 1fr 1fr 20px;
}
.control {
margin-top: 4px;
}
</style>

10
packages/builder/src/helpers/data/utils.js
View file

@ -119,3 +119,13 @@ export function flipHeaderState(headersActivity) {
})
return enabled
}
export default {
breakQueryString,
buildQueryString,
fieldsToSchema,
flipHeaderState,
keyValueToQueryParameters,
queryParametersToKeyValue,
schemaToFields,
}

54
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte Normal file
View file

@ -0,0 +1,54 @@
<script>
import { Input, ModalContent, Modal, Body } from "@budibase/bbui"
export let dynamicVariables
export let datasource
export let binding
let name, modal, valid, allVariableNames
export const show = () => {
modal.show()
}
export const hide = () => {
modal.hide()
}
function checkValid(vars, name) {
if (!name) {
return false
}
return !allVariableNames.find(
varName => varName.toLowerCase() === name.toLowerCase()
)
}
$: valid = checkValid(dynamicVariables, name)
$: allVariableNames = (datasource?.config?.dynamicVariables || []).map(
variable => variable.name
)
$: error = name && !valid ? "Variable name is already in use." : null
async function saveVariable() {
const copiedName = name,
copiedBinding = binding
name = null
binding = null
dynamicVariables[copiedName] = copiedBinding
}
</script>
<Modal bind:this={modal}>
<ModalContent
title="Add dynamic variable"
confirmText="Save"
onConfirm={saveVariable}
disabled={!valid}
>
<Body size="S"
>Specify a name for your new dynamic variable, this must be unique across
your datasource.</Body
>
<Input label="Variable name" bind:value={name} on:input {error} />
</ModalContent>
</Modal>

32
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte
View file

@ -16,17 +16,33 @@
export let query
export let bodyType
let text = ""
let json = ""
$: checkRequestBody(bodyType)
$: updateRequestBody(bodyType, text, json)
function checkRequestBody(type) {
if (!bodyType || !query) {
return
}
const currentType = typeof query?.fields.requestBody
if (objectTypes.includes(type) && currentType !== "object") {
query.fields.requestBody = {}
} else if (textTypes.includes(type) && currentType !== "string") {
query.fields.requestBody = ""
const isObject = objectTypes.includes(type)
const isText = textTypes.includes(type)
if (isText && currentType === "string") {
text = query.fields.requestBody
} else if (isObject && currentType === "object") {
json = query.fields.requestBody
}
}
function updateRequestBody(type, text, json) {
if (type === RawRestBodyTypes.NONE) {
query.fields.requestBody = null
} else if (objectTypes.includes(type)) {
query.fields.requestBody = json
} else {
query.fields.requestBody = text
}
}
@ -49,16 +65,12 @@
<Body size="S" weight="800">THE REQUEST DOES NOT HAVE A BODY</Body>
</div>
{:else if objectTypes.includes(bodyType)}
<KeyValueBuilder
bind:object={query.fields.requestBody}
name="param"
headings
/>
<KeyValueBuilder bind:object={json} name="param" headings />
{:else if textTypes.includes(bodyType)}
<CodeMirrorEditor
height={200}
mode={editorMode(bodyType)}
value={query.fields.requestBody}
value={text}
resize="vertical"
on:change={e => (query.fields.requestBody = e.detail)}
/>

22
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
View file

@ -21,11 +21,11 @@
import { cloneDeep } from "lodash/fp"
import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte"
import { onMount } from "svelte"
let importQueriesModal
let changed
let datasource
let integration, baseDatasource, datasource
let queryList
const querySchema = {
name: {},
queryVerb: { displayName: "Method" },
@ -34,11 +34,12 @@
$: baseDatasource = $datasources.list.find(
ds => ds._id === $datasources.selected
)
$: integration = datasource && $integrations[datasource.source]
$: queryList = $queries.list.filter(
query => query.datasourceId === datasource?._id
)
$: hasChanged(baseDatasource, datasource)
$: updateDatasource(baseDatasource)
function hasChanged(base, ds) {
if (base && ds) {
@ -66,9 +67,12 @@
$goto(`./${query._id}`)
}
onMount(() => {
datasource = cloneDeep(baseDatasource)
})
function updateDatasource(base) {
if (base) {
datasource = cloneDeep(base)
integration = $integrations[datasource.source]
}
}
</script>
<Modal bind:this={importQueriesModal}>
@ -142,7 +146,11 @@
</div>
{/if}
{#if datasource?.source === IntegrationTypes.REST}
<RestExtraConfigForm bind:datasource on:change={hasChanged} />
<RestExtraConfigForm
queries={queryList}
bind:datasource
on:change={hasChanged}
/>
{/if}
</Layout>
</section>

210
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
View file

@ -1,22 +1,22 @@
<script>
import { params } from "@roxi/routify"
import { datasources, integrations, queries, flags } from "stores/backend"
import { datasources, flags, integrations, queries } from "stores/backend"
import {
Layout,
Input,
Select,
Tabs,
Tab,
Banner,
Divider,
Button,
Heading,
RadioGroup,
Label,
Body,
TextArea,
Table,
Button,
Divider,
Heading,
Input,
Label,
Layout,
notifications,
RadioGroup,
Select,
Tab,
Table,
Tabs,
TextArea,
} from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import EditableLabel from "components/common/inputs/EditableLabel.svelte"
@ -26,42 +26,37 @@
import RestBodyInput from "../../_components/RestBodyInput.svelte"
import { capitalise } from "helpers"
import { onMount } from "svelte"
import {
fieldsToSchema,
schemaToFields,
breakQueryString,
buildQueryString,
keyValueToQueryParameters,
queryParametersToKeyValue,
flipHeaderState,
} from "helpers/data/utils"
import restUtils from "helpers/data/utils"
import {
RestBodyTypes as bodyTypes,
SchemaTypeOptions,
} from "constants/backend"
import JSONPreview from "components/integration/JSONPreview.svelte"
import AccessLevelSelect from "components/integration/AccessLevelSelect.svelte"
import DynamicVariableModal from "../../_components/DynamicVariableModal.svelte"
import Placeholder from "assets/bb-spaceship.svg"
import { cloneDeep } from "lodash/fp"
import { RawRestBodyTypes } from "constants/backend"
let query, datasource
let breakQs = {},
bindings = {}
let url = ""
let saveId, isGet
let saveId, url
let response, schema, enabledHeaders
let datasourceType, integrationInfo, queryConfig, responseSuccess
let authConfigId
let dynamicVariables, addVariableModal, varBinding
$: datasourceType = datasource?.source
$: integrationInfo = $integrations[datasourceType]
$: queryConfig = integrationInfo?.query
$: url = buildUrl(url, breakQs)
$: checkQueryName(url)
$: responseSuccess = response?.info?.code >= 200 && response?.info?.code < 400
$: isGet = query?.queryVerb === "read"
$: responseSuccess =
response?.info?.code >= 200 && response?.info?.code <= 206
$: authConfigs = buildAuthConfigs(datasource)
$: schemaReadOnly = !responseSuccess
$: variablesReadOnly = !responseSuccess
$: showVariablesTab = shouldShowVariables(dynamicVariables, variablesReadOnly)
function getSelectedQuery() {
return cloneDeep(
@ -89,7 +84,7 @@
if (!base) {
return base
}
const qs = buildQueryString(qsObj)
const qs = restUtils.buildQueryString(qsObj)
let newUrl = base
if (base.includes("?")) {
newUrl = base.split("?")[0]
@ -97,29 +92,15 @@
return qs.length > 0 ? `${newUrl}?${qs}` : newUrl
}
const buildAuthConfigs = datasource => {
if (datasource?.config?.authConfigs) {
return datasource.config.authConfigs.map(c => ({
label: c.name,
value: c._id,
}))
}
return []
}
function learnMoreBanner() {
window.open("https://docs.budibase.com/building-apps/data/transformers")
}
function buildQuery() {
const newQuery = { ...query }
const queryString = buildQueryString(breakQs)
const queryString = restUtils.buildQueryString(breakQs)
newQuery.fields.path = url.split("?")[0]
newQuery.fields.queryString = queryString
newQuery.fields.authConfigId = authConfigId
newQuery.fields.disabledHeaders = flipHeaderState(enabledHeaders)
newQuery.schema = fieldsToSchema(schema)
newQuery.parameters = keyValueToQueryParameters(bindings)
newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
newQuery.schema = restUtils.fieldsToSchema(schema)
newQuery.parameters = restUtils.keyValueToQueryParameters(bindings)
return newQuery
}
@ -130,8 +111,13 @@
saveId = _id
query = getSelectedQuery()
notifications.success(`Request saved successfully.`)
if (dynamicVariables) {
datasource.config.dynamicVariables = rebuildVariables(saveId)
datasource = await datasources.save(datasource)
}
} catch (err) {
notifications.error(`Error creating query. ${err.message}`)
notifications.error(`Error saving query. ${err.message}`)
}
}
@ -166,6 +152,78 @@
return id
}
const buildAuthConfigs = datasource => {
if (datasource?.config?.authConfigs) {
return datasource.config.authConfigs.map(c => ({
label: c.name,
value: c._id,
}))
}
return []
}
const schemaMenuItems = [
{
text: "Create dynamic variable",
onClick: input => {
varBinding = `{{ data.0.[${input.name}] }}`
addVariableModal.show()
},
},
]
const responseHeadersMenuItems = [
{
text: "Create dynamic variable",
onClick: input => {
varBinding = `{{ info.headers.[${input.name}] }}`
addVariableModal.show()
},
},
]
// convert dynamic variables list to simple key/val object
const getDynamicVariables = (datasource, queryId) => {
const variablesList = datasource?.config?.dynamicVariables
if (variablesList && variablesList.length > 0) {
const filtered = queryId
? variablesList.filter(variable => variable.queryId === queryId)
: variablesList
return filtered.reduce(
(acc, next) => ({ ...acc, [next.name]: next.value }),
{}
)
}
return {}
}
// convert dynamic variables object back to a list, enrich with query id
const rebuildVariables = queryId => {
let variables = []
if (dynamicVariables) {
variables = Object.entries(dynamicVariables).map(entry => {
return {
name: entry[0],
value: entry[1],
queryId,
}
})
}
let existing = datasource?.config?.dynamicVariables || []
// remove existing query variables (for changes and deletions)
existing = existing.filter(variable => variable.queryId !== queryId)
// re-add the new query variables
return [...existing, ...variables]
}
const shouldShowVariables = (dynamicVariables, variablesReadOnly) => {
return !!(
dynamicVariables &&
// show when editable or when read only and not empty
(!variablesReadOnly || Object.keys(dynamicVariables).length > 0)
)
}
onMount(async () => {
query = getSelectedQuery()
// clear any unsaved changes to the datasource
@ -173,14 +231,14 @@
datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
const datasourceUrl = datasource?.config.url
const qs = query?.fields.queryString
breakQs = breakQueryString(qs)
breakQs = restUtils.breakQueryString(qs)
if (datasourceUrl && !query.fields.path?.startsWith(datasourceUrl)) {
const path = query.fields.path
query.fields.path = `${datasource.config.url}/${path ? path : ""}`
}
url = buildUrl(query.fields.path, breakQs)
schema = schemaToFields(query.schema)
bindings = queryParametersToKeyValue(query.parameters)
schema = restUtils.schemaToFields(query.schema)
bindings = restUtils.queryParametersToKeyValue(query.parameters)
authConfigId = getAuthConfigId()
if (!query.fields.disabledHeaders) {
query.fields.disabledHeaders = {}
@ -191,7 +249,7 @@
query.fields.disabledHeaders[header] = false
}
}
enabledHeaders = flipHeaderState(query.fields.disabledHeaders)
enabledHeaders = restUtils.flipHeaderState(query.fields.disabledHeaders)
if (query && !query.transformer) {
query.transformer = "return data"
}
@ -201,11 +259,22 @@
}
}
if (query && !query.fields.bodyType) {
query.fields.bodyType = "none"
if (query.fields.requestBody) {
query.fields.bodyType = RawRestBodyTypes.JSON
} else {
query.fields.bodyType = RawRestBodyTypes.NONE
}
}
dynamicVariables = getDynamicVariables(datasource, query._id)
})
</script>
<DynamicVariableModal
{datasource}
{dynamicVariables}
bind:binding={varBinding}
bind:this={addVariableModal}
/>
{#if query && queryConfig}
<div class="inner">
<div class="top">
@ -275,7 +344,10 @@
{#if !$flags.queryTransformerBanner}
<Banner
extraButtonText="Learn more"
extraButtonAction={learnMoreBanner}
extraButtonAction={() =>
window.open(
"https://docs.budibase.com/building-apps/data/transformers"
)}
on:change={() =>
flags.updateFlag("queryTransformerBanner", true)}
>
@ -341,6 +413,9 @@
name="schema"
headings
options={SchemaTypeOptions}
menuItems={schemaMenuItems}
showMenu={!schemaReadOnly}
readOnly={schemaReadOnly}
/>
</Tab>
{/if}
@ -349,7 +424,12 @@
<TextArea disabled value={response.extra?.raw} height="300" />
</Tab>
<Tab title="Headers">
<KeyValueBuilder object={response.extra?.headers} readOnly />
<KeyValueBuilder
object={response.extra?.headers}
readOnly
menuItems={responseHeadersMenuItems}
showMenu={true}
/>
</Tab>
<Tab title="Preview">
<div class="table">
@ -364,6 +444,28 @@
{/if}
</div>
</Tab>
{/if}
{#if showVariablesTab}
<Tab title="Dynamic Variables">
<Layout noPadding gap="S">
<Body size="S">
Create dynamic variables based on response body or headers
from other queries.
</Body>
<KeyValueBuilder
bind:object={dynamicVariables}
name="Variable"
headings
keyHeading="Name"
keyPlaceholder="Variable name"
valueHeading={`Value`}
valuePlaceholder={`{{ value }}`}
readOnly={variablesReadOnly}
/>
</Layout>
</Tab>
{/if}
{#if response}
<div class="stats">
<Label size="L">
Status: <span class={responseSuccess ? "green" : "red"}

1
packages/builder/src/stores/backend/queries.js
View file

@ -91,6 +91,7 @@ export function createQueriesStore() {
{}
),
datasourceId: query.datasourceId,
queryId: query._id || undefined,
})
if (response.status !== 200) {

8
packages/client/manifest.json
View file

@ -2557,6 +2557,10 @@
"label": "Rows",
"key": "rows"
},
{
"label": "Extra Info",
"key": "info"
},
{
"label": "Rows Length",
"key": "rowsLength"
@ -3278,6 +3282,10 @@
"label": "Rows",
"key": "rows"
},
{
"label": "Extra Info",
"key": "info"
},
{
"label": "Rows Length",
"key": "rowsLength"

12
packages/client/src/api/datasources.js
View file

@ -19,7 +19,8 @@ export const fetchDatasource = async dataSource => {
// Fetch all rows in data source
const { type, tableId, fieldName } = dataSource
let rows = []
let rows = [],
info = {}
if (type === "table") {
rows = await fetchTableData(tableId)
} else if (type === "view") {
@ -32,7 +33,12 @@ export const fetchDatasource = async dataSource => {
parameters[param.name] = param.default
}
}
rows = await executeQuery({ queryId: dataSource._id, parameters })
const { data, ...rest } = await executeQuery({
queryId: dataSource._id,
parameters,
})
info = rest
rows = data
} else if (type === FieldTypes.LINK) {
rows = await fetchRelationshipData({
rowId: dataSource.rowId,
@ -42,7 +48,7 @@ export const fetchDatasource = async dataSource => {
}
// Enrich the result is always an array
return Array.isArray(rows) ? rows : []
return { rows: Array.isArray(rows) ? rows : [], info }
}
/**

2
packages/client/src/api/queries.js
View file

@ -11,7 +11,7 @@ export const executeQuery = async ({ queryId, parameters }) => {
return
}
const res = await API.post({
url: `/api/queries/${queryId}`,
url: `/api/v2/queries/${queryId}`,
body: {
parameters,
},

8
packages/client/src/components/app/DataProvider.svelte
View file

@ -30,6 +30,7 @@
// Provider state
let rows = []
let allRows = []
let info = {}
let schema = {}
let bookmarks = [null]
let pageNumber = 0
@ -120,8 +121,9 @@
// Build our data context
$: dataContext = {
rows,
info,
schema,
rowsLength: rows.length,
rowsLength: rows?.length,
// Undocumented properties. These aren't supposed to be used in builder
// bindings, but are used internally by other components
@ -209,7 +211,9 @@
} else {
// For other data sources like queries or views, fetch all rows from the
// server
allRows = await API.fetchDatasource(dataSource)
const data = await API.fetchDatasource(dataSource)
allRows = data.rows
info = data.info
}
loading = false
loaded = true

18
packages/server/__mocks__/node-fetch.ts
View file

@ -1,5 +1,6 @@
module FetchMock {
const fetch = jest.requireActual("node-fetch")
let failCount = 0
module.exports = async (url: any, opts: any) => {
function json(body: any, status = 200) {
@ -57,6 +58,23 @@ module FetchMock {
],
bookmark: "test",
})
} else if (url.includes("google.com")) {
return json({
url,
opts,
value: "<!doctype html><html itemscope=\"\" itemtype=\"http://schema.org/WebPage\" lang=\"en-GB\"></html>",
})
} else if (url.includes("failonce.com")) {
failCount++
if (failCount === 1) {
return json({ message: "error" }, 500)
} else {
return json({
fails: failCount - 1,
url,
opts,
})
}
}
return fetch(url, opts)
}

1
packages/server/package.json
View file

@ -122,6 +122,7 @@
"pouchdb-replication-stream": "1.2.9",
"server-destroy": "1.0.1",
"svelte": "^3.38.2",
"swagger-parser": "^10.0.3",
"to-json-schema": "0.2.5",
"uuid": "3.3.2",
"validate.js": "0.13.1",

6
packages/server/src/api/controllers/query/import/sources/curl.ts
View file

@ -17,8 +17,12 @@ const parseBody = (curl: any) => {
if (curl.data) {
const keys = Object.keys(curl.data)
if (keys.length) {
const key = keys[0]
let key = keys[0]
try {
// filter out the dollar syntax used by curl for shell support
if (key.startsWith("$")) {
key = key.substring(1)
}
return JSON.parse(key)
} catch (e) {
// do nothing

82
packages/server/src/api/controllers/query/index.js
View file

@ -1,4 +1,3 @@
const { processString } = require("@budibase/string-templates")
const CouchDB = require("../../../db")
const {
generateQueryID,
@ -90,47 +89,6 @@ exports.save = async function (ctx) {
ctx.message = `Query ${query.name} saved successfully.`
}
async function enrichQueryFields(fields, parameters = {}) {
const enrichedQuery = {}
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
enrichedQuery[key] = await processString(fields[key], parameters, {
noHelpers: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}
exports.find = async function (ctx) {
const db = new CouchDB(ctx.appId)
const query = enrichQueries(await db.get(ctx.params.queryId))
@ -146,16 +104,20 @@ exports.preview = async function (ctx) {
const db = new CouchDB(ctx.appId)
const datasource = await db.get(ctx.request.body.datasourceId)
const { fields, parameters, queryVerb, transformer } = ctx.request.body
const enrichedQuery = await enrichQueryFields(fields, parameters)
// preview may not have a queryId as it hasn't been saved, but if it does
// this stops dynamic variables from calling the same query
const { fields, parameters, queryVerb, transformer, queryId } =
ctx.request.body
try {
const { rows, keys, info, extra } = await Runner.run({
appId: ctx.appId,
datasource,
queryVerb,
query: enrichedQuery,
fields,
parameters,
transformer,
queryId,
})
ctx.body = {
@ -169,31 +131,41 @@ exports.preview = async function (ctx) {
}
}
exports.execute = async function (ctx) {
async function execute(ctx, opts = { rowsOnly: false }) {
const db = new CouchDB(ctx.appId)
const query = await db.get(ctx.params.queryId)
const datasource = await db.get(query.datasourceId)
const enrichedQuery = await enrichQueryFields(
query.fields,
ctx.request.body.parameters
)
// call the relevant CRUD method on the integration class
try {
const { rows } = await Runner.run({
const { rows, extra } = await Runner.run({
appId: ctx.appId,
datasource,
queryVerb: query.queryVerb,
query: enrichedQuery,
fields: query.fields,
parameters: ctx.request.body.parameters,
transformer: query.transformer,
queryId: ctx.params.queryId,
})
ctx.body = rows
if (opts && opts.rowsOnly) {
ctx.body = rows
} else {
ctx.body = { data: rows, ...extra }
}
} catch (err) {
ctx.throw(400, err)
}
}
exports.executeV1 = async function (ctx) {
return execute(ctx, { rowsOnly: true })
}
exports.executeV2 = async function (ctx) {
return execute(ctx, { rowsOnly: false })
}
exports.destroy = async function (ctx) {
const db = new CouchDB(ctx.appId)
await db.remove(ctx.params.queryId, ctx.params.revId)

3
packages/server/src/api/controllers/query/validation.js
View file

@ -36,6 +36,7 @@ exports.generateQueryPreviewValidation = () => {
extra: Joi.object().optional(),
datasourceId: Joi.string().required(),
transformer: Joi.string().optional(),
parameters: Joi.object({}).required().unknown(true)
parameters: Joi.object({}).required().unknown(true),
queryId: Joi.string().optional(),
}))
}

9
packages/server/src/api/routes/query.js
View file

@ -41,11 +41,18 @@ router
authorized(PermissionTypes.QUERY, PermissionLevels.READ),
queryController.find
)
// DEPRECATED - use new query endpoint for future work
.post(
"/api/queries/:queryId",
paramResource("queryId"),
authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
queryController.execute
queryController.executeV1
)
.post(
"/api/v2/queries/:queryId",
paramResource("queryId"),
authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
queryController.executeV2
)
.delete(
"/api/queries/:queryId/:revId",

88
packages/server/src/api/routes/tests/query.spec.js
View file

@ -1,5 +1,6 @@
// Mock out postgres for this
jest.mock("pg")
jest.mock("node-fetch")
// Mock isProdAppID to we can later mock the implementation and pretend we are
// using prod app IDs
@ -10,6 +11,7 @@ authDb.isProdAppID = mockIsProdAppID
const setup = require("./utilities")
const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
const { checkCacheForDynamicVariable } = require("../../../threads/utils")
const { basicQuery, basicDatasource } = setup.structures
describe("/queries", () => {
@ -226,4 +228,90 @@ describe("/queries", () => {
.expect(400)
})
})
describe("test variables", () => {
async function restDatasource(cfg) {
return await config.createDatasource({
datasource: {
...basicDatasource().datasource,
source: "REST",
config: cfg || {},
},
})
}
async function dynamicVariableDatasource() {
const datasource = await restDatasource()
const basedOnQuery = await config.createQuery({
...basicQuery(datasource._id),
fields: {
path: "www.google.com",
},
})
await config.updateDatasource({
...datasource,
config: {
dynamicVariables: [
{ queryId: basedOnQuery._id, name: "variable3", value: "{{ data.0.[value] }}" }
]
}
})
return { datasource, query: basedOnQuery }
}
async function preview(datasource, fields) {
return await request
.post(`/api/queries/preview`)
.send({
datasourceId: datasource._id,
parameters: {},
fields,
queryVerb: "read",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
}
it("should work with static variables", async () => {
const datasource = await restDatasource({
staticVariables: {
variable: "google",
variable2: "1",
},
})
const res = await preview(datasource, {
path: "www.{{ variable }}.com",
queryString: "test={{ variable2 }}",
})
// these responses come from the mock
expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
})
it("should work with dynamic variables", async () => {
const { datasource } = await dynamicVariableDatasource()
const res = await preview(datasource, {
path: "www.google.com",
queryString: "test={{ variable3 }}",
})
expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
expect(res.body.rows[0].url).toContain("doctype html")
})
it("check that it automatically retries on fail with cached dynamics", async () => {
const { datasource, query: base } = await dynamicVariableDatasource()
// preview once to cache
await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
// check its in cache
const contents = await checkCacheForDynamicVariable(base._id, "variable3")
expect(contents.rows.length).toEqual(1)
const res = await preview(datasource, {
path: "www.failonce.com",
queryString: "test={{ variable3 }}",
})
expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
expect(res.body.rows[0].fails).toEqual(1)
})
})
})

10
packages/server/src/automations/steps/executeQuery.js
View file

@ -35,6 +35,11 @@ exports.definition = {
type: "object",
description: "The response from the datasource execution",
},
info: {
type: "object",
description:
"Some query types may return extra data, like headers from a REST query",
},
success: {
type: "boolean",
description: "Whether the action was successful",
@ -68,13 +73,16 @@ exports.run = async function ({ inputs, appId, emitter }) {
try {
await queryController.execute(ctx)
const { data, ...rest } = ctx.body
return {
response: ctx.body,
response: data,
info: rest,
success: true,
}
} catch (err) {
return {
success: false,
info: {},
response: automationUtils.getError(err),
}
}

10
packages/server/src/definitions/datasource.ts
View file

@ -240,6 +240,16 @@ export interface RestConfig {
[key: string]: any
}
authConfigs: AuthConfig[]
staticVariables: {
[key: string]: string
}
dynamicVariables: [
{
name: string
queryId: string
value: string
}
]
}
export interface Query {

7
packages/server/src/integrations/rest.ts
View file

@ -48,7 +48,10 @@ module RestModule {
const { performance } = require("perf_hooks")
const FormData = require("form-data")
const { URLSearchParams } = require("url")
const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
const {
parseStringPromise: xmlParser,
Builder: XmlBuilder,
} = require("xml2js")
const SCHEMA: Integration = {
docs: "https://github.com/node-fetch/node-fetch",
@ -211,7 +214,7 @@ module RestModule {
break
case BodyTypes.XML:
if (object != null) {
string = (new XmlBuilder()).buildObject(object)
string = new XmlBuilder().buildObject(object)
}
input.body = string
input.headers["Content-Type"] = "application/xml"

10
packages/server/src/tests/utilities/TestConfiguration.js
View file

@ -316,6 +316,16 @@ class TestConfiguration {
return this.datasource
}
async updateDatasource(datasource) {
const response = await this._req(
datasource,
{ datasourceId: datasource._id },
controllers.datasource.update
)
this.datasource = response.datasource
return this.datasource
}
async createQuery(config = null) {
if (!this.datasource && !config) {
throw "No data source created for query."

203
packages/server/src/threads/query.js
View file

@ -1,77 +1,166 @@
require("./utils").threadSetup()
const threadUtils = require("./utils")
threadUtils.threadSetup()
const ScriptRunner = require("../utilities/scriptRunner")
const { integrations } = require("../integrations")
const { processStringSync } = require("@budibase/string-templates")
const CouchDB = require("../db")
function formatResponse(resp) {
if (typeof resp === "string") {
try {
resp = JSON.parse(resp)
} catch (err) {
resp = { response: resp }
class QueryRunner {
constructor(input, flags = { noRecursiveQuery: false }) {
this.appId = input.appId
this.datasource = input.datasource
this.queryVerb = input.queryVerb
this.fields = input.fields
this.parameters = input.parameters
this.transformer = input.transformer
this.queryId = input.queryId
this.noRecursiveQuery = flags.noRecursiveQuery
this.cachedVariables = []
// allows the response from a query to be stored throughout this
// execution so that if it needs to be re-used for another variable
// it can be
this.queryResponse = {}
this.hasRerun = false
}
async execute() {
let { datasource, fields, queryVerb, transformer } = this
// pre-query, make sure datasource variables are added to parameters
const parameters = await this.addDatasourceVariables()
const query = threadUtils.enrichQueryFields(fields, parameters)
const Integration = integrations[datasource.source]
if (!Integration) {
throw "Integration type does not exist."
}
}
return resp
}
const integration = new Integration(datasource.config)
function hasExtraData(response) {
return (
typeof response === "object" &&
!Array.isArray(response) &&
response.data != null &&
response.info != null
)
}
let output = threadUtils.formatResponse(await integration[queryVerb](query))
let rows = output,
info = undefined,
extra = undefined
if (threadUtils.hasExtraData(output)) {
rows = output.data
info = output.info
extra = output.extra
}
async function runAndTransform(datasource, queryVerb, query, transformer) {
const Integration = integrations[datasource.source]
if (!Integration) {
throw "Integration type does not exist."
}
const integration = new Integration(datasource.config)
// transform as required
if (transformer) {
const runner = new ScriptRunner(transformer, { data: rows })
rows = runner.execute()
}
let output = formatResponse(await integration[queryVerb](query))
let rows = output,
info = undefined,
extra = undefined
if (hasExtraData(output)) {
rows = output.data
info = output.info
extra = output.extra
// if the request fails we retry once, invalidating the cached value
if (
info &&
info.code >= 400 &&
this.cachedVariables.length > 0 &&
!this.hasRerun
) {
this.hasRerun = true
// invalidate the cache value
await threadUtils.invalidateDynamicVariables(this.cachedVariables)
return this.execute()
}
// needs to an array for next step
if (!Array.isArray(rows)) {
rows = [rows]
}
// map into JSON if just raw primitive here
if (rows.find(row => typeof row !== "object")) {
rows = rows.map(value => ({ value }))
}
// get all the potential fields in the schema
let keys = rows.flatMap(Object.keys)
if (integration.end) {
integration.end()
}
return { rows, keys, info, extra }
}
// transform as required
if (transformer) {
const runner = new ScriptRunner(transformer, { data: rows })
rows = runner.execute()
async runAnotherQuery(queryId, parameters) {
const db = new CouchDB(this.appId)
const query = await db.get(queryId)
const datasource = await db.get(query.datasourceId)
return new QueryRunner(
{
appId: this.appId,
datasource,
queryVerb: query.queryVerb,
fields: query.fields,
parameters,
transformer: query.transformer,
},
{ noRecursiveQuery: true }
).execute()
}
// needs to an array for next step
if (!Array.isArray(rows)) {
rows = [rows]
async getDynamicVariable(variable) {
let { parameters } = this
const queryId = variable.queryId,
name = variable.name
let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
if (!value) {
value = this.queryResponse[queryId]
? this.queryResponse[queryId]
: await this.runAnotherQuery(queryId, parameters)
// store incase this query is to be called again
this.queryResponse[queryId] = value
await threadUtils.storeDynamicVariable(queryId, name, value)
} else {
this.cachedVariables.push({ queryId, name })
}
return value
}
// map into JSON if just raw primitive here
if (rows.find(row => typeof row !== "object")) {
rows = rows.map(value => ({ value }))
async addDatasourceVariables() {
let { datasource, parameters, fields } = this
if (!datasource || !datasource.config) {
return parameters
}
const staticVars = datasource.config.staticVariables || {}
const dynamicVars = datasource.config.dynamicVariables || []
for (let [key, value] of Object.entries(staticVars)) {
if (!parameters[key]) {
parameters[key] = value
}
}
if (!this.noRecursiveQuery) {
// need to see if this uses any variables
const stringFields = JSON.stringify(fields)
const foundVars = dynamicVars.filter(variable => {
// don't allow a query to use its own dynamic variable (loop)
if (variable.queryId === this.queryId) {
return false
}
// look for {{ variable }} but allow spaces between handlebars
const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
return regex.test(stringFields)
})
const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
const responses = await Promise.all(dynamics)
for (let i = 0; i < foundVars.length; i++) {
const variable = foundVars[i]
parameters[variable.name] = processStringSync(variable.value, {
data: responses[i].rows,
info: responses[i].extra,
})
// make sure its known that this uses dynamic variables in case it fails
this.hasDynamicVariables = true
}
}
return parameters
}
// get all the potential fields in the schema
let keys = rows.flatMap(Object.keys)
if (integration.end) {
integration.end()
}
return { rows, keys, info, extra }
}
module.exports = (input, callback) => {
runAndTransform(
input.datasource,
input.queryVerb,
input.query,
input.transformer
)
const Runner = new QueryRunner(input)
Runner.execute()
.then(response => {
callback(null, response)
})

114
packages/server/src/threads/utils.js
View file

@ -1,6 +1,26 @@
const env = require("../environment")
const CouchDB = require("../db")
const { init } = require("@budibase/auth")
const redis = require("@budibase/auth/redis")
const { SEPARATOR } = require("@budibase/auth/db")
const { processStringSync } = require("@budibase/string-templates")
const VARIABLE_TTL_SECONDS = 3600
let client
const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/)
const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/)
async function getClient() {
if (!client) {
client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
}
return client
}
process.on("exit", async () => {
if (client) await client.finish()
})
exports.threadSetup = () => {
// don't run this if not threading
@ -11,3 +31,97 @@ exports.threadSetup = () => {
env.setInThread()
init(CouchDB)
}
function makeVariableKey(queryId, variable) {
return `${queryId}${SEPARATOR}${variable}`
}
exports.checkCacheForDynamicVariable = async (queryId, variable) => {
const cache = await getClient()
return cache.get(makeVariableKey(queryId, variable))
}
exports.invalidateDynamicVariables = async cachedVars => {
let promises = []
for (let variable of cachedVars) {
promises.push(
client.delete(makeVariableKey(variable.queryId, variable.name))
)
}
await Promise.all(promises)
}
exports.storeDynamicVariable = async (queryId, variable, value) => {
const cache = await getClient()
await cache.store(
makeVariableKey(queryId, variable),
value,
VARIABLE_TTL_SECONDS
)
}
exports.formatResponse = resp => {
if (typeof resp === "string") {
try {
resp = JSON.parse(resp)
} catch (err) {
resp = { response: resp }
}
}
return resp
}
exports.hasExtraData = response => {
return (
typeof response === "object" &&
!Array.isArray(response) &&
response.data != null &&
response.info != null
)
}
exports.enrichQueryFields = (fields, parameters = {}) => {
const enrichedQuery = {}
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
let value = fields[key]
// add triple brace to avoid escaping e.g. '=' in cookie header
if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
value = `{${value}}`
}
enrichedQuery[key] = processStringSync(value, parameters, {
noHelpers: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}

925
packages/server/yarn.lock
View file

File diff suppressed because it is too large Load diff

5
packages/worker/src/api/controllers/global/auth.js
View file

@ -12,7 +12,7 @@ const {
hash,
platformLogout,
} = authPkg.utils
const { Cookies } = authPkg.constants
const { Cookies, Headers } = authPkg.constants
const { passport } = authPkg.auth
const { checkResetPasswordCode } = require("../../../utilities/redis")
const {
@ -60,7 +60,10 @@ async function authInternal(ctx, user, err = null, info = null) {
return ctx.throw(403, info ? info : "Unauthorized")
}
// set a cookie for browser access
setCookie(ctx, user.token, Cookies.Auth, { sign: false })
// set the token in a header as well for APIs
ctx.set(Headers.TOKEN, user.token)
// get rid of any app cookies on login
// have to check test because this breaks cypress
if (!env.isTest()) {