1
0
Fork 0
mirror of synced 2024-06-29 19:41:03 +12:00

Merge branch 'feature/query-variables' of github.com:Budibase/budibase into rest-pagination

This commit is contained in:
Andrew Kingston 2021-12-17 13:12:28 +00:00
commit 7769ba4d6a
20 changed files with 653 additions and 204 deletions

View file

@ -16,6 +16,7 @@ exports.Databases = {
USER_CACHE: "users",
FLAGS: "flags",
APP_METADATA: "appMetadata",
QUERY_VARS: "queryVars",
}
exports.SEPARATOR = SEPARATOR

View file

@ -1,9 +1,18 @@
<script>
import { Divider, Heading, ActionButton, Badge, Body } from "@budibase/bbui"
import {
Divider,
Heading,
ActionButton,
Badge,
Body,
Layout,
} from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import RestAuthenticationBuilder from "./auth/RestAuthenticationBuilder.svelte"
import ViewDynamicVariables from "./variables/ViewDynamicVariables.svelte"
export let datasource
export let queries
let addHeader
</script>
@ -43,6 +52,36 @@
</Body>
<RestAuthenticationBuilder bind:configs={datasource.config.authConfigs} />
<Divider size="S" />
<div class="section-header">
<div class="badge">
<Heading size="S">Variables</Heading>
<Badge quiet grey>Optional</Badge>
</div>
</div>
<Body size="S"
>Variables enabled you to store and reuse values in queries. Static variables
use constant values while dynamic values can be bound to the response headers
or body of a query</Body
>
<Heading size="XS">Static</Heading>
<Layout noPadding gap="XS">
<KeyValueBuilder
name="Variable"
keyPlaceholder="Name"
headings
bind:object={datasource.config.staticVariables}
on:change
/>
</Layout>
<div />
<Heading size="XS">Dynamic</Heading>
<Body size="S">
Dynamic variables are evaluated when a dependant query is executed. The value
is cached for 24 hours and will re-evaluate if the dependendent query fails.
</Body>
<ViewDynamicVariables {queries} {datasource} />
<style>
.section-header {
display: flex;

View file

@ -0,0 +1,47 @@
<script>
import { Body, Table } from "@budibase/bbui"
import { queries as queriesStore } from "stores/backend"
import { goto } from "@roxi/routify"
export let datasource
export let queries
let dynamicVariables = []
$: enrichDynamicVariables(datasource, queries)
const dynamicVariableSchema = {
name: "",
value: "",
query: "",
}
const onClick = dynamicVariable => {
const queryId = dynamicVariable.queryId
queriesStore.select({ _id: queryId })
$goto(`./${queryId}`)
}
/**
* Add the query name to the dynamic variables
*/
function enrichDynamicVariables(ds, possibleQueries) {
dynamicVariables = []
ds.config.dynamicVariables?.forEach(dv => {
const query = possibleQueries.find(query => query._id === dv.queryId)
if (query) {
dynamicVariables.push({ ...dv, query: query.name })
}
})
}
</script>
<Table
on:click={({ detail }) => onClick(detail)}
schema={dynamicVariableSchema}
data={dynamicVariables}
allowEditColumns={false}
allowEditRows={false}
allowSelectRows={false}
/>
<Body size="S" />

View file

@ -6,6 +6,8 @@
Label,
Toggle,
Select,
ActionMenu,
MenuItem,
} from "@budibase/bbui"
import { createEventDispatcher } from "svelte"
import { lowercase } from "helpers"
@ -23,7 +25,11 @@
export let toggle
export let keyPlaceholder = "Key"
export let valuePlaceholder = "Value"
export let valueHeading
export let keyHeading
export let tooltip
export let menuItems
export let showMenu = false
let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
let fieldActivity = buildFieldActivity(activity)
@ -76,17 +82,24 @@
{#if Object.keys(object || {}).length > 0}
{#if headings}
<div class="container" class:container-active={toggle}>
<Label {tooltip}>{keyPlaceholder}</Label>
<Label>{valuePlaceholder}</Label>
<Label {tooltip}>{keyHeading || keyPlaceholder}</Label>
<Label>{valueHeading || valuePlaceholder}</Label>
{#if toggle}
<Label>Active</Label>
{/if}
</div>
{/if}
<div class="container" class:container-active={toggle} class:readOnly>
<div
class="container"
class:container-active={toggle}
class:container-menu={showMenu}
class:readOnly
class:readOnly-menu={readOnly && showMenu}
>
{#each fields as field, idx}
<Input
placeholder={keyPlaceholder}
readonly={readOnly}
bind:value={field.name}
on:change={changed}
/>
@ -95,6 +108,7 @@
{:else}
<Input
placeholder={valuePlaceholder}
readonly={readOnly}
bind:value={field.value}
on:change={changed}
/>
@ -105,6 +119,18 @@
{#if !readOnly}
<Icon hoverable name="Close" on:click={() => deleteEntry(idx)} />
{/if}
{#if menuItems?.length > 0 && showMenu}
<ActionMenu>
<div slot="control" class="control icon">
<Icon size="S" hoverable name="MoreSmallList" />
</div>
{#each menuItems as item}
<MenuItem on:click={() => item.onClick(field)}>
{item.text}
</MenuItem>
{/each}
</ActionMenu>
{/if}
{/each}
</div>
{/if}
@ -127,7 +153,16 @@
.container-active {
grid-template-columns: 1fr 1fr 50px 20px;
}
.container-menu {
grid-template-columns: 1fr 1fr 20px 20px;
}
.readOnly {
grid-template-columns: 1fr 1fr;
}
.readOnly-menu {
grid-template-columns: 1fr 1fr 20px;
}
.control {
margin-top: 4px;
}
</style>

View file

@ -119,3 +119,62 @@ export function flipHeaderState(headersActivity) {
})
return enabled
}
// convert dynamic variables list to simple key/val object
export function variablesToObject(datasource) {
const variablesList = datasource?.config?.dynamicVariables
if (variablesList && variablesList.length > 0) {
return variablesList.reduce(
(acc, next) => ({ ...acc, [next.name]: next.value }),
{}
)
}
return {}
}
// convert dynamic variables object back to a list, enrich with query id
export function rebuildVariables(queryId, variables) {
let vars = []
if (variables) {
vars = Object.entries(variables).map(entry => {
return {
name: entry[0],
value: entry[1],
queryId,
}
})
}
return vars
}
export function shouldShowVariables(dynamicVariables, variablesReadOnly) {
return !!(
dynamicVariables &&
// show when editable or when read only and not empty
(!variablesReadOnly || Object.keys(dynamicVariables).length > 0)
)
}
export function buildAuthConfigs(datasource) {
if (datasource?.config?.authConfigs) {
return datasource.config.authConfigs.map(c => ({
label: c.name,
value: c._id,
}))
}
return []
}
export default {
breakQueryString,
buildQueryString,
fieldsToSchema,
flipHeaderState,
keyValueToQueryParameters,
queryParametersToKeyValue,
schemaToFields,
variablesToObject,
rebuildVariables,
shouldShowVariables,
buildAuthConfigs,
}

View file

@ -0,0 +1,45 @@
<script>
import { Input, ModalContent, Modal } from "@budibase/bbui"
export let dynamicVariables
export let binding
let name, modal, valid
export const show = () => {
modal.show()
}
export const hide = () => {
modal.hide()
}
function checkValid(vars, name) {
if (!name) {
return false
}
const varKeys = Object.keys(vars || {})
return varKeys.find(key => key.toLowerCase() === name.toLowerCase()) == null
}
$: valid = checkValid(dynamicVariables, name)
$: error = name && !valid ? "Variable name is already in use." : null
async function saveVariable() {
const copiedName = name,
copiedBinding = binding
name = null
binding = null
dynamicVariables[copiedName] = copiedBinding
}
</script>
<Modal bind:this={modal}>
<ModalContent
title="Add dynamic variable"
confirmText="Save"
onConfirm={saveVariable}
disabled={!valid}
>
<Input label="Variable name" bind:value={name} on:input {error} />
</ModalContent>
</Modal>

View file

@ -21,11 +21,11 @@
import { cloneDeep } from "lodash/fp"
import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte"
import { onMount } from "svelte"
let importQueriesModal
let changed
let datasource
let integration, baseDatasource, datasource
let queryList
const querySchema = {
name: {},
queryVerb: { displayName: "Method" },
@ -34,11 +34,12 @@
$: baseDatasource = $datasources.list.find(
ds => ds._id === $datasources.selected
)
$: integration = datasource && $integrations[datasource.source]
$: queryList = $queries.list.filter(
query => query.datasourceId === datasource?._id
)
$: hasChanged(baseDatasource, datasource)
$: updateDatasource(baseDatasource)
function hasChanged(base, ds) {
if (base && ds) {
@ -66,9 +67,12 @@
$goto(`./${query._id}`)
}
onMount(() => {
datasource = cloneDeep(baseDatasource)
})
function updateDatasource(base) {
if (base) {
datasource = cloneDeep(base)
integration = $integrations[datasource.source]
}
}
</script>
<Modal bind:this={importQueriesModal}>
@ -142,7 +146,11 @@
</div>
{/if}
{#if datasource?.source === IntegrationTypes.REST}
<RestExtraConfigForm bind:datasource on:change={hasChanged} />
<RestExtraConfigForm
queries={queryList}
bind:datasource
on:change={hasChanged}
/>
{/if}
</Layout>
</section>

View file

@ -1,22 +1,22 @@
<script>
import { params } from "@roxi/routify"
import { datasources, integrations, queries, flags } from "stores/backend"
import { datasources, flags, integrations, queries } from "stores/backend"
import {
Layout,
Input,
Select,
Tabs,
Tab,
Banner,
Divider,
Button,
Heading,
RadioGroup,
Label,
Body,
TextArea,
Table,
Button,
Divider,
Heading,
Input,
Label,
Layout,
notifications,
RadioGroup,
Select,
Tab,
Table,
Tabs,
TextArea,
} from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import EditableLabel from "components/common/inputs/EditableLabel.svelte"
@ -26,15 +26,7 @@
import RestBodyInput from "../../_components/RestBodyInput.svelte"
import { capitalise } from "helpers"
import { onMount } from "svelte"
import {
fieldsToSchema,
schemaToFields,
breakQueryString,
buildQueryString,
keyValueToQueryParameters,
queryParametersToKeyValue,
flipHeaderState,
} from "helpers/data/utils"
import restUtils from "helpers/data/utils"
import {
RestBodyTypes as bodyTypes,
SchemaTypeOptions,
@ -43,27 +35,32 @@
} from "constants/backend"
import JSONPreview from "components/integration/JSONPreview.svelte"
import AccessLevelSelect from "components/integration/AccessLevelSelect.svelte"
import DynamicVariableModal from "../../_components/DynamicVariableModal.svelte"
import Placeholder from "assets/bb-spaceship.svg"
import { cloneDeep } from "lodash/fp"
let query, datasource
let breakQs = {},
bindings = {}
let url = ""
let saveId, isGet
let saveId, url
let response, schema, enabledHeaders
let datasourceType, integrationInfo, queryConfig, responseSuccess
let authConfigId
let dynamicVariables, addVariableModal, varBinding
$: datasourceType = datasource?.source
$: integrationInfo = $integrations[datasourceType]
$: queryConfig = integrationInfo?.query
$: url = buildUrl(url, breakQs)
$: checkQueryName(url)
$: responseSuccess = response?.info?.code >= 200 && response?.info?.code < 400
$: isGet = query?.queryVerb === "read"
$: responseSuccess =
response?.info?.code >= 200 && response?.info?.code <= 206
$: authConfigs = buildAuthConfigs(datasource)
$: authConfigs = restUtils.buildAuthConfigs(datasource)
$: schemaReadOnly = !responseSuccess
$: variablesReadOnly = !responseSuccess
$: showVariablesTab = restUtils.shouldShowVariables(
dynamicVariables,
variablesReadOnly
)
function getSelectedQuery() {
return cloneDeep(
@ -91,7 +88,7 @@
if (!base) {
return base
}
const qs = buildQueryString(qsObj)
const qs = restUtils.buildQueryString(qsObj)
let newUrl = base
if (base.includes("?")) {
newUrl = base.split("?")[0]
@ -99,29 +96,15 @@
return qs.length > 0 ? `${newUrl}?${qs}` : newUrl
}
const buildAuthConfigs = datasource => {
if (datasource?.config?.authConfigs) {
return datasource.config.authConfigs.map(c => ({
label: c.name,
value: c._id,
}))
}
return []
}
function learnMoreBanner() {
window.open("https://docs.budibase.com/building-apps/data/transformers")
}
function buildQuery() {
const newQuery = { ...query }
const queryString = buildQueryString(breakQs)
const queryString = restUtils.buildQueryString(breakQs)
newQuery.fields.path = url.split("?")[0]
newQuery.fields.queryString = queryString
newQuery.fields.authConfigId = authConfigId
newQuery.fields.disabledHeaders = flipHeaderState(enabledHeaders)
newQuery.schema = fieldsToSchema(schema)
newQuery.parameters = keyValueToQueryParameters(bindings)
newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
newQuery.schema = restUtils.fieldsToSchema(schema)
newQuery.parameters = restUtils.keyValueToQueryParameters(bindings)
return newQuery
}
@ -132,8 +115,16 @@
saveId = _id
query = getSelectedQuery()
notifications.success(`Request saved successfully.`)
if (dynamicVariables) {
datasource.config.dynamicVariables = restUtils.rebuildVariables(
saveId,
dynamicVariables
)
datasource = await datasources.save(datasource)
}
} catch (err) {
notifications.error(`Error creating query. ${err.message}`)
notifications.error(`Error saving query. ${err.message}`)
}
}
@ -168,6 +159,25 @@
return id
}
const schemaMenuItems = [
{
text: "Create dynamic variable",
onClick: input => {
varBinding = `{{ data.0.[${input.name}] }}`
addVariableModal.show()
},
},
]
const responseHeadersMenuItems = [
{
text: "Create dynamic variable",
onClick: input => {
varBinding = `{{ info.headers.[${input.name}] }}`
addVariableModal.show()
},
},
]
onMount(async () => {
query = getSelectedQuery()
// clear any unsaved changes to the datasource
@ -175,14 +185,14 @@
datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
const datasourceUrl = datasource?.config.url
const qs = query?.fields.queryString
breakQs = breakQueryString(qs)
breakQs = restUtils.breakQueryString(qs)
if (datasourceUrl && !query.fields.path?.startsWith(datasourceUrl)) {
const path = query.fields.path
query.fields.path = `${datasource.config.url}/${path ? path : ""}`
}
url = buildUrl(query.fields.path, breakQs)
schema = schemaToFields(query.schema)
bindings = queryParametersToKeyValue(query.parameters)
schema = restUtils.schemaToFields(query.schema)
bindings = restUtils.queryParametersToKeyValue(query.parameters)
authConfigId = getAuthConfigId()
if (!query.fields.disabledHeaders) {
query.fields.disabledHeaders = {}
@ -193,7 +203,7 @@
query.fields.disabledHeaders[header] = false
}
}
enabledHeaders = flipHeaderState(query.fields.disabledHeaders)
enabledHeaders = restUtils.flipHeaderState(query.fields.disabledHeaders)
if (query && !query.transformer) {
query.transformer = "return data"
}
@ -208,9 +218,16 @@
if (query && !query.pagination) {
query.pagination = {}
}
dynamicVariables = restUtils.variablesToObject(datasource)
})
</script>
<DynamicVariableModal
{datasource}
{dynamicVariables}
bind:binding={varBinding}
bind:this={addVariableModal}
/>
{#if query && queryConfig}
<div class="inner">
<div class="top">
@ -317,7 +334,10 @@
{#if !$flags.queryTransformerBanner}
<Banner
extraButtonText="Learn more"
extraButtonAction={learnMoreBanner}
extraButtonAction={() =>
window.open(
"https://docs.budibase.com/building-apps/data/transformers"
)}
on:change={() =>
flags.updateFlag("queryTransformerBanner", true)}
>
@ -383,6 +403,9 @@
name="schema"
headings
options={SchemaTypeOptions}
menuItems={schemaMenuItems}
showMenu={!schemaReadOnly}
readOnly={schemaReadOnly}
/>
</Tab>
{/if}
@ -391,7 +414,12 @@
<TextArea disabled value={response.extra?.raw} height="300" />
</Tab>
<Tab title="Headers">
<KeyValueBuilder object={response.extra?.headers} readOnly />
<KeyValueBuilder
object={response.extra?.headers}
readOnly
menuItems={responseHeadersMenuItems}
showMenu={true}
/>
</Tab>
<Tab title="Preview">
<div class="table">
@ -406,6 +434,27 @@
{/if}
</div>
</Tab>
{/if}
{#if showVariablesTab}
<Tab title="Dynamic Variables">
<Layout noPadding gap="S">
<Body size="S"
>{"Create dynamic variables to use body and headers results in other queries"}</Body
>
<KeyValueBuilder
bind:object={dynamicVariables}
name="Variable"
headings
keyHeading="Name"
keyPlaceholder="Variable name"
valueHeading={`Value`}
valuePlaceholder={`{{ value }}`}
readOnly={variablesReadOnly}
/>
</Layout>
</Tab>
{/if}
{#if response}
<div class="stats">
<Label size="L">
Status: <span class={responseSuccess ? "green" : "red"}

View file

@ -2457,6 +2457,10 @@
"label": "Rows",
"key": "rows"
},
{
"label": "Extra Info",
"key": "info"
},
{
"label": "Rows Length",
"key": "rowsLength"
@ -3178,6 +3182,10 @@
"label": "Rows",
"key": "rows"
},
{
"label": "Extra Info",
"key": "info"
},
{
"label": "Rows Length",
"key": "rowsLength"

View file

@ -11,7 +11,7 @@ export const executeQuery = async ({ queryId, parameters }) => {
return
}
const res = await API.post({
url: `/api/queries/${queryId}`,
url: `/api/v2/queries/${queryId}`,
body: {
parameters,
},

View file

@ -66,6 +66,7 @@
// Build our data context
$: dataContext = {
rows: $fetch.rows,
info: $fetch.info,
schema: $fetch.schema,
rowsLength: $fetch.rows.length,

View file

@ -40,6 +40,7 @@ export default class TableFetch {
// State of the fetch
store = writable({
rows: [],
info: null,
schema: null,
loading: false,
loaded: false,
@ -145,12 +146,16 @@ export default class TableFetch {
// Actually fetch data
const page = await this.getPage()
if (page.info) {
console.log("new info", page.info)
}
this.store.update($store => ({
...$store,
loading: false,
loaded: true,
pageNumber: 0,
rows: page.rows,
info: page.info,
cursors: page.hasNextPage ? [null, page.cursor] : [null],
}))
}
@ -163,7 +168,7 @@ export default class TableFetch {
const { query } = get(this.store)
// Get the actual data
let { rows, hasNextPage, cursor } = await this.getData()
let { rows, info, hasNextPage, cursor } = await this.getData()
// If we don't support searching, do a client search
if (!this.supportsSearch) {
@ -182,6 +187,7 @@ export default class TableFetch {
return {
rows,
info,
hasNextPage,
cursor,
}
@ -194,6 +200,7 @@ export default class TableFetch {
async getData() {
return {
rows: [],
info: null,
hasNextPage: false,
cursor: null,
}
@ -272,9 +279,9 @@ export default class TableFetch {
if (get(this.store).loading) {
return
}
const { rows } = await this.getPage()
this.store.update($store => ({ ...$store, loading: true }))
this.store.update($store => ({ ...$store, rows, loading: false }))
const { rows, info } = await this.getPage()
this.store.update($store => ({ ...$store, rows, info, loading: false }))
}
/**
@ -313,7 +320,7 @@ export default class TableFetch {
loading: true,
cursor: nextCursor,
}))
const { rows, hasNextPage, cursor } = await this.getPage()
const { rows, info, hasNextPage, cursor } = await this.getPage()
// Update state
this.store.update($store => {
@ -325,6 +332,7 @@ export default class TableFetch {
...$store,
pageNumber: pageNumber + 1,
rows,
info,
cursors,
loading: false,
}
@ -347,7 +355,7 @@ export default class TableFetch {
loading: true,
cursor: prevCursor,
}))
const { rows } = await this.getPage()
const { rows, info } = await this.getPage()
// Update state
this.store.update($store => {
@ -355,6 +363,7 @@ export default class TableFetch {
...$store,
pageNumber: $store.pageNumber - 1,
rows,
info,
loading: false,
}
})

View file

@ -22,9 +22,13 @@ export default class ViewFetch extends DataFetch {
}
}
const res = await executeQuery({ queryId: datasource?._id, parameters })
const { data, ...rest } = await executeQuery({
queryId: datasource?._id,
parameters,
})
return {
rows: res || [],
rows: data || [],
info: rest,
}
}
}

View file

@ -121,6 +121,7 @@
"pouchdb-replication-stream": "1.2.9",
"server-destroy": "1.0.1",
"svelte": "^3.38.2",
"swagger-parser": "^10.0.3",
"to-json-schema": "0.2.5",
"uuid": "3.3.2",
"validate.js": "0.13.1",

View file

@ -1,4 +1,3 @@
const { processString } = require("@budibase/string-templates")
const CouchDB = require("../../../db")
const {
generateQueryID,
@ -90,47 +89,6 @@ exports.save = async function (ctx) {
ctx.message = `Query ${query.name} saved successfully.`
}
async function enrichQueryFields(fields, parameters = {}) {
const enrichedQuery = {}
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
enrichedQuery[key] = await processString(fields[key], parameters, {
noHelpers: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}
exports.find = async function (ctx) {
const db = new CouchDB(ctx.appId)
const query = enrichQueries(await db.get(ctx.params.queryId))
@ -148,13 +106,14 @@ exports.preview = async function (ctx) {
const datasource = await db.get(ctx.request.body.datasourceId)
const { fields, parameters, queryVerb, transformer } = ctx.request.body
const enrichedQuery = await enrichQueryFields(fields, parameters)
try {
const { rows, keys, info, extra } = await Runner.run({
appId: ctx.appId,
datasource,
queryVerb,
query: enrichedQuery,
fields,
parameters,
transformer,
})
@ -169,31 +128,40 @@ exports.preview = async function (ctx) {
}
}
exports.execute = async function (ctx) {
async function execute(ctx, opts = { rowsOnly: false }) {
const db = new CouchDB(ctx.appId)
const query = await db.get(ctx.params.queryId)
const datasource = await db.get(query.datasourceId)
const enrichedQuery = await enrichQueryFields(
query.fields,
ctx.request.body.parameters
)
// call the relevant CRUD method on the integration class
try {
const { rows } = await Runner.run({
const { rows, extra } = await Runner.run({
appId: ctx.appId,
datasource,
queryVerb: query.queryVerb,
query: enrichedQuery,
fields: query.fields,
parameters: ctx.request.body.parameter,
transformer: query.transformer,
})
ctx.body = rows
if (opts && opts.rowsOnly) {
ctx.body = rows
} else {
ctx.body = { data: rows, ...extra }
}
} catch (err) {
ctx.throw(400, err)
}
}
exports.executeV1 = async function (ctx) {
return execute(ctx, { rowsOnly: true })
}
exports.executeV2 = async function (ctx) {
return execute(ctx, { rowsOnly: false })
}
exports.destroy = async function (ctx) {
const db = new CouchDB(ctx.appId)
await db.remove(ctx.params.queryId, ctx.params.revId)

View file

@ -41,11 +41,18 @@ router
authorized(PermissionTypes.QUERY, PermissionLevels.READ),
queryController.find
)
// DEPRECATED - use new query endpoint for future work
.post(
"/api/queries/:queryId",
paramResource("queryId"),
authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
queryController.execute
queryController.executeV1
)
.post(
"/api/v2/queries/:queryId",
paramResource("queryId"),
authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
queryController.executeV2
)
.delete(
"/api/queries/:queryId/:revId",

View file

@ -35,6 +35,11 @@ exports.definition = {
type: "object",
description: "The response from the datasource execution",
},
info: {
type: "object",
description:
"Some query types may return extra data, like headers from a REST query",
},
success: {
type: "boolean",
description: "Whether the action was successful",
@ -68,13 +73,16 @@ exports.run = async function ({ inputs, appId, emitter }) {
try {
await queryController.execute(ctx)
const { data, ...rest } = ctx.body
return {
response: ctx.body,
response: data,
info: rest,
success: true,
}
} catch (err) {
return {
success: false,
info: {},
response: automationUtils.getError(err),
}
}

View file

@ -1,77 +1,134 @@
require("./utils").threadSetup()
const threadUtils = require("./utils")
threadUtils.threadSetup()
const ScriptRunner = require("../utilities/scriptRunner")
const { integrations } = require("../integrations")
const { processStringSync } = require("@budibase/string-templates")
const CouchDB = require("../db")
function formatResponse(resp) {
if (typeof resp === "string") {
try {
resp = JSON.parse(resp)
} catch (err) {
resp = { response: resp }
class QueryRunner {
constructor(input, flags = { noRecursiveQuery: false }) {
this.appId = input.appId
this.datasource = input.datasource
this.queryVerb = input.queryVerb
this.fields = input.fields
this.parameters = input.parameters
this.transformer = input.transformer
this.noRecursiveQuery = flags.noRecursiveQuery
}
async execute() {
let { datasource, fields, queryVerb, transformer } = this
// pre-query, make sure datasource variables are added to parameters
const parameters = await this.addDatasourceVariables()
const query = threadUtils.enrichQueryFields(fields, parameters)
const Integration = integrations[datasource.source]
if (!Integration) {
throw "Integration type does not exist."
}
}
return resp
}
const integration = new Integration(datasource.config)
function hasExtraData(response) {
return (
typeof response === "object" &&
!Array.isArray(response) &&
response.data != null &&
response.info != null
)
}
let output = threadUtils.formatResponse(await integration[queryVerb](query))
let rows = output,
info = undefined,
extra = undefined
if (threadUtils.hasExtraData(output)) {
rows = output.data
info = output.info
extra = output.extra
}
async function runAndTransform(datasource, queryVerb, query, transformer) {
const Integration = integrations[datasource.source]
if (!Integration) {
throw "Integration type does not exist."
}
const integration = new Integration(datasource.config)
// transform as required
if (transformer) {
const runner = new ScriptRunner(transformer, { data: rows })
rows = runner.execute()
}
let output = formatResponse(await integration[queryVerb](query))
let rows = output,
info = undefined,
extra = undefined
if (hasExtraData(output)) {
rows = output.data
info = output.info
extra = output.extra
// needs to an array for next step
if (!Array.isArray(rows)) {
rows = [rows]
}
// map into JSON if just raw primitive here
if (rows.find(row => typeof row !== "object")) {
rows = rows.map(value => ({ value }))
}
// get all the potential fields in the schema
let keys = rows.flatMap(Object.keys)
if (integration.end) {
integration.end()
}
return { rows, keys, info, extra }
}
// transform as required
if (transformer) {
const runner = new ScriptRunner(transformer, { data: rows })
rows = runner.execute()
async runAnotherQuery(queryId, parameters) {
const db = new CouchDB(this.appId)
const query = await db.get(queryId)
const datasource = await db.get(query.datasourceId)
return new QueryRunner(
{
appId: this.appId,
datasource,
queryVerb: query.queryVerb,
fields: query.fields,
parameters,
transformer: query.transformer,
},
{ noRecursiveQuery: true }
).execute()
}
// needs to an array for next step
if (!Array.isArray(rows)) {
rows = [rows]
async getDynamicVariable(variable) {
let { parameters } = this
const queryId = variable.queryId,
name = variable.name
let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
if (!value) {
value = await this.runAnotherQuery(queryId, parameters)
await threadUtils.storeDynamicVariable(queryId, name, value)
}
return value
}
// map into JSON if just raw primitive here
if (rows.find(row => typeof row !== "object")) {
rows = rows.map(value => ({ value }))
async addDatasourceVariables() {
let { datasource, parameters, fields } = this
if (!datasource || !datasource.config) {
return parameters
}
const staticVars = datasource.config.staticVariables || {}
const dynamicVars = datasource.config.dynamicVariables || []
for (let [key, value] of Object.entries(staticVars)) {
if (!parameters[key]) {
parameters[key] = value
}
}
if (!this.noRecursiveQuery) {
// need to see if this uses any variables
const stringFields = JSON.stringify(fields)
const foundVars = dynamicVars.filter(variable => {
// look for {{ variable }} but allow spaces between handlebars
const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
return regex.test(stringFields)
})
const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
const responses = await Promise.all(dynamics)
for (let i = 0; i < foundVars.length; i++) {
const variable = foundVars[i]
parameters[variable.name] = processStringSync(variable.value, {
data: responses[i].rows,
info: responses[i].extra,
})
}
}
return parameters
}
// get all the potential fields in the schema
let keys = rows.flatMap(Object.keys)
if (integration.end) {
integration.end()
}
return { rows, keys, info, extra }
}
module.exports = (input, callback) => {
runAndTransform(
input.datasource,
input.queryVerb,
input.query,
input.transformer
)
const Runner = new QueryRunner(input)
Runner.execute()
.then(response => {
callback(null, response)
})

View file

@ -1,6 +1,23 @@
const env = require("../environment")
const CouchDB = require("../db")
const { init } = require("@budibase/auth")
const redis = require("@budibase/auth/redis")
const { SEPARATOR } = require("@budibase/auth/db")
const { processStringSync } = require("@budibase/string-templates")
const VARIABLE_TTL_SECONDS = 3600
let client
async function getClient() {
if (!client) {
client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
}
return client
}
process.on("exit", async () => {
if (client) await client.finish()
})
exports.threadSetup = () => {
// don't run this if not threading
@ -11,3 +28,82 @@ exports.threadSetup = () => {
env.setInThread()
init(CouchDB)
}
function makeVariableKey(queryId, variable) {
return `${queryId}${SEPARATOR}${variable}`
}
exports.checkCacheForDynamicVariable = async (queryId, variable) => {
const cache = await getClient()
return cache.get(makeVariableKey(queryId, variable))
}
exports.storeDynamicVariable = async (queryId, variable, value) => {
const cache = await getClient()
await cache.store(
makeVariableKey(queryId, variable),
value,
VARIABLE_TTL_SECONDS
)
}
exports.formatResponse = resp => {
if (typeof resp === "string") {
try {
resp = JSON.parse(resp)
} catch (err) {
resp = { response: resp }
}
}
return resp
}
exports.hasExtraData = response => {
return (
typeof response === "object" &&
!Array.isArray(response) &&
response.data != null &&
response.info != null
)
}
exports.enrichQueryFields = (fields, parameters = {}) => {
const enrichedQuery = {}
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
enrichedQuery[key] = processStringSync(fields[key], parameters, {
noHelpers: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}

View file

@ -27,7 +27,7 @@
resolved "https://registry.yarnpkg.com/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz#b789a362e055b0340d04712eafe7027ddc1ac267"
integrity sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==
"@apidevtools/swagger-parser@^10.0.3":
"@apidevtools/swagger-parser@10.0.3":
version "10.0.3"
resolved "https://registry.yarnpkg.com/@apidevtools/swagger-parser/-/swagger-parser-10.0.3.tgz#32057ae99487872c4dd96b314a1ab4b95d89eaf5"
integrity sha512-sNiLY51vZOmSPFZA5TF35KZ2HbgYklQnTSDnkghamzLb3EkNtcQnrBQEj5AOCxHpTtXpqMCRM1CrmV2rG6nw4g==
@ -983,10 +983,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/auth@^1.0.19-alpha.1":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.22.tgz#a93ea2fea46e00138ad3fa129c9ea19b056654e2"
integrity sha512-eHCNEzGl6HxYlMpfRTXBokq2ALTK5f+CDSgJmGaL/jfPc2NlzCI5NoigZUkSrdwDiYZnnWLfDDR4dArYyLlFuA==
"@budibase/auth@^1.0.23":
version "1.0.24"
resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.24.tgz#69a015a7fe9dcc9ea9e471a7d76d2c0544de0b16"
integrity sha512-pX+x+ib5X8s9ek4TZVmddRXlCE3kFmK5EH7Qk0Wmcn9tZwbaZy1XXd4qlJ7aFGSmvy1VNbz5AXIyRtliescaEQ==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1056,10 +1056,10 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/bbui@^1.0.22":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.22.tgz#ac3bd3a8699bd0be84aac3c5dff9d093e5b08462"
integrity sha512-8/5rXEOwkr0OcQD1fn5GpmI3d5dS1cIJBAODjTVtlZrTdacwlz5W2j3zIh+CBG0X7zhVxEze3zs2b1vDNTvK6A==
"@budibase/bbui@^1.0.24":
version "1.0.24"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.24.tgz#15a976b500c4103bc3e94ed665c6c557bb6271bf"
integrity sha512-4zr+fSChOelHDqNNM2OiWWbWOL57JGBVBx333GiMr2P7N0MoepjwHEp9MlITWl5mUfS8rxgWTC9mSSMpRdpSUQ==
dependencies:
"@adobe/spectrum-css-workflow-icons" "^1.2.1"
"@spectrum-css/actionbutton" "^1.0.1"
@ -1106,14 +1106,14 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/client@^1.0.19-alpha.1":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.22.tgz#80d6c3fb2b57a050199dde4a4b3e82b221601c25"
integrity sha512-Cpao7l2lIWyJZJs8+zq1wFnQGaWRTDiRG+HkkjvqQZDkZexlo89zWPkY56NBbMT1qAXd6K3zAdRNNKVCBCtOaA==
"@budibase/client@^1.0.23":
version "1.0.24"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.24.tgz#69be5b66cc701c6018c816728975a076230c052a"
integrity sha512-ppb+exifHCfuDGqs1cQ0kzc3wWaOGkVgq/8Rdb7R7BLIOKYkB67L3FvkqTl+39wrc32CpmzOXXQ73keE4l6P8Q==
dependencies:
"@budibase/bbui" "^1.0.22"
"@budibase/bbui" "^1.0.24"
"@budibase/standard-components" "^0.9.139"
"@budibase/string-templates" "^1.0.22"
"@budibase/string-templates" "^1.0.24"
regexparam "^1.3.0"
shortid "^2.2.15"
svelte-spa-router "^3.0.5"
@ -1163,10 +1163,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/string-templates@^1.0.19-alpha.1", "@budibase/string-templates@^1.0.22":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.22.tgz#b795c61e53d541c0aa346a90d04b50dcca6ae117"
integrity sha512-1ZhxzL75kVhP44fJlCWwqmGIPjZol1eB/xi3O11xJPYQ7lfzeJcGUpksvlgbLgBlw+MKkgppK7gEoMP247E0Qw==
"@budibase/string-templates@^1.0.23", "@budibase/string-templates@^1.0.24":
version "1.0.24"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.24.tgz#8f317ff18159d8f8e36179de33252e019c85c267"
integrity sha512-KAwyYxtm3Mrsm44MhchSxOtB8ucZ2JI93VPBQn0hjdzHtMuAkadtFNPZFeD3pKpCjDmI13XuW+BmJqCF3B0tYg==
dependencies:
"@budibase/handlebars-helpers" "^0.11.7"
dayjs "^1.10.4"
@ -11927,6 +11927,13 @@ svg.select.js@^3.0.1:
dependencies:
svg.js "^2.6.5"
swagger-parser@^10.0.3:
version "10.0.3"
resolved "https://registry.yarnpkg.com/swagger-parser/-/swagger-parser-10.0.3.tgz#04cb01c18c3ac192b41161c77f81e79309135d03"
integrity sha512-nF7oMeL4KypldrQhac8RyHerJeGPD1p2xDh900GPvc+Nk7nWP6jX2FcC7WmkinMoAmoO774+AFXcWsW8gMWEIg==
dependencies:
"@apidevtools/swagger-parser" "10.0.3"
symbol-tree@^3.2.2, symbol-tree@^3.2.4:
version "3.2.4"
resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2"