2021-12-17 11:43:14 +13:00
|
|
|
const threadUtils = require("./utils")
|
|
|
|
threadUtils.threadSetup()
|
2021-11-12 01:11:09 +13:00
|
|
|
const ScriptRunner = require("../utilities/scriptRunner")
|
|
|
|
const { integrations } = require("../integrations")
|
2022-04-15 00:06:22 +12:00
|
|
|
const { processStringSync } = require("@budibase/string-templates")
|
2022-01-28 13:05:39 +13:00
|
|
|
const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
|
2022-03-03 06:40:50 +13:00
|
|
|
const { isSQL } = require("../integrations/utils")
|
2022-04-15 00:06:22 +12:00
|
|
|
const {
|
|
|
|
enrichQueryFields,
|
|
|
|
interpolateSQL,
|
|
|
|
} = require("../integrations/queries/sql")
|
2022-04-05 06:08:56 +12:00
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
class QueryRunner {
|
|
|
|
constructor(input, flags = { noRecursiveQuery: false }) {
|
|
|
|
this.datasource = input.datasource
|
|
|
|
this.queryVerb = input.queryVerb
|
|
|
|
this.fields = input.fields
|
|
|
|
this.parameters = input.parameters
|
2022-01-05 22:16:10 +13:00
|
|
|
this.pagination = input.pagination
|
2021-12-17 11:43:14 +13:00
|
|
|
this.transformer = input.transformer
|
2021-12-18 03:08:48 +13:00
|
|
|
this.queryId = input.queryId
|
2021-12-17 11:43:14 +13:00
|
|
|
this.noRecursiveQuery = flags.noRecursiveQuery
|
2021-12-18 06:56:28 +13:00
|
|
|
this.cachedVariables = []
|
2022-06-14 21:14:05 +12:00
|
|
|
// Additional context items for enrichment
|
|
|
|
this.ctx = input.ctx
|
2021-12-18 07:21:36 +13:00
|
|
|
// allows the response from a query to be stored throughout this
|
|
|
|
// execution so that if it needs to be re-used for another variable
|
|
|
|
// it can be
|
|
|
|
this.queryResponse = {}
|
2021-12-18 06:56:28 +13:00
|
|
|
this.hasRerun = false
|
2021-12-17 05:58:15 +13:00
|
|
|
}
|
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
async execute() {
|
|
|
|
let { datasource, fields, queryVerb, transformer } = this
|
2022-03-03 11:45:10 +13:00
|
|
|
const Integration = integrations[datasource.source]
|
|
|
|
if (!Integration) {
|
|
|
|
throw "Integration type does not exist."
|
|
|
|
}
|
2022-07-02 04:27:24 +12:00
|
|
|
|
|
|
|
datasource.config.authConfigs = enrichQueryFields(
|
|
|
|
datasource.config.authConfigs,
|
|
|
|
this.ctx
|
|
|
|
)
|
|
|
|
|
2022-03-03 11:45:10 +13:00
|
|
|
const integration = new Integration(datasource.config)
|
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
// pre-query, make sure datasource variables are added to parameters
|
|
|
|
const parameters = await this.addDatasourceVariables()
|
2022-06-14 21:14:05 +12:00
|
|
|
|
|
|
|
// Enrich the parameters with the addition context items.
|
|
|
|
// 'user' is now a reserved variable key in mapping parameters
|
|
|
|
const enrichedParameters = enrichQueryFields(parameters, this.ctx)
|
|
|
|
const enrichedContext = { ...enrichedParameters, ...this.ctx }
|
|
|
|
|
|
|
|
// Parse global headers
|
2022-06-15 21:09:47 +12:00
|
|
|
if (datasource.config.defaultHeaders) {
|
|
|
|
datasource.config.defaultHeaders = enrichQueryFields(
|
|
|
|
datasource.config.defaultHeaders,
|
|
|
|
enrichedContext
|
|
|
|
)
|
|
|
|
}
|
2022-06-14 21:14:05 +12:00
|
|
|
|
2022-03-03 06:40:50 +13:00
|
|
|
let query
|
|
|
|
// handle SQL injections by interpolating the variables
|
|
|
|
if (isSQL(datasource)) {
|
2022-06-14 21:14:05 +12:00
|
|
|
query = interpolateSQL(fields, enrichedParameters, integration)
|
2022-03-03 06:40:50 +13:00
|
|
|
} else {
|
2022-06-14 21:14:05 +12:00
|
|
|
query = enrichQueryFields(fields, enrichedContext)
|
2022-03-03 06:40:50 +13:00
|
|
|
}
|
2022-01-06 03:26:13 +13:00
|
|
|
|
|
|
|
// Add pagination values for REST queries
|
|
|
|
if (this.pagination) {
|
|
|
|
query.paginationValues = this.pagination
|
|
|
|
}
|
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
let output = threadUtils.formatResponse(await integration[queryVerb](query))
|
|
|
|
let rows = output,
|
|
|
|
info = undefined,
|
2022-01-06 06:28:57 +13:00
|
|
|
extra = undefined,
|
|
|
|
pagination = undefined
|
2021-12-17 11:43:14 +13:00
|
|
|
if (threadUtils.hasExtraData(output)) {
|
|
|
|
rows = output.data
|
|
|
|
info = output.info
|
|
|
|
extra = output.extra
|
2022-01-06 06:28:57 +13:00
|
|
|
pagination = output.pagination
|
2021-12-17 05:58:15 +13:00
|
|
|
}
|
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
// transform as required
|
|
|
|
if (transformer) {
|
2022-01-12 00:17:35 +13:00
|
|
|
const runner = new ScriptRunner(transformer, {
|
|
|
|
data: rows,
|
2022-06-14 21:14:05 +12:00
|
|
|
params: enrichedParameters,
|
2022-01-12 00:17:35 +13:00
|
|
|
})
|
2021-12-17 11:43:14 +13:00
|
|
|
rows = runner.execute()
|
2021-12-17 05:58:15 +13:00
|
|
|
}
|
|
|
|
|
2021-12-18 06:56:28 +13:00
|
|
|
// if the request fails we retry once, invalidating the cached value
|
|
|
|
if (
|
|
|
|
info &&
|
|
|
|
info.code >= 400 &&
|
|
|
|
this.cachedVariables.length > 0 &&
|
|
|
|
!this.hasRerun
|
|
|
|
) {
|
|
|
|
this.hasRerun = true
|
|
|
|
// invalidate the cache value
|
|
|
|
await threadUtils.invalidateDynamicVariables(this.cachedVariables)
|
|
|
|
return this.execute()
|
|
|
|
}
|
|
|
|
|
2022-01-21 05:01:09 +13:00
|
|
|
// check for undefined response
|
|
|
|
if (!rows) {
|
|
|
|
rows = []
|
|
|
|
}
|
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
// needs to an array for next step
|
|
|
|
if (!Array.isArray(rows)) {
|
|
|
|
rows = [rows]
|
2021-11-11 08:35:09 +13:00
|
|
|
}
|
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
// map into JSON if just raw primitive here
|
|
|
|
if (rows.find(row => typeof row !== "object")) {
|
|
|
|
rows = rows.map(value => ({ value }))
|
|
|
|
}
|
2021-12-07 07:23:18 +13:00
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
// get all the potential fields in the schema
|
|
|
|
let keys = rows.flatMap(Object.keys)
|
2021-11-11 08:35:09 +13:00
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
if (integration.end) {
|
|
|
|
integration.end()
|
|
|
|
}
|
2021-11-11 08:35:09 +13:00
|
|
|
|
2022-01-06 06:28:57 +13:00
|
|
|
return { rows, keys, info, extra, pagination }
|
2021-11-11 08:35:09 +13:00
|
|
|
}
|
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
async runAnotherQuery(queryId, parameters) {
|
2022-01-28 13:05:39 +13:00
|
|
|
const db = getAppDB()
|
2021-12-17 11:43:14 +13:00
|
|
|
const query = await db.get(queryId)
|
|
|
|
const datasource = await db.get(query.datasourceId)
|
|
|
|
return new QueryRunner(
|
|
|
|
{
|
|
|
|
datasource,
|
|
|
|
queryVerb: query.queryVerb,
|
|
|
|
fields: query.fields,
|
|
|
|
parameters,
|
|
|
|
transformer: query.transformer,
|
|
|
|
},
|
|
|
|
{ noRecursiveQuery: true }
|
|
|
|
).execute()
|
2021-11-11 08:35:09 +13:00
|
|
|
}
|
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
async getDynamicVariable(variable) {
|
|
|
|
let { parameters } = this
|
|
|
|
const queryId = variable.queryId,
|
|
|
|
name = variable.name
|
|
|
|
let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
|
|
|
|
if (!value) {
|
2021-12-18 07:21:36 +13:00
|
|
|
value = this.queryResponse[queryId]
|
|
|
|
? this.queryResponse[queryId]
|
|
|
|
: await this.runAnotherQuery(queryId, parameters)
|
|
|
|
// store incase this query is to be called again
|
|
|
|
this.queryResponse[queryId] = value
|
2021-12-17 11:43:14 +13:00
|
|
|
await threadUtils.storeDynamicVariable(queryId, name, value)
|
2021-12-18 06:56:28 +13:00
|
|
|
} else {
|
|
|
|
this.cachedVariables.push({ queryId, name })
|
2021-12-17 11:43:14 +13:00
|
|
|
}
|
|
|
|
return value
|
2021-11-11 08:35:09 +13:00
|
|
|
}
|
|
|
|
|
2021-12-17 11:43:14 +13:00
|
|
|
async addDatasourceVariables() {
|
|
|
|
let { datasource, parameters, fields } = this
|
|
|
|
if (!datasource || !datasource.config) {
|
|
|
|
return parameters
|
|
|
|
}
|
|
|
|
const staticVars = datasource.config.staticVariables || {}
|
|
|
|
const dynamicVars = datasource.config.dynamicVariables || []
|
|
|
|
for (let [key, value] of Object.entries(staticVars)) {
|
|
|
|
if (!parameters[key]) {
|
|
|
|
parameters[key] = value
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!this.noRecursiveQuery) {
|
|
|
|
// need to see if this uses any variables
|
|
|
|
const stringFields = JSON.stringify(fields)
|
|
|
|
const foundVars = dynamicVars.filter(variable => {
|
2021-12-18 03:08:48 +13:00
|
|
|
// don't allow a query to use its own dynamic variable (loop)
|
|
|
|
if (variable.queryId === this.queryId) {
|
|
|
|
return false
|
|
|
|
}
|
2021-12-17 11:43:14 +13:00
|
|
|
// look for {{ variable }} but allow spaces between handlebars
|
|
|
|
const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
|
|
|
|
return regex.test(stringFields)
|
|
|
|
})
|
|
|
|
const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
|
|
|
|
const responses = await Promise.all(dynamics)
|
|
|
|
for (let i = 0; i < foundVars.length; i++) {
|
|
|
|
const variable = foundVars[i]
|
2022-02-08 06:56:01 +13:00
|
|
|
parameters[variable.name] = processStringSync(
|
|
|
|
variable.value,
|
|
|
|
{
|
|
|
|
data: responses[i].rows,
|
|
|
|
info: responses[i].extra,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
escapeNewlines: true,
|
|
|
|
}
|
|
|
|
)
|
2021-12-18 06:56:28 +13:00
|
|
|
// make sure its known that this uses dynamic variables in case it fails
|
|
|
|
this.hasDynamicVariables = true
|
2021-12-17 11:43:14 +13:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return parameters
|
2021-11-11 08:35:09 +13:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
module.exports = (input, callback) => {
|
2022-05-04 03:04:59 +12:00
|
|
|
doInAppContext(input.appId, async () => {
|
2022-01-28 13:05:39 +13:00
|
|
|
const Runner = new QueryRunner(input)
|
2022-05-04 03:04:59 +12:00
|
|
|
try {
|
|
|
|
const response = await Runner.execute()
|
|
|
|
callback(null, response)
|
|
|
|
} catch (err) {
|
|
|
|
callback(err)
|
|
|
|
}
|
2022-01-28 13:05:39 +13:00
|
|
|
})
|
2021-11-11 08:35:09 +13:00
|
|
|
}
|