diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 312ecf313d..212a54b87c 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -70,6 +70,7 @@ module.exports = { ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, QUERY_THREAD_TIMEOUT: process.env.QUERY_THREAD_TIMEOUT, + SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index ffa405f016..3c3132c887 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -9,8 +9,12 @@ import { } from "../../definitions/datasource" import { isIsoDateString, SqlClients } from "../utils" import SqlTableQueryBuilder from "./sqlTable" +import environment from "../../environment" -const BASE_LIMIT = 5000 +const envLimit = environment.SQL_MAX_ROWS + ? parseInt(environment.SQL_MAX_ROWS) + : null +const BASE_LIMIT = envLimit || 5000 type KnexQuery = Knex.QueryBuilder | Knex // these are invalid dates sent by the client, need to convert them to a real max date diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js index 3ce138b72f..270c2cc713 100644 --- a/packages/server/src/threads/query.js +++ b/packages/server/src/threads/query.js @@ -33,10 +33,12 @@ class QueryRunner { return fields } const bindings = findHBSBlocks(sql) - let variables = [] + let variables = [], + arrays = [] for (let binding of bindings) { - let variable = integration.getBindingIdentifier() - variables.push(binding) + // look for array/list operations in the SQL statement, which will need handled later + const listRegex = new RegExp(`(in|IN|In|iN)( )+${binding}`) + const listRegexMatch = sql.match(listRegex) // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}' const charConstRegex = new RegExp(`'[^']*${binding}[^']*'`) const charConstMatch = sql.match(charConstRegex) @@ -46,15 +48,45 @@ class QueryRunner { part2 = `'${part2.substring(0, part2.length - 1)}'` sql = sql.replace( charConstMatch[0], - integration.getStringConcat([part1, variable, part2]) + integration.getStringConcat([ + part1, + integration.getBindingIdentifier(), + part2, + ]) + ) + } + // generate SQL parameterised array + else if (listRegexMatch) { + arrays.push(binding) + // determine the length of the array + const value = this.enrichQueryFields([binding], parameters)[0].split( + "," + ) + // build a string like ($1, $2, $3) + sql = sql.replace( + binding, + `(${Array.apply(null, Array(value.length)) + .map(() => integration.getBindingIdentifier()) + .join(",")})` ) } else { - sql = sql.replace(binding, variable) + sql = sql.replace(binding, integration.getBindingIdentifier()) } + variables.push(binding) } // replicate the knex structure fields.sql = sql fields.bindings = this.enrichQueryFields(variables, parameters) + // check for arrays in the data + let updated = [] + for (let i = 0; i < variables.length; i++) { + if (arrays.includes(variables[i])) { + updated = updated.concat(fields.bindings[i].split(",")) + } else { + updated.push(fields.bindings[i]) + } + } + fields.bindings = updated return fields }