Changing how SQL vars are generated so that when new SQL implementations are added they must implement a generation mechanism.
This commit is contained in:
parent
1b45a9190d
commit
546bbc2ff6
8 changed files with 149 additions and 1088 deletions
|
@ -5,5 +5,8 @@ export interface DatasourcePlus extends IntegrationBase {
|
|||
tables: Record<string, Table>
|
||||
schemaErrors: Record<string, string>
|
||||
|
||||
// if the datasource supports the use of bindings directly (to protect against SQL injection)
|
||||
// this returns the format of the identifier
|
||||
getBindingIdentifier(): string
|
||||
buildSchema(datasourceId: string, entities: Record<string, Table>): any
|
||||
}
|
||||
|
|
|
@ -6,11 +6,10 @@ import {
|
|||
} from "../definitions/datasource"
|
||||
import { OAuth2Client } from "google-auth-library"
|
||||
import { DatasourcePlus } from "./base/datasourcePlus"
|
||||
import { Row, Table, TableSchema } from "../definitions/common"
|
||||
import { Table, TableSchema } from "../definitions/common"
|
||||
import { buildExternalTableId } from "./utils"
|
||||
import { DataSourceOperation, FieldTypes } from "../constants"
|
||||
import { GoogleSpreadsheet } from "google-spreadsheet"
|
||||
import { table } from "console"
|
||||
|
||||
module GoogleSheetsModule {
|
||||
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
|
||||
|
@ -112,6 +111,10 @@ module GoogleSheetsModule {
|
|||
this.client = new GoogleSpreadsheet(spreadsheetId)
|
||||
}
|
||||
|
||||
getBindingIdentifier() {
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the spreadsheet ID out from a valid google sheets URL
|
||||
* @param spreadsheetId - the URL or standard spreadsheetId of the google sheet
|
||||
|
|
|
@ -79,34 +79,9 @@ module MSSQLModule {
|
|||
},
|
||||
}
|
||||
|
||||
async function internalQuery(
|
||||
client: any,
|
||||
query: SqlQuery,
|
||||
operation: string | undefined = undefined
|
||||
) {
|
||||
const request = client.request()
|
||||
try {
|
||||
if (Array.isArray(query.bindings)) {
|
||||
let count = 0
|
||||
for (let binding of query.bindings) {
|
||||
request.input(`p${count++}`, binding)
|
||||
}
|
||||
}
|
||||
// this is a hack to get the inserted ID back,
|
||||
// no way to do this with Knex nicely
|
||||
const sql =
|
||||
operation === Operation.CREATE
|
||||
? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
|
||||
: query.sql
|
||||
return await request.query(sql)
|
||||
} catch (err) {
|
||||
// @ts-ignore
|
||||
throw new Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
class SqlServerIntegration extends Sql implements DatasourcePlus {
|
||||
private readonly config: MSSQLConfig
|
||||
private index: number = 0
|
||||
static pool: any
|
||||
public tables: Record<string, Table> = {}
|
||||
public schemaErrors: Record<string, string> = {}
|
||||
|
@ -121,6 +96,33 @@ module MSSQLModule {
|
|||
TABLES_SQL =
|
||||
"SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'"
|
||||
|
||||
async internalQuery(
|
||||
query: SqlQuery,
|
||||
operation: string | undefined = undefined
|
||||
) {
|
||||
const client = this.client
|
||||
const request = client.request()
|
||||
this.index = 0
|
||||
try {
|
||||
if (Array.isArray(query.bindings)) {
|
||||
let count = 0
|
||||
for (let binding of query.bindings) {
|
||||
request.input(`p${count++}`, binding)
|
||||
}
|
||||
}
|
||||
// this is a hack to get the inserted ID back,
|
||||
// no way to do this with Knex nicely
|
||||
const sql =
|
||||
operation === Operation.CREATE
|
||||
? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
|
||||
: query.sql
|
||||
return await request.query(sql)
|
||||
} catch (err) {
|
||||
// @ts-ignore
|
||||
throw new Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
getDefinitionSQL(tableName: string) {
|
||||
return `select *
|
||||
from INFORMATION_SCHEMA.COLUMNS
|
||||
|
@ -165,6 +167,10 @@ module MSSQLModule {
|
|||
}
|
||||
}
|
||||
|
||||
getBindingIdentifier(): string {
|
||||
return `(@p${this.index++})`
|
||||
}
|
||||
|
||||
async connect() {
|
||||
try {
|
||||
this.client = await this.pool.connect()
|
||||
|
@ -175,7 +181,7 @@ module MSSQLModule {
|
|||
}
|
||||
|
||||
async runSQL(sql: string) {
|
||||
return (await internalQuery(this.client, getSqlQuery(sql))).recordset
|
||||
return (await this.internalQuery(getSqlQuery(sql))).recordset
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -238,33 +244,32 @@ module MSSQLModule {
|
|||
|
||||
async read(query: SqlQuery | string) {
|
||||
await this.connect()
|
||||
const response = await internalQuery(this.client, getSqlQuery(query))
|
||||
const response = await this.internalQuery(getSqlQuery(query))
|
||||
return response.recordset
|
||||
}
|
||||
|
||||
async create(query: SqlQuery | string) {
|
||||
await this.connect()
|
||||
const response = await internalQuery(this.client, getSqlQuery(query))
|
||||
const response = await this.internalQuery(getSqlQuery(query))
|
||||
return response.recordset || [{ created: true }]
|
||||
}
|
||||
|
||||
async update(query: SqlQuery | string) {
|
||||
await this.connect()
|
||||
const response = await internalQuery(this.client, getSqlQuery(query))
|
||||
const response = await this.internalQuery(getSqlQuery(query))
|
||||
return response.recordset || [{ updated: true }]
|
||||
}
|
||||
|
||||
async delete(query: SqlQuery | string) {
|
||||
await this.connect()
|
||||
const response = await internalQuery(this.client, getSqlQuery(query))
|
||||
const response = await this.internalQuery(getSqlQuery(query))
|
||||
return response.recordset || [{ deleted: true }]
|
||||
}
|
||||
|
||||
async query(json: QueryJson) {
|
||||
await this.connect()
|
||||
const operation = this._operation(json)
|
||||
const queryFn = (query: any, op: string) =>
|
||||
internalQuery(this.client, query, op)
|
||||
const queryFn = (query: any, op: string) => this.internalQuery(query, op)
|
||||
const processFn = (result: any) =>
|
||||
result.recordset ? result.recordset : [{ [operation]: true }]
|
||||
return this.queryWithReturning(json, queryFn, processFn)
|
||||
|
|
|
@ -80,33 +80,6 @@ module MySQLModule {
|
|||
},
|
||||
}
|
||||
|
||||
function internalQuery(
|
||||
client: any,
|
||||
query: SqlQuery,
|
||||
connect: boolean = true
|
||||
): Promise<any[] | any> {
|
||||
// Node MySQL is callback based, so we must wrap our call in a promise
|
||||
return new Promise((resolve, reject) => {
|
||||
if (connect) {
|
||||
client.connect()
|
||||
}
|
||||
return client.query(
|
||||
query.sql,
|
||||
query.bindings || {},
|
||||
(error: any, results: object[]) => {
|
||||
if (error) {
|
||||
reject(error)
|
||||
} else {
|
||||
resolve(results)
|
||||
}
|
||||
if (connect) {
|
||||
client.end()
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
class MySQLIntegration extends Sql implements DatasourcePlus {
|
||||
private config: MySQLConfig
|
||||
private readonly client: any
|
||||
|
@ -122,14 +95,44 @@ module MySQLModule {
|
|||
this.client = mysql.createConnection(config)
|
||||
}
|
||||
|
||||
getBindingIdentifier(): string {
|
||||
return "?"
|
||||
}
|
||||
|
||||
internalQuery(
|
||||
query: SqlQuery,
|
||||
connect: boolean = true
|
||||
): Promise<any[] | any> {
|
||||
const client = this.client
|
||||
// Node MySQL is callback based, so we must wrap our call in a promise
|
||||
return new Promise((resolve, reject) => {
|
||||
if (connect) {
|
||||
client.connect()
|
||||
}
|
||||
return client.query(
|
||||
query.sql,
|
||||
query.bindings || {},
|
||||
(error: any, results: object[]) => {
|
||||
if (error) {
|
||||
reject(error)
|
||||
} else {
|
||||
resolve(results)
|
||||
}
|
||||
if (connect) {
|
||||
client.end()
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async buildSchema(datasourceId: string, entities: Record<string, Table>) {
|
||||
const tables: { [key: string]: Table } = {}
|
||||
const database = this.config.database
|
||||
this.client.connect()
|
||||
|
||||
// get the tables first
|
||||
const tablesResp = await internalQuery(
|
||||
this.client,
|
||||
const tablesResp = await this.internalQuery(
|
||||
{ sql: "SHOW TABLES;" },
|
||||
false
|
||||
)
|
||||
|
@ -141,8 +144,7 @@ module MySQLModule {
|
|||
for (let tableName of tableNames) {
|
||||
const primaryKeys = []
|
||||
const schema: TableSchema = {}
|
||||
const descResp = await internalQuery(
|
||||
this.client,
|
||||
const descResp = await this.internalQuery(
|
||||
{ sql: `DESCRIBE \`${tableName}\`;` },
|
||||
false
|
||||
)
|
||||
|
@ -182,27 +184,27 @@ module MySQLModule {
|
|||
}
|
||||
|
||||
async create(query: SqlQuery | string) {
|
||||
const results = await internalQuery(this.client, getSqlQuery(query))
|
||||
const results = await this.internalQuery(getSqlQuery(query))
|
||||
return results.length ? results : [{ created: true }]
|
||||
}
|
||||
|
||||
async read(query: SqlQuery | string) {
|
||||
return internalQuery(this.client, getSqlQuery(query))
|
||||
return this.internalQuery(getSqlQuery(query))
|
||||
}
|
||||
|
||||
async update(query: SqlQuery | string) {
|
||||
const results = await internalQuery(this.client, getSqlQuery(query))
|
||||
const results = await this.internalQuery(getSqlQuery(query))
|
||||
return results.length ? results : [{ updated: true }]
|
||||
}
|
||||
|
||||
async delete(query: SqlQuery | string) {
|
||||
const results = await internalQuery(this.client, getSqlQuery(query))
|
||||
const results = await this.internalQuery(getSqlQuery(query))
|
||||
return results.length ? results : [{ deleted: true }]
|
||||
}
|
||||
|
||||
async query(json: QueryJson) {
|
||||
this.client.connect()
|
||||
const queryFn = (query: any) => internalQuery(this.client, query, false)
|
||||
const queryFn = (query: any) => this.internalQuery(query, false)
|
||||
const output = await this.queryWithReturning(json, queryFn)
|
||||
this.client.end()
|
||||
return output
|
||||
|
|
|
@ -137,6 +137,7 @@ module OracleModule {
|
|||
|
||||
class OracleIntegration extends Sql implements DatasourcePlus {
|
||||
private readonly config: OracleConfig
|
||||
private index: number = 1
|
||||
|
||||
public tables: Record<string, Table> = {}
|
||||
public schemaErrors: Record<string, string> = {}
|
||||
|
@ -174,6 +175,10 @@ module OracleModule {
|
|||
this.config = config
|
||||
}
|
||||
|
||||
getBindingIdentifier(): string {
|
||||
return `:${this.index++}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Map the flat tabular columns and constraints data into a nested object
|
||||
*/
|
||||
|
@ -343,6 +348,7 @@ module OracleModule {
|
|||
private async internalQuery<T>(query: SqlQuery): Promise<Result<T>> {
|
||||
let connection
|
||||
try {
|
||||
this.index = 1
|
||||
connection = await this.getConnection()
|
||||
|
||||
const options: ExecuteOptions = { autoCommit: true }
|
||||
|
|
|
@ -103,30 +103,11 @@ module PostgresModule {
|
|||
},
|
||||
}
|
||||
|
||||
async function internalQuery(client: any, query: SqlQuery) {
|
||||
// need to handle a specific issue with json data types in postgres,
|
||||
// new lines inside the JSON data will break it
|
||||
if (query && query.sql) {
|
||||
const matches = query.sql.match(JSON_REGEX)
|
||||
if (matches && matches.length > 0) {
|
||||
for (let match of matches) {
|
||||
const escaped = escapeDangerousCharacters(match)
|
||||
query.sql = query.sql.replace(match, escaped)
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
return await client.query(query.sql, query.bindings || [])
|
||||
} catch (err) {
|
||||
// @ts-ignore
|
||||
throw new Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
class PostgresIntegration extends Sql implements DatasourcePlus {
|
||||
static pool: any
|
||||
private readonly client: any
|
||||
private readonly config: PostgresConfig
|
||||
private index: number = 1
|
||||
public tables: Record<string, Table> = {}
|
||||
public schemaErrors: Record<string, string> = {}
|
||||
|
||||
|
@ -163,6 +144,32 @@ module PostgresModule {
|
|||
this.setSchema()
|
||||
}
|
||||
|
||||
getBindingIdentifier(): string {
|
||||
return `$${this.index++}`
|
||||
}
|
||||
|
||||
async internalQuery(query: SqlQuery) {
|
||||
const client = this.client
|
||||
this.index = 1
|
||||
// need to handle a specific issue with json data types in postgres,
|
||||
// new lines inside the JSON data will break it
|
||||
if (query && query.sql) {
|
||||
const matches = query.sql.match(JSON_REGEX)
|
||||
if (matches && matches.length > 0) {
|
||||
for (let match of matches) {
|
||||
const escaped = escapeDangerousCharacters(match)
|
||||
query.sql = query.sql.replace(match, escaped)
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
return await client.query(query.sql, query.bindings || [])
|
||||
} catch (err) {
|
||||
// @ts-ignore
|
||||
throw new Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
setSchema() {
|
||||
if (!this.config.schema) {
|
||||
this.config.schema = "public"
|
||||
|
@ -241,22 +248,22 @@ module PostgresModule {
|
|||
}
|
||||
|
||||
async create(query: SqlQuery | string) {
|
||||
const response = await internalQuery(this.client, getSqlQuery(query))
|
||||
const response = await this.internalQuery(getSqlQuery(query))
|
||||
return response.rows.length ? response.rows : [{ created: true }]
|
||||
}
|
||||
|
||||
async read(query: SqlQuery | string) {
|
||||
const response = await internalQuery(this.client, getSqlQuery(query))
|
||||
const response = await this.internalQuery(getSqlQuery(query))
|
||||
return response.rows
|
||||
}
|
||||
|
||||
async update(query: SqlQuery | string) {
|
||||
const response = await internalQuery(this.client, getSqlQuery(query))
|
||||
const response = await this.internalQuery(getSqlQuery(query))
|
||||
return response.rows.length ? response.rows : [{ updated: true }]
|
||||
}
|
||||
|
||||
async delete(query: SqlQuery | string) {
|
||||
const response = await internalQuery(this.client, getSqlQuery(query))
|
||||
const response = await this.internalQuery(getSqlQuery(query))
|
||||
return response.rows.length ? response.rows : [{ deleted: true }]
|
||||
}
|
||||
|
||||
|
@ -266,11 +273,11 @@ module PostgresModule {
|
|||
if (Array.isArray(input)) {
|
||||
const responses = []
|
||||
for (let query of input) {
|
||||
responses.push(await internalQuery(this.client, query))
|
||||
responses.push(await this.internalQuery(query))
|
||||
}
|
||||
return responses
|
||||
} else {
|
||||
const response = await internalQuery(this.client, input)
|
||||
const response = await this.internalQuery(input)
|
||||
return response.rows.length ? response.rows : [{ [operation]: true }]
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,7 +2,6 @@ const threadUtils = require("./utils")
|
|||
threadUtils.threadSetup()
|
||||
const ScriptRunner = require("../utilities/scriptRunner")
|
||||
const { integrations } = require("../integrations")
|
||||
const { SourceNames } = require("../definitions/datasource")
|
||||
const {
|
||||
processStringSync,
|
||||
findHBSBlocks,
|
||||
|
@ -28,29 +27,12 @@ class QueryRunner {
|
|||
this.hasRerun = false
|
||||
}
|
||||
|
||||
interpolateSQL(fields, parameters) {
|
||||
let { datasource } = this
|
||||
interpolateSQL(fields, parameters, integration) {
|
||||
let sql = fields.sql
|
||||
const bindings = findHBSBlocks(sql)
|
||||
let index = 1
|
||||
let variables = []
|
||||
for (let binding of bindings) {
|
||||
let variable
|
||||
switch (datasource.source) {
|
||||
case SourceNames.POSTGRES:
|
||||
variable = `$${index}`
|
||||
break
|
||||
case SourceNames.SQL_SERVER:
|
||||
variable = `(@p${index - 1})`
|
||||
break
|
||||
case SourceNames.MYSQL:
|
||||
variable = "?"
|
||||
break
|
||||
case SourceNames.ORACLE:
|
||||
variable = `:${index}`
|
||||
break
|
||||
}
|
||||
index++
|
||||
let variable = integration.getBindingIdentifier()
|
||||
variables.push(binding)
|
||||
sql = sql.replace(binding, variable)
|
||||
}
|
||||
|
@ -62,12 +44,18 @@ class QueryRunner {
|
|||
|
||||
async execute() {
|
||||
let { datasource, fields, queryVerb, transformer } = this
|
||||
const Integration = integrations[datasource.source]
|
||||
if (!Integration) {
|
||||
throw "Integration type does not exist."
|
||||
}
|
||||
const integration = new Integration(datasource.config)
|
||||
|
||||
// pre-query, make sure datasource variables are added to parameters
|
||||
const parameters = await this.addDatasourceVariables()
|
||||
let query
|
||||
// handle SQL injections by interpolating the variables
|
||||
if (isSQL(datasource)) {
|
||||
query = this.interpolateSQL(fields, parameters)
|
||||
query = this.interpolateSQL(fields, parameters, integration)
|
||||
} else {
|
||||
query = this.enrichQueryFields(fields, parameters)
|
||||
}
|
||||
|
@ -77,12 +65,6 @@ class QueryRunner {
|
|||
query.paginationValues = this.pagination
|
||||
}
|
||||
|
||||
const Integration = integrations[datasource.source]
|
||||
if (!Integration) {
|
||||
throw "Integration type does not exist."
|
||||
}
|
||||
const integration = new Integration(datasource.config)
|
||||
|
||||
let output = threadUtils.formatResponse(await integration[queryVerb](query))
|
||||
let rows = output,
|
||||
info = undefined,
|
||||
|
|
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue