1
0
Fork 0
mirror of synced 2024-06-12 07:24:43 +12:00

Merge branch 'master' into pull-images-prior-to-tests

This commit is contained in:
Sam Rose 2024-03-25 13:51:38 +00:00 committed by GitHub
commit 419880e4fb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 183 additions and 56 deletions

View file

@ -6,7 +6,7 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
import env from "../../../environment"
import { events, context, utils, constants } from "@budibase/backend-core"
import sdk from "../../../sdk"
import { QueryEvent } from "../../../threads/definitions"
import { QueryEvent, QueryEventParameters } from "../../../threads/definitions"
import {
ConfigType,
Query,
@ -18,7 +18,6 @@ import {
FieldType,
ExecuteQueryRequest,
ExecuteQueryResponse,
QueryParameter,
PreviewQueryRequest,
PreviewQueryResponse,
} from "@budibase/types"
@ -29,7 +28,7 @@ const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: env.QUERY_THREAD_TIMEOUT,
})
function validateQueryInputs(parameters: Record<string, string>) {
function validateQueryInputs(parameters: QueryEventParameters) {
for (let entry of Object.entries(parameters)) {
const [key, value] = entry
if (typeof value !== "string") {
@ -100,10 +99,18 @@ export async function save(ctx: UserCtx<Query, Query>) {
const datasource = await sdk.datasources.get(query.datasourceId)
let eventFn
if (!query._id) {
if (!query._id && !query._rev) {
query._id = generateQueryID(query.datasourceId)
// flag to state whether the default bindings are empty strings (old behaviour) or null
query.nullDefaultSupport = true
eventFn = () => events.query.created(datasource, query)
} else {
// check if flag has previously been set, don't let it change
// allow it to be explicitly set to false via API incase this is ever needed
const existingQuery = await db.get<Query>(query._id)
if (existingQuery.nullDefaultSupport && query.nullDefaultSupport == null) {
query.nullDefaultSupport = true
}
eventFn = () => events.query.updated(datasource, query)
}
const response = await db.put(query)
@ -135,16 +142,20 @@ function getAuthConfig(ctx: UserCtx) {
}
function enrichParameters(
queryParameters: QueryParameter[],
requestParameters: Record<string, string> = {}
): Record<string, string> {
query: Query,
requestParameters: QueryEventParameters = {}
): QueryEventParameters {
const paramNotSet = (val: unknown) => val === "" || val == undefined
// first check parameters are all valid
validateQueryInputs(requestParameters)
// make sure parameters are fully enriched with defaults
for (let parameter of queryParameters) {
if (!requestParameters[parameter.name]) {
requestParameters[parameter.name] = parameter.default
for (const parameter of query.parameters) {
let value: string | null =
requestParameters[parameter.name] || parameter.default
if (query.nullDefaultSupport && paramNotSet(value)) {
value = null
}
requestParameters[parameter.name] = value
}
return requestParameters
}
@ -157,10 +168,15 @@ export async function preview(
)
// preview may not have a queryId as it hasn't been saved, but if it does
// this stops dynamic variables from calling the same query
const { fields, parameters, queryVerb, transformer, queryId, schema } =
ctx.request.body
const queryId = ctx.request.body.queryId
// the body contains the makings of a query, which has not been saved yet
const query: Query = ctx.request.body
// hasn't been saved, new query
if (!queryId && !query._id) {
query.nullDefaultSupport = true
}
let existingSchema = schema
let existingSchema = query.schema
if (queryId && !existingSchema) {
try {
const db = context.getAppDB()
@ -268,13 +284,14 @@ export async function preview(
try {
const inputs: QueryEvent = {
appId: ctx.appId,
datasource,
queryVerb,
fields,
parameters: enrichParameters(parameters),
transformer,
queryVerb: query.queryVerb,
fields: query.fields,
parameters: enrichParameters(query),
transformer: query.transformer,
schema: query.schema,
nullDefaultSupport: query.nullDefaultSupport,
queryId,
schema,
datasource,
// have to pass down to the thread runner - can't put into context now
environmentVariables: envVars,
ctx: {
@ -336,14 +353,12 @@ async function execute(
queryVerb: query.queryVerb,
fields: query.fields,
pagination: ctx.request.body.pagination,
parameters: enrichParameters(
query.parameters,
ctx.request.body.parameters
),
parameters: enrichParameters(query, ctx.request.body.parameters),
transformer: query.transformer,
queryId: ctx.params.queryId,
// have to pass down to the thread runner - can't put into context now
environmentVariables: envVars,
nullDefaultSupport: query.nullDefaultSupport,
ctx: {
user: ctx.user,
auth: { ...authConfigCtx },

View file

@ -143,7 +143,10 @@ describe("/api/env/variables", () => {
delete response.body.datasource.config
expect(events.query.previewed).toHaveBeenCalledWith(
response.body.datasource,
queryPreview
{
...queryPreview,
nullDefaultSupport: true,
}
)
expect(pg.Client).toHaveBeenCalledWith({ password: "test", ssl: undefined })
})

View file

@ -12,19 +12,22 @@ const createTableSQL: Record<string, string> = {
CREATE TABLE test_table (
id serial PRIMARY KEY,
name VARCHAR ( 50 ) NOT NULL,
birthday TIMESTAMP
birthday TIMESTAMP,
number INT
);`,
[SourceName.MYSQL]: `
CREATE TABLE test_table (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(50) NOT NULL,
birthday TIMESTAMP
birthday TIMESTAMP,
number INT
);`,
[SourceName.SQL_SERVER]: `
CREATE TABLE test_table (
id INT IDENTITY(1,1) PRIMARY KEY,
name NVARCHAR(50) NOT NULL,
birthday DATETIME
birthday DATETIME,
number INT
);`,
}
@ -36,7 +39,7 @@ describe.each([
["mysql", databaseTestProviders.mysql],
["mssql", databaseTestProviders.mssql],
["mariadb", databaseTestProviders.mariadb],
])("queries (%s)", (__, dsProvider) => {
])("queries (%s)", (dbName, dsProvider) => {
const config = setup.getConfig()
let datasource: Datasource
@ -51,7 +54,7 @@ describe.each([
transformer: "return data",
readable: true,
}
return await config.api.query.create({ ...defaultQuery, ...query })
return await config.api.query.save({ ...defaultQuery, ...query })
}
async function rawQuery(sql: string): Promise<any> {
@ -221,26 +224,31 @@ describe.each([
id: 1,
name: "one",
birthday: null,
number: null,
},
{
id: 2,
name: "two",
birthday: null,
number: null,
},
{
id: 3,
name: "three",
birthday: null,
number: null,
},
{
id: 4,
name: "four",
birthday: null,
number: null,
},
{
id: 5,
name: "five",
birthday: null,
number: null,
},
])
})
@ -263,6 +271,7 @@ describe.each([
id: 2,
name: "one",
birthday: null,
number: null,
},
])
})
@ -291,6 +300,7 @@ describe.each([
id: 1,
name: "one",
birthday: null,
number: null,
},
])
})
@ -329,7 +339,9 @@ describe.each([
])
const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
expect(rows).toEqual([{ id: 1, name: "foo", birthday: null }])
expect(rows).toEqual([
{ id: 1, name: "foo", birthday: null, number: null },
])
})
it("should be able to execute an update that updates no rows", async () => {
@ -398,4 +410,55 @@ describe.each([
expect(rows).toHaveLength(0)
})
})
// this parameter really only impacts SQL queries
describe("confirm nullDefaultSupport", () => {
const queryParams = {
fields: {
sql: "INSERT INTO test_table (name, number) VALUES ({{ bindingName }}, {{ bindingNumber }})",
},
parameters: [
{
name: "bindingName",
default: "",
},
{
name: "bindingNumber",
default: "",
},
],
queryVerb: "create",
}
it("should error for old queries", async () => {
const query = await createQuery(queryParams)
await config.api.query.save({ ...query, nullDefaultSupport: false })
let error: string | undefined
try {
await config.api.query.execute(query._id!, {
parameters: {
bindingName: "testing",
},
})
} catch (err: any) {
error = err.message
}
if (dbName === "mssql") {
expect(error).toBeUndefined()
} else {
expect(error).toBeDefined()
expect(error).toContain("integer")
}
})
it("should not error for new queries", async () => {
const query = await createQuery(queryParams)
const results = await config.api.query.execute(query._id!, {
parameters: {
bindingName: "testing",
},
})
expect(results).toEqual({ data: [{ created: true }] })
})
})
})

View file

@ -31,7 +31,7 @@ describe("/queries", () => {
) {
combinedQuery.fields.extra.collection = collection
}
return await config.api.query.create(combinedQuery)
return await config.api.query.save(combinedQuery)
}
async function withClient<T>(
@ -464,7 +464,7 @@ describe("/queries", () => {
})
})
it("should ignore be able to save deeply nested data", async () => {
it("should be able to save deeply nested data", async () => {
const data = {
foo: "bar",
data: [

View file

@ -78,6 +78,7 @@ describe("/queries", () => {
_rev: res.body._rev,
_id: res.body._id,
...query,
nullDefaultSupport: true,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
@ -103,6 +104,7 @@ describe("/queries", () => {
_rev: res.body._rev,
_id: res.body._id,
...query,
nullDefaultSupport: true,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
@ -130,6 +132,7 @@ describe("/queries", () => {
_id: query._id,
createdAt: new Date().toISOString(),
...basicQuery(datasource._id),
nullDefaultSupport: true,
updatedAt: new Date().toISOString(),
readable: true,
},
@ -245,10 +248,10 @@ describe("/queries", () => {
expect(responseBody.rows.length).toEqual(1)
expect(events.query.previewed).toHaveBeenCalledTimes(1)
delete datasource.config
expect(events.query.previewed).toHaveBeenCalledWith(
datasource,
queryPreview
)
expect(events.query.previewed).toHaveBeenCalledWith(datasource, {
...queryPreview,
nullDefaultSupport: true,
})
})
it("should apply authorization to endpoint", async () => {

View file

@ -5,9 +5,10 @@ import sdk from "../../sdk"
const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
export async function interpolateSQL(
fields: { [key: string]: any },
fields: { sql: string; bindings: any[] },
parameters: { [key: string]: any },
integration: DatasourcePlus
integration: DatasourcePlus,
opts: { nullDefaultSupport: boolean }
) {
let sql = fields.sql
if (!sql || typeof sql !== "string") {
@ -64,7 +65,14 @@ export async function interpolateSQL(
}
// replicate the knex structure
fields.sql = sql
fields.bindings = await sdk.queries.enrichContext(variables, parameters)
fields.bindings = await sdk.queries.enrichArrayContext(variables, parameters)
if (opts.nullDefaultSupport) {
for (let index in fields.bindings) {
if (fields.bindings[index] === "") {
fields.bindings[index] = null
}
}
}
// check for arrays in the data
let updated: string[] = []
for (let i = 0; i < variables.length; i++) {

View file

@ -65,14 +65,33 @@ export async function fetch(opts: { enrich: boolean } = { enrich: true }) {
return updateSchemas(queries)
}
export async function enrichArrayContext(
fields: any[],
inputs = {}
): Promise<any[]> {
const map: Record<string, any> = {}
for (let index in fields) {
map[index] = fields[index]
}
const output = await enrichContext(map, inputs)
const outputArray: any[] = []
for (let [key, value] of Object.entries(output)) {
outputArray[parseInt(key)] = value
}
return outputArray
}
export async function enrichContext(
fields: Record<string, any>,
inputs = {}
): Promise<Record<string, any>> {
const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
const enrichedQuery: Record<string, any> = {}
if (!fields || !inputs) {
return enrichedQuery
}
if (Array.isArray(fields)) {
return enrichArrayContext(fields, inputs)
}
const env = await getEnvironmentVariables()
const parameters = { ...inputs, env }
// enrich the fields with dynamic parameters

View file

@ -8,7 +8,7 @@ import {
import { Expectations, TestAPI } from "./base"
export class QueryAPI extends TestAPI {
create = async (body: Query): Promise<Query> => {
save = async (body: Query): Promise<Query> => {
return await this._post<Query>(`/api/queries`, { body })
}

View file

@ -1,21 +1,20 @@
import { Datasource, QuerySchema, Row } from "@budibase/types"
import { Datasource, Row, Query } from "@budibase/types"
export type WorkerCallback = (error: any, response?: any) => void
export interface QueryEvent {
export interface QueryEvent
extends Omit<Query, "datasourceId" | "name" | "parameters" | "readable"> {
appId?: string
datasource: Datasource
queryVerb: string
fields: { [key: string]: any }
parameters: { [key: string]: unknown }
pagination?: any
transformer: any
queryId?: string
environmentVariables?: Record<string, string>
parameters: QueryEventParameters
ctx?: any
schema?: Record<string, QuerySchema | string>
}
export type QueryEventParameters = Record<string, string | null>
export interface QueryResponse {
rows: Row[]
keys: string[]

View file

@ -26,10 +26,11 @@ class QueryRunner {
fields: any
parameters: any
pagination: any
transformer: string
transformer: string | null
cachedVariables: any[]
ctx: any
queryResponse: any
nullDefaultSupport: boolean
noRecursiveQuery: boolean
hasRerun: boolean
hasRefreshedOAuth: boolean
@ -45,6 +46,7 @@ class QueryRunner {
this.transformer = input.transformer
this.queryId = input.queryId!
this.schema = input.schema
this.nullDefaultSupport = !!input.nullDefaultSupport
this.noRecursiveQuery = flags.noRecursiveQuery
this.cachedVariables = []
// Additional context items for enrichment
@ -59,7 +61,14 @@ class QueryRunner {
}
async execute(): Promise<QueryResponse> {
let { datasource, fields, queryVerb, transformer, schema } = this
let {
datasource,
fields,
queryVerb,
transformer,
schema,
nullDefaultSupport,
} = this
let datasourceClone = cloneDeep(datasource)
let fieldsClone = cloneDeep(fields)
@ -100,10 +109,12 @@ class QueryRunner {
)
}
let query
let query: Record<string, any>
// handle SQL injections by interpolating the variables
if (isSQL(datasourceClone)) {
query = await interpolateSQL(fieldsClone, enrichedContext, integration)
query = await interpolateSQL(fieldsClone, enrichedContext, integration, {
nullDefaultSupport,
})
} else {
query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
}
@ -137,7 +148,9 @@ class QueryRunner {
data: rows,
params: enrichedParameters,
}
rows = vm.withContext(ctx, () => vm.execute(transformer))
if (transformer != null) {
rows = vm.withContext(ctx, () => vm.execute(transformer!))
}
}
// if the request fails we retry once, invalidating the cached value
@ -191,13 +204,15 @@ class QueryRunner {
})
return new QueryRunner(
{
datasource,
schema: query.schema,
queryVerb: query.queryVerb,
fields: query.fields,
parameters,
transformer: query.transformer,
queryId,
nullDefaultSupport: query.nullDefaultSupport,
ctx: this.ctx,
parameters,
datasource,
queryId,
},
{ noRecursiveQuery: true }
).execute()

View file

@ -15,6 +15,8 @@ export interface Query extends Document {
schema: Record<string, QuerySchema | string>
readable: boolean
queryVerb: string
// flag to state whether the default bindings are empty strings (old behaviour) or null
nullDefaultSupport?: boolean
}
export interface QueryPreview extends Omit<Query, "_id"> {