From ae4bc362c325807a8d73ab07797572b261fd25eb Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 21 Feb 2024 21:30:22 +0000 Subject: [PATCH 1/3] =?UTF-8?q?Revert=20"Dynamic=20schema=20generation=20f?= =?UTF-8?q?or=20query=20arrays:=20bug=20fix=20and=20refactor=20(#13?= =?UTF-8?q?=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 0c9d13c79a5b11f4dc19017e3983bc28a295717b. --- .../builder/src/stores/builder/queries.js | 12 +- .../server/src/api/controllers/query/index.ts | 145 ++++----- .../src/api/controllers/query/validation.ts | 40 +-- packages/server/src/api/routes/query.ts | 2 +- .../src/api/routes/tests/datasource.spec.ts | 18 +- .../routes/tests/environmentVariables.spec.ts | 12 +- .../routes/tests/queries/query.seq.spec.ts | 305 +++++------------- .../src/tests/utilities/TestConfiguration.ts | 22 ++ .../server/src/tests/utilities/api/query.ts | 16 - .../server/src/tests/utilities/structures.ts | 2 +- packages/server/src/threads/definitions.ts | 4 +- packages/server/src/threads/query.ts | 2 +- packages/types/src/documents/app/query.ts | 2 +- 13 files changed, 203 insertions(+), 379 deletions(-) diff --git a/packages/builder/src/stores/builder/queries.js b/packages/builder/src/stores/builder/queries.js index 32e8501452..7f5f83a792 100644 --- a/packages/builder/src/stores/builder/queries.js +++ b/packages/builder/src/stores/builder/queries.js @@ -75,7 +75,17 @@ export function createQueriesStore() { } const preview = async query => { - const result = await API.previewQuery(query) + const parameters = query.parameters.reduce( + (acc, next) => ({ + ...acc, + [next.name]: next.default, + }), + {} + ) + const result = await API.previewQuery({ + ...query, + parameters, + }) // Assume all the fields are strings and create a basic schema from the // unique fields returned by the server const schema = {} diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 768c921150..89330f3216 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -20,7 +20,6 @@ import { type ExecuteQueryRequest, type ExecuteQueryResponse, type Row, - QueryParameter, } from "@budibase/types" import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core" @@ -119,21 +118,6 @@ function getAuthConfig(ctx: UserCtx) { return authConfigCtx } -function enrichParameters( - queryParameters: QueryParameter[], - requestParameters: { [key: string]: string } = {} -): { - [key: string]: string -} { - // make sure parameters are fully enriched with defaults - for (let parameter of queryParameters) { - if (!requestParameters[parameter.name]) { - requestParameters[parameter.name] = parameter.default - } - } - return requestParameters -} - export async function preview(ctx: UserCtx) { const { datasource, envVars } = await sdk.datasources.getWithEnvVars( ctx.request.body.datasourceId @@ -158,68 +142,6 @@ export async function preview(ctx: UserCtx) { const authConfigCtx: any = getAuthConfig(ctx) - function getFieldMetadata(field: any, key: string): QuerySchema { - const makeQuerySchema = ( - type: FieldType, - name: string, - subtype?: string - ): QuerySchema => ({ - type, - name, - subtype, - }) - // Because custom queries have no fixed schema, we dynamically determine the schema, - // however types cannot be determined from null. We have no 'unknown' type, so we default to string. - let type = typeof field, - fieldMetadata = makeQuerySchema(FieldType.STRING, key) - if (field != null) - switch (type) { - case "boolean": - fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key) - break - case "object": - if (field instanceof Date) { - fieldMetadata = makeQuerySchema(FieldType.DATETIME, key) - } else if (Array.isArray(field)) { - if (field.some(item => JsonUtils.hasSchema(item))) { - fieldMetadata = makeQuerySchema( - FieldType.JSON, - key, - JsonFieldSubType.ARRAY - ) - } else { - fieldMetadata = makeQuerySchema(FieldType.ARRAY, key) - } - } else { - fieldMetadata = makeQuerySchema(FieldType.JSON, key) - } - break - case "number": - fieldMetadata = makeQuerySchema(FieldType.NUMBER, key) - break - } - return fieldMetadata - } - - function buildNestedSchema( - nestedSchemaFields: { - [key: string]: Record - }, - key: string, - fieldArray: any[] - ) { - let schema: { [key: string]: any } = {} - // build the schema by aggregating all row objects in the array - for (const item of fieldArray) { - if (JsonUtils.hasSchema(item)) { - for (const [key, value] of Object.entries(item)) { - schema[key] = getFieldMetadata(value, key) - } - } - } - nestedSchemaFields[key] = schema - } - function getSchemaFields( rows: any[], keys: string[] @@ -233,16 +155,51 @@ export async function preview(ctx: UserCtx) { const nestedSchemaFields: { [key: string]: Record } = {} + const makeQuerySchema = ( + type: FieldType, + name: string, + subtype?: string + ): QuerySchema => ({ + type, + name, + subtype, + }) if (rows?.length > 0) { - for (let key of new Set(keys)) { - const fieldMetadata = getFieldMetadata(rows[0][key], key) + for (let key of [...new Set(keys)] as string[]) { + const field = rows[0][key] + let type = typeof field, + fieldMetadata = makeQuerySchema(FieldType.STRING, key) + if (field) + switch (type) { + case "boolean": + fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key) + break + case "object": + if (field instanceof Date) { + fieldMetadata = makeQuerySchema(FieldType.DATETIME, key) + } else if (Array.isArray(field)) { + if (JsonUtils.hasSchema(field[0])) { + fieldMetadata = makeQuerySchema( + FieldType.JSON, + key, + JsonFieldSubType.ARRAY + ) + } else { + fieldMetadata = makeQuerySchema(FieldType.ARRAY, key) + } + nestedSchemaFields[key] = getSchemaFields( + field, + Object.keys(field[0]) + ).previewSchema + } else { + fieldMetadata = makeQuerySchema(FieldType.JSON, key) + } + break + case "number": + fieldMetadata = makeQuerySchema(FieldType.NUMBER, key) + break + } previewSchema[key] = fieldMetadata - if ( - fieldMetadata.type === FieldType.JSON && - fieldMetadata.subtype === JsonFieldSubType.ARRAY - ) { - buildNestedSchema(nestedSchemaFields, key, rows[0][key]) - } } } return { previewSchema, nestedSchemaFields } @@ -254,7 +211,7 @@ export async function preview(ctx: UserCtx) { datasource, queryVerb, fields, - parameters: enrichParameters(parameters), + parameters, transformer, queryId, schema, @@ -309,6 +266,15 @@ async function execute( if (!opts.isAutomation) { authConfigCtx = getAuthConfig(ctx) } + const enrichedParameters = ctx.request.body.parameters || {} + // make sure parameters are fully enriched with defaults + if (query && query.parameters) { + for (let parameter of query.parameters) { + if (!enrichedParameters[parameter.name]) { + enrichedParameters[parameter.name] = parameter.default + } + } + } // call the relevant CRUD method on the integration class try { @@ -318,10 +284,7 @@ async function execute( queryVerb: query.queryVerb, fields: query.fields, pagination: ctx.request.body.pagination, - parameters: enrichParameters( - query.parameters, - ctx.request.body.parameters - ), + parameters: enrichedParameters, transformer: query.transformer, queryId: ctx.params.queryId, // have to pass down to the thread runner - can't put into context now diff --git a/packages/server/src/api/controllers/query/validation.ts b/packages/server/src/api/controllers/query/validation.ts index 7d4958f1e6..339035c945 100644 --- a/packages/server/src/api/controllers/query/validation.ts +++ b/packages/server/src/api/controllers/query/validation.ts @@ -3,10 +3,11 @@ import Joi from "joi" const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("") -function baseQueryValidation() { - return { - _id: OPTIONAL_STRING, - _rev: OPTIONAL_STRING, +export function queryValidation() { + return Joi.object({ + _id: Joi.string(), + _rev: Joi.string(), + name: Joi.string().required(), fields: Joi.object().required(), datasourceId: Joi.string().required(), readable: Joi.boolean(), @@ -16,19 +17,11 @@ function baseQueryValidation() { default: Joi.string().allow(""), }) ), - queryVerb: Joi.string().required(), + queryVerb: Joi.string().allow().required(), extra: Joi.object().optional(), schema: Joi.object({}).required().unknown(true), transformer: OPTIONAL_STRING, flags: Joi.object().optional(), - queryId: OPTIONAL_STRING, - } -} - -export function queryValidation() { - return Joi.object({ - ...baseQueryValidation(), - name: Joi.string().required(), }).unknown(true) } @@ -39,10 +32,19 @@ export function generateQueryValidation() { export function generateQueryPreviewValidation() { // prettier-ignore - return auth.joiValidator.body( - Joi.object({ - ...baseQueryValidation(), - name: OPTIONAL_STRING, - }).unknown(true) - ) + return auth.joiValidator.body(Joi.object({ + _id: OPTIONAL_STRING, + _rev: OPTIONAL_STRING, + readable: Joi.boolean().optional(), + fields: Joi.object().required(), + queryVerb: Joi.string().required(), + name: OPTIONAL_STRING, + flags: Joi.object().optional(), + schema: Joi.object().optional(), + extra: Joi.object().optional(), + datasourceId: Joi.string().required(), + transformer: OPTIONAL_STRING, + parameters: Joi.object({}).required().unknown(true), + queryId: OPTIONAL_STRING, + }).unknown(true)) } diff --git a/packages/server/src/api/routes/query.ts b/packages/server/src/api/routes/query.ts index eb857d0637..fd9c51da4d 100644 --- a/packages/server/src/api/routes/query.ts +++ b/packages/server/src/api/routes/query.ts @@ -8,8 +8,8 @@ import { paramResource, } from "../../middleware/resourceId" import { - generateQueryValidation, generateQueryPreviewValidation, + generateQueryValidation, } from "../controllers/query/validation" const { BUILDER, PermissionType, PermissionLevel } = permissions diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 41229b0a2a..73bb5056ce 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -7,7 +7,6 @@ import sdk from "../../../sdk" import tk from "timekeeper" import { mocks } from "@budibase/backend-core/tests" -import { QueryPreview } from "@budibase/types" tk.freeze(mocks.date.MOCK_DATE) @@ -64,17 +63,14 @@ describe("/datasources", () => { datasource: any, fields: { path: string; queryString: string } ) { - const queryPreview: QueryPreview = { + return config.previewQuery( + request, + config, + datasource, fields, - datasourceId: datasource._id, - parameters: [], - transformer: null, - queryVerb: "read", - name: datasource.name, - schema: {}, - readable: true, - } - return config.api.query.previewQuery(queryPreview) + undefined, + "" + ) } it("should invalidate changed or removed variables", async () => { diff --git a/packages/server/src/api/routes/tests/environmentVariables.spec.ts b/packages/server/src/api/routes/tests/environmentVariables.spec.ts index 22114a1da3..aacf89ea6d 100644 --- a/packages/server/src/api/routes/tests/environmentVariables.spec.ts +++ b/packages/server/src/api/routes/tests/environmentVariables.spec.ts @@ -14,7 +14,6 @@ jest.mock("pg", () => { import * as setup from "./utilities" import { mocks } from "@budibase/backend-core/tests" import { env, events } from "@budibase/backend-core" -import { QueryPreview } from "@budibase/types" const structures = setup.structures @@ -121,19 +120,16 @@ describe("/api/env/variables", () => { .expect(200) expect(response.body.datasource._id).toBeDefined() - const queryPreview: QueryPreview = { + const query = { datasourceId: response.body.datasource._id, - parameters: [], + parameters: {}, fields: {}, queryVerb: "read", name: response.body.datasource.name, - transformer: null, - schema: {}, - readable: true, } const res = await request .post(`/api/queries/preview`) - .send(queryPreview) + .send(query) .set(config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) @@ -143,7 +139,7 @@ describe("/api/env/variables", () => { delete response.body.datasource.config expect(events.query.previewed).toBeCalledWith( response.body.datasource, - queryPreview + query ) expect(pg.Client).toHaveBeenCalledWith({ password: "test", ssl: undefined }) }) diff --git a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts index 52d35fa782..ba41ba3d16 100644 --- a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts +++ b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts @@ -1,7 +1,5 @@ import tk from "timekeeper" -const pg = require("pg") - // Mock out postgres for this jest.mock("pg") jest.mock("node-fetch") @@ -24,13 +22,7 @@ import { checkCacheForDynamicVariable } from "../../../../threads/utils" const { basicQuery, basicDatasource } = setup.structures import { events, db as dbCore } from "@budibase/backend-core" -import { - Datasource, - Query, - SourceName, - QueryPreview, - QueryParameter, -} from "@budibase/types" +import { Datasource, Query, SourceName } from "@budibase/types" tk.freeze(Date.now()) @@ -226,26 +218,28 @@ describe("/queries", () => { describe("preview", () => { it("should be able to preview the query", async () => { - const queryPreview: QueryPreview = { + const query = { datasourceId: datasource._id, - queryVerb: "read", + parameters: {}, fields: {}, - parameters: [], - transformer: "return data", - name: datasource.name!, - schema: {}, - readable: true, + queryVerb: "read", + name: datasource.name, } - const responseBody = await config.api.query.previewQuery(queryPreview) + const res = await request + .post(`/api/queries/preview`) + .send(query) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) // these responses come from the mock - expect(responseBody.schema).toEqual({ + expect(res.body.schema).toEqual({ a: { type: "string", name: "a" }, b: { type: "number", name: "b" }, }) - expect(responseBody.rows.length).toEqual(1) + expect(res.body.rows.length).toEqual(1) expect(events.query.previewed).toBeCalledTimes(1) delete datasource.config - expect(events.query.previewed).toBeCalledWith(datasource, queryPreview) + expect(events.query.previewed).toBeCalledWith(datasource, query) }) it("should apply authorization to endpoint", async () => { @@ -255,128 +249,6 @@ describe("/queries", () => { url: `/api/queries/preview`, }) }) - - it("should not error when trying to generate a nested schema for an empty array", async () => { - const queryPreview: QueryPreview = { - datasourceId: datasource._id, - parameters: [], - fields: {}, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - const rows = [ - { - contacts: [], - }, - ] - pg.queryMock.mockImplementation(() => ({ - rows, - })) - - const responseBody = await config.api.query.previewQuery(queryPreview) - expect(responseBody).toEqual({ - nestedSchemaFields: {}, - rows, - schema: { - contacts: { type: "array", name: "contacts" }, - }, - }) - expect(responseBody.rows.length).toEqual(1) - delete datasource.config - }) - - it("should generate a nested schema based on all the nested items", async () => { - const queryPreview: QueryPreview = { - datasourceId: datasource._id, - parameters: [], - fields: {}, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - const rows = [ - { - contacts: [ - { - address: "123 Lane", - }, - { - address: "456 Drive", - }, - { - postcode: "BT1 12N", - lat: 54.59, - long: -5.92, - }, - { - city: "Belfast", - }, - { - address: "789 Avenue", - phoneNumber: "0800-999-5555", - }, - { - name: "Name", - isActive: false, - }, - ], - }, - ] - pg.queryMock.mockImplementation(() => ({ - rows, - })) - - const responseBody = await config.api.query.previewQuery(queryPreview) - expect(responseBody).toEqual({ - nestedSchemaFields: { - contacts: { - address: { - type: "string", - name: "address", - }, - postcode: { - type: "string", - name: "postcode", - }, - lat: { - type: "number", - name: "lat", - }, - long: { - type: "number", - name: "long", - }, - city: { - type: "string", - name: "city", - }, - phoneNumber: { - type: "string", - name: "phoneNumber", - }, - name: { - type: "string", - name: "name", - }, - isActive: { - type: "boolean", - name: "isActive", - }, - }, - }, - rows, - schema: { - contacts: { type: "json", name: "contacts", subtype: "array" }, - }, - }) - expect(responseBody.rows.length).toEqual(1) - delete datasource.config - }) }) describe("execute", () => { @@ -411,17 +283,7 @@ describe("/queries", () => { describe("variables", () => { async function preview(datasource: Datasource, fields: any) { - const queryPreview: QueryPreview = { - datasourceId: datasource._id!, - parameters: [], - fields, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - return await config.api.query.previewQuery(queryPreview) + return config.previewQuery(request, config, datasource, fields, undefined) } it("should work with static variables", async () => { @@ -431,31 +293,31 @@ describe("/queries", () => { variable2: "1", }, }) - const responseBody = await preview(datasource, { + const res = await preview(datasource, { path: "www.{{ variable }}.com", queryString: "test={{ variable2 }}", }) // these responses come from the mock - expect(responseBody.schema).toEqual({ + expect(res.body.schema).toEqual({ opts: { type: "json", name: "opts" }, url: { type: "string", name: "url" }, value: { type: "string", name: "value" }, }) - expect(responseBody.rows[0].url).toEqual("http://www.google.com?test=1") + expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1") }) it("should work with dynamic variables", async () => { const { datasource } = await config.dynamicVariableDatasource() - const responseBody = await preview(datasource, { + const res = await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}", }) - expect(responseBody.schema).toEqual({ + expect(res.body.schema).toEqual({ opts: { type: "json", name: "opts" }, url: { type: "string", name: "url" }, value: { type: "string", name: "value" }, }) - expect(responseBody.rows[0].url).toContain("doctype%20html") + expect(res.body.rows[0].url).toContain("doctype%20html") }) it("check that it automatically retries on fail with cached dynamics", async () => { @@ -469,16 +331,16 @@ describe("/queries", () => { // check its in cache const contents = await checkCacheForDynamicVariable(base._id, "variable3") expect(contents.rows.length).toEqual(1) - const responseBody = await preview(datasource, { + const res = await preview(datasource, { path: "www.failonce.com", queryString: "test={{ variable3 }}", }) - expect(responseBody.schema).toEqual({ + expect(res.body.schema).toEqual({ fails: { type: "number", name: "fails" }, opts: { type: "json", name: "opts" }, url: { type: "string", name: "url" }, }) - expect(responseBody.rows[0].fails).toEqual(1) + expect(res.body.rows[0].fails).toEqual(1) }) it("deletes variables when linked query is deleted", async () => { @@ -509,37 +371,24 @@ describe("/queries", () => { async function previewGet( datasource: Datasource, fields: any, - params: QueryParameter[] + params: any ) { - const queryPreview: QueryPreview = { - datasourceId: datasource._id!, - parameters: params, - fields, - queryVerb: "read", - name: datasource.name!, - transformer: "return data", - schema: {}, - readable: true, - } - return await config.api.query.previewQuery(queryPreview) + return config.previewQuery(request, config, datasource, fields, params) } async function previewPost( datasource: Datasource, fields: any, - params: QueryParameter[] + params: any ) { - const queryPreview: QueryPreview = { - datasourceId: datasource._id!, - parameters: params, + return config.previewQuery( + request, + config, + datasource, fields, - queryVerb: "create", - name: datasource.name!, - transformer: null, - schema: {}, - readable: false, - } - return await config.api.query.previewQuery(queryPreview) + params, + "create" + ) } it("should parse global and query level header mappings", async () => { @@ -551,7 +400,7 @@ describe("/queries", () => { emailHdr: "{{[user].[email]}}", }, }) - const responseBody = await previewGet( + const res = await previewGet( datasource, { path: "www.google.com", @@ -561,17 +410,17 @@ describe("/queries", () => { secondHdr: "1234", }, }, - [] + undefined ) - const parsedRequest = JSON.parse(responseBody.extra.raw) + const parsedRequest = JSON.parse(res.body.extra.raw) expect(parsedRequest.opts.headers).toEqual({ test: "headerVal", emailHdr: userDetails.email, queryHdr: userDetails.firstName, secondHdr: "1234", }) - expect(responseBody.rows[0].url).toEqual( + expect(res.body.rows[0].url).toEqual( "http://www.google.com?email=" + userDetails.email.replace("@", "%40") ) }) @@ -581,21 +430,21 @@ describe("/queries", () => { const datasource = await config.restDatasource() - const responseBody = await previewGet( + const res = await previewGet( datasource, { path: "www.google.com", queryString: "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}", }, - [ - { name: "myEmail", default: "{{[user].[email]}}" }, - { name: "myName", default: "{{[user].[firstName]}}" }, - { name: "testParam", default: "1234" }, - ] + { + myEmail: "{{[user].[email]}}", + myName: "{{[user].[firstName]}}", + testParam: "1234", + } ) - expect(responseBody.rows[0].url).toEqual( + expect(res.body.rows[0].url).toEqual( "http://www.google.com?test=" + userDetails.email.replace("@", "%40") + "&testName=" + @@ -608,7 +457,7 @@ describe("/queries", () => { const userDetails = config.getUserDetails() const datasource = await config.restDatasource() - const responseBody = await previewPost( + const res = await previewPost( datasource, { path: "www.google.com", @@ -617,14 +466,16 @@ describe("/queries", () => { "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}", bodyType: "text", }, - [{ name: "testParam", default: "1234" }] + { + testParam: "1234", + } ) - const parsedRequest = JSON.parse(responseBody.extra.raw) + const parsedRequest = JSON.parse(res.body.extra.raw) expect(parsedRequest.opts.body).toEqual( `This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234` ) - expect(responseBody.rows[0].url).toEqual( + expect(res.body.rows[0].url).toEqual( "http://www.google.com?testParam=1234" ) }) @@ -633,7 +484,7 @@ describe("/queries", () => { const userDetails = config.getUserDetails() const datasource = await config.restDatasource() - const responseBody = await previewPost( + const res = await previewPost( datasource, { path: "www.google.com", @@ -642,16 +493,16 @@ describe("/queries", () => { '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', bodyType: "json", }, - [ - { name: "testParam", default: "1234" }, - { name: "userRef", default: "{{[user].[firstName]}}" }, - ] + { + testParam: "1234", + userRef: "{{[user].[firstName]}}", + } ) - const parsedRequest = JSON.parse(responseBody.extra.raw) + const parsedRequest = JSON.parse(res.body.extra.raw) const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}` expect(parsedRequest.opts.body).toEqual(test) - expect(responseBody.rows[0].url).toEqual( + expect(res.body.rows[0].url).toEqual( "http://www.google.com?testParam=1234" ) }) @@ -660,7 +511,7 @@ describe("/queries", () => { const userDetails = config.getUserDetails() const datasource = await config.restDatasource() - const responseBody = await previewPost( + const res = await previewPost( datasource, { path: "www.google.com", @@ -670,17 +521,17 @@ describe("/queries", () => { "{{userId}} testing ", bodyType: "xml", }, - [ - { name: "testParam", default: "1234" }, - { name: "userId", default: "{{[user].[firstName]}}" }, - ] + { + testParam: "1234", + userId: "{{[user].[firstName]}}", + } ) - const parsedRequest = JSON.parse(responseBody.extra.raw) + const parsedRequest = JSON.parse(res.body.extra.raw) const test = ` ${userDetails.email} 1234 ${userDetails.firstName} testing ` expect(parsedRequest.opts.body).toEqual(test) - expect(responseBody.rows[0].url).toEqual( + expect(res.body.rows[0].url).toEqual( "http://www.google.com?testParam=1234" ) }) @@ -689,7 +540,7 @@ describe("/queries", () => { const userDetails = config.getUserDetails() const datasource = await config.restDatasource() - const responseBody = await previewPost( + const res = await previewPost( datasource, { path: "www.google.com", @@ -698,13 +549,13 @@ describe("/queries", () => { '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', bodyType: "form", }, - [ - { name: "testParam", default: "1234" }, - { name: "userRef", default: "{{[user].[firstName]}}" }, - ] + { + testParam: "1234", + userRef: "{{[user].[firstName]}}", + } ) - const parsedRequest = JSON.parse(responseBody.extra.raw) + const parsedRequest = JSON.parse(res.body.extra.raw) const emailData = parsedRequest.opts.body._streams[1] expect(emailData).toEqual(userDetails.email) @@ -715,7 +566,7 @@ describe("/queries", () => { const userRef = parsedRequest.opts.body._streams[7] expect(userRef).toEqual(userDetails.firstName) - expect(responseBody.rows[0].url).toEqual( + expect(res.body.rows[0].url).toEqual( "http://www.google.com?testParam=1234" ) }) @@ -724,7 +575,7 @@ describe("/queries", () => { const userDetails = config.getUserDetails() const datasource = await config.restDatasource() - const responseBody = await previewPost( + const res = await previewPost( datasource, { path: "www.google.com", @@ -733,12 +584,12 @@ describe("/queries", () => { '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', bodyType: "encoded", }, - [ - { name: "testParam", default: "1234" }, - { name: "userRef", default: "{{[user].[firstName]}}" }, - ] + { + testParam: "1234", + userRef: "{{[user].[firstName]}}", + } ) - const parsedRequest = JSON.parse(responseBody.extra.raw) + const parsedRequest = JSON.parse(res.body.extra.raw) expect(parsedRequest.opts.body.email).toEqual(userDetails.email) expect(parsedRequest.opts.body.queryCode).toEqual("1234") diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 22bb66b130..8e6ecdfeb1 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -866,6 +866,28 @@ export default class TestConfiguration { // QUERY + async previewQuery( + request: any, + config: any, + datasource: any, + fields: any, + params: any, + verb?: string + ) { + return request + .post(`/api/queries/preview`) + .send({ + datasourceId: datasource._id, + parameters: params || {}, + fields, + queryVerb: verb || "read", + name: datasource.name, + }) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + } + async createQuery(config?: any) { if (!this.datasource && !config) { throw "No datasource created for query." diff --git a/packages/server/src/tests/utilities/api/query.ts b/packages/server/src/tests/utilities/api/query.ts index b0eac5c8b7..350fe03c74 100644 --- a/packages/server/src/tests/utilities/api/query.ts +++ b/packages/server/src/tests/utilities/api/query.ts @@ -1,7 +1,6 @@ import TestConfiguration from "../TestConfiguration" import { Query, - QueryPreview, type ExecuteQueryRequest, type ExecuteQueryResponse, } from "@budibase/types" @@ -42,19 +41,4 @@ export class QueryAPI extends TestAPI { return res.body } - - previewQuery = async (queryPreview: QueryPreview) => { - const res = await this.request - .post(`/api/queries/preview`) - .send(queryPreview) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - if (res.status !== 200) { - throw new Error(JSON.stringify(res.body)) - } - - return res.body - } } diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index 2fecf15fd6..fe82311810 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -366,7 +366,7 @@ export function basicDatasource(): { datasource: Datasource } { export function basicQuery(datasourceId: string): Query { return { - datasourceId, + datasourceId: datasourceId, name: "New Query", parameters: [], fields: {}, diff --git a/packages/server/src/threads/definitions.ts b/packages/server/src/threads/definitions.ts index 14b97c57b1..52f5576d9d 100644 --- a/packages/server/src/threads/definitions.ts +++ b/packages/server/src/threads/definitions.ts @@ -7,10 +7,10 @@ export interface QueryEvent { datasource: Datasource queryVerb: string fields: { [key: string]: any } - parameters: { [key: string]: unknown } + parameters: { [key: string]: any } pagination?: any transformer: any - queryId?: string + queryId: string environmentVariables?: Record ctx?: any schema?: Record diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 6cdccc7868..9366f2b12c 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -43,7 +43,7 @@ class QueryRunner { this.parameters = input.parameters this.pagination = input.pagination this.transformer = input.transformer - this.queryId = input.queryId! + this.queryId = input.queryId this.schema = input.schema this.noRecursiveQuery = flags.noRecursiveQuery this.cachedVariables = [] diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts index b1b0a1d780..f4547b9774 100644 --- a/packages/types/src/documents/app/query.ts +++ b/packages/types/src/documents/app/query.ts @@ -19,7 +19,7 @@ export interface Query extends Document { } export interface QueryPreview extends Omit { - queryId?: string + queryId: string } export interface QueryParameter { From 6aa577106699f8425033c1f5d8703833c20bcbd0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 21 Feb 2024 18:35:27 -0300 Subject: [PATCH 2/3] acct-portal-sub --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 4384bc742c..92129b0d62 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 4384bc742ca22fb1e9bf91843e65ae929daf17e2 +Subproject commit 92129b0d6251a7b4dabb367c15b09c079399b763 From fcd2ed14e3f77d1270c823c0d6f16384fd3771c5 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 21 Feb 2024 18:46:40 -0300 Subject: [PATCH 3/3] revert acct portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 92129b0d62..97329c0318 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 92129b0d6251a7b4dabb367c15b09c079399b763 +Subproject commit 97329c0318ef0f4bbbd2b9ce30d6976bc6505272