From f3d466f25585f477b254441cb3784ed6876109e1 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 25 Jun 2024 08:51:35 +0100 Subject: [PATCH] fix issue where schema wasn't updating types when a query was run (#14004) * fix issue where schema wasn't updating types when a query was run * add tests for schema matching --- .../integration/RestQueryViewer.svelte | 6 +- .../server/src/api/controllers/query/index.ts | 4 +- .../routes/tests/queries/generic-sql.spec.ts | 61 +++++++++++++++++++ .../api/routes/tests/queries/mongodb.spec.ts | 61 +++++++++++++++++++ .../src/api/routes/tests/queries/rest.spec.ts | 55 +++++++++++++++++ 5 files changed, 182 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte index b44831550e..0489167823 100644 --- a/packages/builder/src/components/integration/RestQueryViewer.svelte +++ b/packages/builder/src/components/integration/RestQueryViewer.svelte @@ -233,9 +233,9 @@ response.info = response.info || { code: 200 } // if existing schema, copy over what it is if (schema) { - for (let [name, field] of Object.entries(schema)) { - if (response.schema[name]) { - response.schema[name] = field + for (let [name, field] of Object.entries(response.schema)) { + if (!schema[name]) { + schema[name] = field } } } diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index b52cea553f..54f672c3f3 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -311,8 +311,8 @@ export async function preview( // if existing schema, update to include any previous schema keys if (existingSchema) { - for (let key of Object.keys(previewSchema)) { - if (existingSchema[key]) { + for (let key of Object.keys(existingSchema)) { + if (!previewSchema[key]) { previewSchema[key] = existingSchema[key] } } diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index e72a091688..3ed19f5eee 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -250,6 +250,67 @@ describe.each( expect(events.query.previewed).toHaveBeenCalledTimes(1) }) + it("should update schema when column type changes from number to string", async () => { + const tableName = "schema_change_test" + await client.schema.dropTableIfExists(tableName) + + await client.schema.createTable(tableName, table => { + table.increments("id").primary() + table.string("name") + table.integer("data") + }) + + await client(tableName).insert({ + name: "test", + data: 123, + }) + + const firstPreview = await config.api.query.preview({ + datasourceId: datasource._id!, + name: "Test Query", + queryVerb: "read", + fields: { + sql: `SELECT * FROM ${tableName}`, + }, + parameters: [], + transformer: "return data", + schema: {}, + readable: true, + }) + + expect(firstPreview.schema).toEqual( + expect.objectContaining({ + data: { type: "number", name: "data" }, + }) + ) + + await client.schema.alterTable(tableName, table => { + table.string("data").alter() + }) + + await client(tableName).update({ + data: "string value", + }) + + const secondPreview = await config.api.query.preview({ + datasourceId: datasource._id!, + name: "Test Query", + queryVerb: "read", + fields: { + sql: `SELECT * FROM ${tableName}`, + }, + parameters: [], + transformer: "return data", + schema: firstPreview.schema, + readable: true, + }) + + expect(secondPreview.schema).toEqual( + expect.objectContaining({ + data: { type: "string", name: "data" }, + }) + ) + }) it("should work with static variables", async () => { await config.api.datasource.update({ ...datasource, diff --git a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts index c79ae68a36..4822729478 100644 --- a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts +++ b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts @@ -137,6 +137,67 @@ describe("/queries", () => { }) }) + it("should update schema when structure changes from object to array", async () => { + const name = generator.guid() + + await withCollection(async collection => { + await collection.insertOne({ name, field: { subfield: "value" } }) + }) + + const firstPreview = await config.api.query.preview({ + name: "Test Query", + datasourceId: datasource._id!, + fields: { + json: { name: { $eq: name } }, + extra: { + collection, + actionType: "findOne", + }, + }, + schema: {}, + queryVerb: "read", + parameters: [], + transformer: "return data", + readable: true, + }) + + expect(firstPreview.schema).toEqual( + expect.objectContaining({ + field: { type: "json", name: "field" }, + }) + ) + + await withCollection(async collection => { + await collection.updateOne( + { name }, + { $set: { field: ["value1", "value2"] } } + ) + }) + + const secondPreview = await config.api.query.preview({ + name: "Test Query", + datasourceId: datasource._id!, + fields: { + json: { name: { $eq: name } }, + extra: { + collection, + actionType: "findOne", + }, + }, + schema: firstPreview.schema, + queryVerb: "read", + parameters: [], + transformer: "return data", + readable: true, + }) + + expect(secondPreview.schema).toEqual( + expect.objectContaining({ + field: { type: "array", name: "field" }, + }) + ) + }) + it("should generate a nested schema based on all of the nested items", async () => { const name = generator.guid() const item = { diff --git a/packages/server/src/api/routes/tests/queries/rest.spec.ts b/packages/server/src/api/routes/tests/queries/rest.spec.ts index 1d5483017b..29bbbf3a61 100644 --- a/packages/server/src/api/routes/tests/queries/rest.spec.ts +++ b/packages/server/src/api/routes/tests/queries/rest.spec.ts @@ -92,6 +92,61 @@ describe("rest", () => { expect(cached.rows[0].name).toEqual("one") }) + it("should update schema when structure changes from JSON to array", async () => { + const datasource = await config.api.datasource.create({ + name: generator.guid(), + type: "test", + source: SourceName.REST, + config: {}, + }) + + nock("http://www.example.com") + .get("/") + .reply(200, [{ obj: {}, id: "1" }]) + + const firstResponse = await config.api.query.preview({ + datasourceId: datasource._id!, + name: "test query", + parameters: [], + queryVerb: "read", + transformer: "", + schema: {}, + readable: true, + fields: { + path: "www.example.com", + }, + }) + + expect(firstResponse.schema).toEqual({ + obj: { type: "json", name: "obj" }, + id: { type: "string", name: "id" }, + }) + + nock.cleanAll() + + nock("http://www.example.com") + .get("/") + .reply(200, [{ obj: [], id: "1" }]) + + const secondResponse = await config.api.query.preview({ + datasourceId: datasource._id!, + name: "test query", + parameters: [], + queryVerb: "read", + transformer: "", + schema: firstResponse.schema, + readable: true, + fields: { + path: "www.example.com", + }, + }) + + expect(secondResponse.schema).toEqual({ + obj: { type: "array", name: "obj" }, + id: { type: "string", name: "id" }, + }) + }) + it("should parse global and query level header mappings", async () => { const datasource = await config.api.datasource.create({ name: generator.guid(),