From c2320e4f5bac2419ea41e10a0b344fc6ca2d2910 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 14 Jun 2024 17:20:08 +0100 Subject: [PATCH 1/6] Convert postgres.spec.ts to use Knex instead of rawQuery. --- .../src/integration-test/postgres.spec.ts | 170 ++++++++---------- .../src/integrations/tests/utils/index.ts | 17 ++ .../src/integrations/tests/utils/mssql.ts | 15 ++ .../src/integrations/tests/utils/mysql.ts | 15 ++ .../src/integrations/tests/utils/postgres.ts | 15 ++ 5 files changed, 135 insertions(+), 97 deletions(-) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index cf4752978c..32eb91ca62 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -1,9 +1,3 @@ -import fetch from "node-fetch" -import { - generateMakeRequest, - MakeRequestResponse, -} from "../api/routes/public/tests/utils" - import * as setup from "../api/routes/tests/utilities" import { Datasource, FieldType } from "@budibase/types" import _ from "lodash" @@ -11,29 +5,22 @@ import { generator } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource, - rawQuery, + knexClient, } from "../integrations/tests/utils" - -// @ts-ignore -fetch.mockSearch() +import { Knex } from "knex" const config = setup.getConfig()! -jest.mock("../websockets") - describe("postgres integrations", () => { - let makeRequest: MakeRequestResponse, - rawDatasource: Datasource, - datasource: Datasource + let rawDatasource: Datasource + let datasource: Datasource + let client: Knex beforeAll(async () => { await config.init() - const apiKey = await config.generateApiKey() - - makeRequest = generateMakeRequest(apiKey, true) - rawDatasource = await getDatasource(DatabaseName.POSTGRES) datasource = await config.api.datasource.create(rawDatasource) + client = await knexClient(rawDatasource) }) afterAll(config.end) @@ -46,11 +33,13 @@ describe("postgres integrations", () => { }) afterEach(async () => { - await rawQuery(rawDatasource, `DROP TABLE IF EXISTS "${tableName}"`) + await client.schema.dropTableIfExists(tableName) }) it("recognises when a table has no primary key", async () => { - await rawQuery(rawDatasource, `CREATE TABLE "${tableName}" (id SERIAL)`) + await client.schema.createTable(tableName, table => { + table.increments("id", { primaryKey: false }) + }) const response = await config.api.datasource.fetchSchema({ datasourceId: datasource._id!, @@ -62,10 +51,9 @@ describe("postgres integrations", () => { }) it("recognises when a table is using a reserved column name", async () => { - await rawQuery( - rawDatasource, - `CREATE TABLE "${tableName}" (_id SERIAL PRIMARY KEY) ` - ) + await client.schema.createTable(tableName, table => { + table.increments("_id").primary() + }) const response = await config.api.datasource.fetchSchema({ datasourceId: datasource._id!, @@ -81,20 +69,15 @@ describe("postgres integrations", () => { .guid() .replaceAll("-", "") .substring(0, 6)}` - const enumColumnName = "status" - await rawQuery( - rawDatasource, - ` - CREATE TYPE order_status AS ENUM ('pending', 'processing', 'shipped', 'delivered', 'cancelled'); - - CREATE TABLE ${tableName} ( - order_id SERIAL PRIMARY KEY, - customer_name VARCHAR(100) NOT NULL, - ${enumColumnName} order_status - ); - ` - ) + await client.schema.createTable(tableName, table => { + table.increments("order_id").primary() + table.string("customer_name").notNullable() + table.enum("status", ["pending", "processing", "shipped"], { + useNative: true, + enumName: `${tableName}_status`, + }) + }) const response = await config.api.datasource.fetchSchema({ datasourceId: datasource._id!, @@ -103,13 +86,14 @@ describe("postgres integrations", () => { const table = response.datasource.entities?.[tableName] expect(table).toBeDefined() - expect(table?.schema[enumColumnName].type).toEqual(FieldType.OPTIONS) + expect(table?.schema["status"].type).toEqual(FieldType.OPTIONS) }) }) describe("Integration compatibility with postgres search_path", () => { let rawDatasource: Datasource, datasource: Datasource, + client: Knex, schema1: string, schema2: string @@ -118,54 +102,55 @@ describe("postgres integrations", () => { schema2 = generator.guid().replaceAll("-", "") rawDatasource = await getDatasource(DatabaseName.POSTGRES) - const dbConfig = rawDatasource.config! + client = await knexClient(rawDatasource) - await rawQuery(rawDatasource, `CREATE SCHEMA "${schema1}";`) - await rawQuery(rawDatasource, `CREATE SCHEMA "${schema2}";`) + await client.schema.createSchema(schema1) + await client.schema.createSchema(schema2) - const pathConfig: any = { - ...rawDatasource, - config: { - ...dbConfig, - schema: `${schema1}, ${schema2}`, - }, - } - datasource = await config.api.datasource.create(pathConfig) + rawDatasource.config!.schema = `${schema1}, ${schema2}` + + client = await knexClient(rawDatasource) + datasource = await config.api.datasource.create(rawDatasource) }) afterEach(async () => { - await rawQuery(rawDatasource, `DROP SCHEMA "${schema1}" CASCADE;`) - await rawQuery(rawDatasource, `DROP SCHEMA "${schema2}" CASCADE;`) + await client.schema.dropSchema(schema1, true) + await client.schema.dropSchema(schema2, true) }) it("discovers tables from any schema in search path", async () => { - await rawQuery( - rawDatasource, - `CREATE TABLE "${schema1}".table1 (id1 SERIAL PRIMARY KEY);` - ) - await rawQuery( - rawDatasource, - `CREATE TABLE "${schema2}".table2 (id2 SERIAL PRIMARY KEY);` - ) - const response = await makeRequest("post", "/api/datasources/info", { - datasource: datasource, + await client.schema.createTable(`${schema1}.table1`, table => { + table.increments("id1").primary() }) - expect(response.status).toBe(200) - expect(response.body.tableNames).toBeDefined() - expect(response.body.tableNames).toEqual( + + await client.schema.createTable(`${schema2}.table2`, table => { + table.increments("id2").primary() + }) + + const response = await config.api.datasource.info(datasource) + expect(response.tableNames).toBeDefined() + expect(response.tableNames).toEqual( expect.arrayContaining(["table1", "table2"]) ) }) it("does not mix columns from different tables", async () => { const repeated_table_name = "table_same_name" - await rawQuery( - rawDatasource, - `CREATE TABLE "${schema1}".${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);` + + await client.schema.createTable( + `${schema1}.${repeated_table_name}`, + table => { + table.increments("id").primary() + table.string("val1") + } ) - await rawQuery( - rawDatasource, - `CREATE TABLE "${schema2}".${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);` + + await client.schema.createTable( + `${schema2}.${repeated_table_name}`, + table => { + table.increments("id2").primary() + table.string("val2") + } ) const response = await config.api.datasource.fetchSchema({ @@ -182,15 +167,11 @@ describe("postgres integrations", () => { describe("check custom column types", () => { beforeAll(async () => { - await rawQuery( - rawDatasource, - `CREATE TABLE binaryTable ( - id BYTEA PRIMARY KEY, - column1 TEXT, - column2 INT - ); - ` - ) + await client.schema.createTable("binaryTable", table => { + table.binary("id").primary() + table.string("column1") + table.integer("column2") + }) }) it("should handle binary columns", async () => { @@ -198,7 +179,7 @@ describe("postgres integrations", () => { datasourceId: datasource._id!, }) expect(response.datasource.entities).toBeDefined() - const table = response.datasource.entities?.["binarytable"] + const table = response.datasource.entities?.["binaryTable"] expect(table).toBeDefined() expect(table?.schema.id.externalType).toBe("bytea") const row = await config.api.row.save(table?._id!, { @@ -214,14 +195,10 @@ describe("postgres integrations", () => { describe("check fetching null/not null table", () => { beforeAll(async () => { - await rawQuery( - rawDatasource, - `CREATE TABLE nullableTable ( - order_id SERIAL PRIMARY KEY, - order_number INT NOT NULL - ); - ` - ) + await client.schema.createTable("nullableTable", table => { + table.increments("order_id").primary() + table.integer("order_number").notNullable() + }) }) it("should be able to change the table to allow nullable and refetch this", async () => { @@ -230,25 +207,24 @@ describe("postgres integrations", () => { }) const entities = response.datasource.entities expect(entities).toBeDefined() - const nullableTable = entities?.["nullabletable"] + const nullableTable = entities?.["nullableTable"] expect(nullableTable).toBeDefined() expect( nullableTable?.schema["order_number"].constraints?.presence ).toEqual(true) + // need to perform these calls raw to the DB so that the external state of the DB differs to what Budibase // is aware of - therefore we can try to fetch and make sure BB updates correctly - await rawQuery( - rawDatasource, - `ALTER TABLE nullableTable - ALTER COLUMN order_number DROP NOT NULL; - ` - ) + await client.schema.alterTable("nullableTable", table => { + table.setNullable("order_number") + }) + const responseAfter = await config.api.datasource.fetchSchema({ datasourceId: datasource._id!, }) const entitiesAfter = responseAfter.datasource.entities expect(entitiesAfter).toBeDefined() - const nullableTableAfter = entitiesAfter?.["nullabletable"] + const nullableTableAfter = entitiesAfter?.["nullableTable"] expect(nullableTableAfter).toBeDefined() expect( nullableTableAfter?.schema["order_number"].constraints?.presence diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 64617461bb..fd7d2373bb 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -65,6 +65,23 @@ export async function rawQuery(ds: Datasource, sql: string): Promise { } } +export async function knexClient(ds: Datasource) { + switch (ds.source) { + case SourceName.POSTGRES: { + return postgres.knexClient(ds) + } + case SourceName.MYSQL: { + return mysql.knexClient(ds) + } + case SourceName.SQL_SERVER: { + return mssql.knexClient(ds) + } + default: { + throw new Error(`Unsupported source: ${ds.source}`) + } + } +} + export async function startContainer(container: GenericContainer) { const imageName = (container as any).imageName.string as string const key = imageName.replaceAll("/", "-").replaceAll(":", "-") diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index 57c5fe8049..3b3533eb7f 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -3,6 +3,7 @@ import { GenericContainer, Wait } from "testcontainers" import mssql from "mssql" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." +import knex from "knex" let ports: Promise @@ -72,3 +73,17 @@ export async function rawQuery(ds: Datasource, sql: string) { await pool.close() } } + +export async function knexClient(ds: Datasource) { + if (!ds.config) { + throw new Error("Datasource config is missing") + } + if (ds.source !== SourceName.SQL_SERVER) { + throw new Error("Datasource source is not MSSQL") + } + + return knex({ + client: "mssql", + connection: ds.config, + }) +} diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts index 560d6bb2d4..cb6667bc56 100644 --- a/packages/server/src/integrations/tests/utils/mysql.ts +++ b/packages/server/src/integrations/tests/utils/mysql.ts @@ -4,6 +4,7 @@ import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait- import mysql from "mysql2/promise" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." +import knex from "knex" let ports: Promise @@ -77,3 +78,17 @@ export async function rawQuery(ds: Datasource, sql: string) { connection.end() } } + +export async function knexClient(ds: Datasource) { + if (!ds.config) { + throw new Error("Datasource config is missing") + } + if (ds.source !== SourceName.MYSQL) { + throw new Error("Datasource source is not MySQL") + } + + return knex({ + client: "mysql", + connection: ds.config, + }) +} diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index 8c0cd886e8..20b56331c1 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -3,6 +3,7 @@ import { GenericContainer, Wait } from "testcontainers" import pg from "pg" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." +import knex from "knex" let ports: Promise @@ -66,3 +67,17 @@ export async function rawQuery(ds: Datasource, sql: string) { await client.end() } } + +export async function knexClient(ds: Datasource) { + if (!ds.config) { + throw new Error("Datasource config is missing") + } + if (ds.source !== SourceName.POSTGRES) { + throw new Error("Datasource source is not Postgres") + } + + return knex({ + client: "pg", + connection: ds.config, + }) +} From c8c0dfb84d6746af82a09ed19ee294920384c292 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 09:23:30 +0000 Subject: [PATCH 2/6] Bump tar from 6.1.15 to 6.2.1 in /packages/server Bumps [tar](https://github.com/isaacs/node-tar) from 6.1.15 to 6.2.1. - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v6.1.15...v6.2.1) --- updated-dependencies: - dependency-name: tar dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- packages/server/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/package.json b/packages/server/package.json index b3beac7ffb..28b1e876c2 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -110,7 +110,7 @@ "server-destroy": "1.0.1", "snowflake-promise": "^4.5.0", "socket.io": "4.6.1", - "tar": "6.1.15", + "tar": "6.2.1", "to-json-schema": "0.2.5", "uuid": "^8.3.2", "validate.js": "0.13.1", From 84118f9e8cf69e5f9fe6d39c02abf11f81103a4b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 17 Jun 2024 14:39:45 +0100 Subject: [PATCH 3/6] Remove makeRequest from mysql.spec.ts --- .../server/src/integration-test/mysql.spec.ts | 61 ++++++------------- .../src/integration-test/postgres.spec.ts | 2 +- 2 files changed, 21 insertions(+), 42 deletions(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 4dff40b61d..dc4f79bba9 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -1,8 +1,4 @@ import fetch from "node-fetch" -import { - generateMakeRequest, - MakeRequestResponse, -} from "../api/routes/public/tests/utils" import * as setup from "../api/routes/tests/utilities" import { Datasource, FieldType } from "@budibase/types" import { @@ -12,6 +8,7 @@ import { } from "../integrations/tests/utils" import { generator } from "@budibase/backend-core/tests" import { tableForDatasource } from "../../src/tests/utilities/structures" +import { Knex } from "knex" // @ts-ignore fetch.mockSearch() @@ -40,15 +37,12 @@ jest.mock("../websockets", () => ({ })) describe("mysql integrations", () => { - let makeRequest: MakeRequestResponse, - rawDatasource: Datasource, - datasource: Datasource + let rawDatasource: Datasource + let datasource: Datasource + let client: Knex beforeAll(async () => { await config.init() - const apiKey = await config.generateApiKey() - - makeRequest = generateMakeRequest(apiKey, true) rawDatasource = await getDatasource(DatabaseName.MYSQL) datasource = await config.api.datasource.create(rawDatasource) @@ -59,11 +53,8 @@ describe("mysql integrations", () => { it("validate table schema", async () => { // Creating a table so that `entities` is populated. await config.api.table.save(tableForDatasource(datasource)) - - const res = await makeRequest("get", `/api/datasources/${datasource._id}`) - - expect(res.status).toBe(200) - expect(res.body).toEqual({ + const res = await config.api.datasource.get(datasource._id!) + expect(res).toEqual({ config: { database: expect.any(String), host: datasource.config!.host, @@ -114,14 +105,9 @@ describe("mysql integrations", () => { rawDatasource, `CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);` ) - const response = await makeRequest("post", "/api/datasources/info", { - datasource: datasource, - }) - expect(response.status).toBe(200) - expect(response.body.tableNames).toBeDefined() - expect(response.body.tableNames).toEqual( - expect.arrayContaining(["table1"]) - ) + const res = await config.api.datasource.info(datasource) + expect(res.tableNames).toBeDefined() + expect(res.tableNames).toEqual(expect.arrayContaining(["table1"])) }) it("does not mix columns from different tables", async () => { @@ -134,19 +120,13 @@ describe("mysql integrations", () => { rawDatasource, `CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);` ) - const response = await makeRequest( - "post", - `/api/datasources/${datasource._id}/schema`, - { - tablesFilter: [repeated_table_name], - } - ) - expect(response.status).toBe(200) - expect( - response.body.datasource.entities[repeated_table_name].schema - ).toBeDefined() - const schema = - response.body.datasource.entities[repeated_table_name].schema + + const res = await config.api.datasource.fetchSchema({ + datasourceId: datasource._id!, + tablesFilter: [repeated_table_name], + }) + expect(res.datasource.entities![repeated_table_name].schema).toBeDefined() + const schema = res.datasource.entities![repeated_table_name].schema expect(Object.keys(schema).sort()).toEqual(["id", "val1"]) }) }) @@ -175,12 +155,11 @@ describe("mysql integrations", () => { await rawQuery(rawDatasource, createTableQuery) - const response = await makeRequest( - "post", - `/api/datasources/${datasource._id}/schema` - ) + const res = await config.api.datasource.fetchSchema({ + datasourceId: datasource._id!, + }) - const table = response.body.datasource.entities[tableName] + const table = res.datasource.entities![tableName] expect(table).toBeDefined() expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 32eb91ca62..9dd6a951b5 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -14,7 +14,7 @@ const config = setup.getConfig()! describe("postgres integrations", () => { let rawDatasource: Datasource let datasource: Datasource - let client: Knex + let client: Knex beforeAll(async () => { await config.init() From 16cacb3de777b56736d183b9daa388af4321cea0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 17 Jun 2024 15:48:58 +0100 Subject: [PATCH 4/6] Convert mysql.spec.ts away from rawQuery. --- .../server/src/integration-test/mysql.spec.ts | 115 ++++++------------ .../src/integration-test/postgres.spec.ts | 2 +- .../src/integrations/tests/utils/mssql.ts | 2 +- .../src/integrations/tests/utils/mysql.ts | 2 +- 4 files changed, 40 insertions(+), 81 deletions(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index dc4f79bba9..eb81b8920f 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -1,16 +1,12 @@ -import fetch from "node-fetch" import * as setup from "../api/routes/tests/utilities" import { Datasource, FieldType } from "@budibase/types" import { DatabaseName, getDatasource, - rawQuery, + knexClient, } from "../integrations/tests/utils" import { generator } from "@budibase/backend-core/tests" -import { tableForDatasource } from "../../src/tests/utilities/structures" import { Knex } from "knex" -// @ts-ignore -fetch.mockSearch() function uniqueTableName(length?: number): string { return generator @@ -21,21 +17,6 @@ function uniqueTableName(length?: number): string { const config = setup.getConfig()! -jest.mock("../websockets", () => ({ - clientAppSocket: jest.fn(), - gridAppSocket: jest.fn(), - initialise: jest.fn(), - builderSocket: { - emitTableUpdate: jest.fn(), - emitTableDeletion: jest.fn(), - emitDatasourceUpdate: jest.fn(), - emitDatasourceDeletion: jest.fn(), - emitScreenUpdate: jest.fn(), - emitAppMetadataUpdate: jest.fn(), - emitAppPublish: jest.fn(), - }, -})) - describe("mysql integrations", () => { let rawDatasource: Datasource let datasource: Datasource @@ -43,68 +24,40 @@ describe("mysql integrations", () => { beforeAll(async () => { await config.init() - rawDatasource = await getDatasource(DatabaseName.MYSQL) datasource = await config.api.datasource.create(rawDatasource) + client = await knexClient(rawDatasource) }) afterAll(config.end) - it("validate table schema", async () => { - // Creating a table so that `entities` is populated. - await config.api.table.save(tableForDatasource(datasource)) - const res = await config.api.datasource.get(datasource._id!) - expect(res).toEqual({ - config: { - database: expect.any(String), - host: datasource.config!.host, - password: "--secret-value--", - port: datasource.config!.port, - user: "root", - }, - plus: true, - source: "MYSQL", - type: "datasource_plus", - isSQL: true, - _id: expect.any(String), - _rev: expect.any(String), - createdAt: expect.any(String), - updatedAt: expect.any(String), - entities: expect.any(Object), - }) - }) - describe("Integration compatibility with mysql search_path", () => { - let datasource: Datasource, rawDatasource: Datasource + let datasource: Datasource + let rawDatasource: Datasource + let client: Knex const database = generator.guid() const database2 = generator.guid() beforeAll(async () => { rawDatasource = await getDatasource(DatabaseName.MYSQL) + client = await knexClient(rawDatasource) - await rawQuery(rawDatasource, `CREATE DATABASE \`${database}\`;`) - await rawQuery(rawDatasource, `CREATE DATABASE \`${database2}\`;`) + await client.raw(`CREATE DATABASE \`${database}\`;`) + await client.raw(`CREATE DATABASE \`${database2}\`;`) - const pathConfig: any = { - ...rawDatasource, - config: { - ...rawDatasource.config!, - database, - }, - } - datasource = await config.api.datasource.create(pathConfig) + rawDatasource.config!.database = database + datasource = await config.api.datasource.create(rawDatasource) }) afterAll(async () => { - await rawQuery(rawDatasource, `DROP DATABASE \`${database}\`;`) - await rawQuery(rawDatasource, `DROP DATABASE \`${database2}\`;`) + await client.raw(`DROP DATABASE \`${database}\`;`) + await client.raw(`DROP DATABASE \`${database2}\`;`) }) it("discovers tables from any schema in search path", async () => { - await rawQuery( - rawDatasource, - `CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);` - ) + await client.schema.createTable(`${database}.table1`, table => { + table.increments("id1").primary() + }) const res = await config.api.datasource.info(datasource) expect(res.tableNames).toBeDefined() expect(res.tableNames).toEqual(expect.arrayContaining(["table1"])) @@ -112,13 +65,19 @@ describe("mysql integrations", () => { it("does not mix columns from different tables", async () => { const repeated_table_name = "table_same_name" - await rawQuery( - rawDatasource, - `CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);` + await client.schema.createTable( + `${database}.${repeated_table_name}`, + table => { + table.increments("id").primary() + table.string("val1") + } ) - await rawQuery( - rawDatasource, - `CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);` + await client.schema.createTable( + `${database2}.${repeated_table_name}`, + table => { + table.increments("id2").primary() + table.string("val2") + } ) const res = await config.api.datasource.fetchSchema({ @@ -139,21 +98,21 @@ describe("mysql integrations", () => { }) afterEach(async () => { - await rawQuery(rawDatasource, `DROP TABLE IF EXISTS \`${tableName}\``) + await client.schema.dropTableIfExists(tableName) }) it("recognises enum columns as options", async () => { const enumColumnName = "status" - const createTableQuery = ` - CREATE TABLE \`${tableName}\` ( - \`order_id\` INT AUTO_INCREMENT PRIMARY KEY, - \`customer_name\` VARCHAR(100) NOT NULL, - \`${enumColumnName}\` ENUM('pending', 'processing', 'shipped', 'delivered', 'cancelled') - ); - ` - - await rawQuery(rawDatasource, createTableQuery) + await client.schema.createTable(tableName, table => { + table.increments("order_id").primary() + table.string("customer_name", 100).notNullable() + table.enum( + enumColumnName, + ["pending", "processing", "shipped", "delivered", "cancelled"], + { useNative: true, enumName: `${tableName}_${enumColumnName}` } + ) + }) const res = await config.api.datasource.fetchSchema({ datasourceId: datasource._id!, diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 9dd6a951b5..3e34ce097b 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -93,7 +93,7 @@ describe("postgres integrations", () => { describe("Integration compatibility with postgres search_path", () => { let rawDatasource: Datasource, datasource: Datasource, - client: Knex, + client: Knex, schema1: string, schema2: string diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index 3b3533eb7f..3f661f4914 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -7,7 +7,7 @@ import knex from "knex" let ports: Promise -export async function getDatasource(): Promise { +export async function GetDatasource(): Promise { if (!ports) { ports = startContainer( new GenericContainer("mcr.microsoft.com/mssql/server:2022-latest") diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts index cb6667bc56..1975e07730 100644 --- a/packages/server/src/integrations/tests/utils/mysql.ts +++ b/packages/server/src/integrations/tests/utils/mysql.ts @@ -88,7 +88,7 @@ export async function knexClient(ds: Datasource) { } return knex({ - client: "mysql", + client: "mysql2", connection: ds.config, }) } From c107ab937d2a9602f9c4805274c821c6fde47494 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 17 Jun 2024 16:15:51 +0100 Subject: [PATCH 5/6] Remove rawQuery entirely. --- .../routes/tests/queries/generic-sql.spec.ts | 96 ++++++++----------- .../server/src/integration-test/mysql.spec.ts | 3 +- .../src/integration-test/postgres.spec.ts | 14 ++- .../src/integrations/tests/utils/index.ts | 17 ---- .../src/integrations/tests/utils/mariadb.ts | 5 +- .../src/integrations/tests/utils/mssql.ts | 24 +---- .../src/integrations/tests/utils/mysql.ts | 21 +--- .../src/integrations/tests/utils/postgres.ts | 22 +---- 8 files changed, 57 insertions(+), 145 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index e8a38dcfaa..d44acb84d9 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -10,37 +10,11 @@ import * as setup from "../utilities" import { DatabaseName, getDatasource, - rawQuery, + knexClient, } from "../../../../integrations/tests/utils" import { Expectations } from "src/tests/utilities/api/base" import { events } from "@budibase/backend-core" - -const createTableSQL: Record = { - [SourceName.POSTGRES]: ` - CREATE TABLE test_table ( - id serial PRIMARY KEY, - name VARCHAR ( 50 ) NOT NULL, - birthday TIMESTAMP, - number INT - );`, - [SourceName.MYSQL]: ` - CREATE TABLE test_table ( - id INT AUTO_INCREMENT PRIMARY KEY, - name VARCHAR(50) NOT NULL, - birthday TIMESTAMP, - number INT - );`, - [SourceName.SQL_SERVER]: ` - CREATE TABLE test_table ( - id INT IDENTITY(1,1) PRIMARY KEY, - name NVARCHAR(50) NOT NULL, - birthday DATETIME, - number INT - );`, -} - -const insertSQL = `INSERT INTO test_table (name) VALUES ('one'), ('two'), ('three'), ('four'), ('five')` -const dropTableSQL = `DROP TABLE test_table;` +import { Knex } from "knex" describe.each( [ @@ -53,6 +27,7 @@ describe.each( const config = setup.getConfig() let rawDatasource: Datasource let datasource: Datasource + let client: Knex async function createQuery( query: Partial, @@ -82,21 +57,34 @@ describe.each( rawDatasource = await dsProvider datasource = await config.api.datasource.create(rawDatasource) - // The Datasource API does not return the password, but we need - // it later to connect to the underlying database, so we fill it - // back in here. + // The Datasource API doesn ot return the password, but we need it later to + // connect to the underlying database, so we fill it back in here. datasource.config!.password = rawDatasource.config!.password - await rawQuery(datasource, createTableSQL[datasource.source]) - await rawQuery(datasource, insertSQL) + client = await knexClient(rawDatasource) + + await client.schema.createTable("test_table", table => { + table.increments("id").primary() + table.string("name") + table.timestamp("birthday") + table.integer("number") + }) + + await client("test_table").insert([ + { name: "one" }, + { name: "two" }, + { name: "three" }, + { name: "four" }, + { name: "five" }, + ]) jest.clearAllMocks() }) afterEach(async () => { const ds = await config.api.datasource.get(datasource._id!) - config.api.datasource.delete(ds) - await rawQuery(datasource, dropTableSQL) + await config.api.datasource.delete(ds) + await client.schema.dropTable("test_table") }) afterAll(async () => { @@ -207,7 +195,7 @@ describe.each( }, }) - await config.publish() + await config.api.application.publish(config.getAppId()) const prodQuery = await config.api.query.getProd(query._id!) expect(prodQuery._id).toEqual(query._id) @@ -429,11 +417,11 @@ describe.each( }, ]) - const rows = await rawQuery( - datasource, - "SELECT * FROM test_table WHERE name = 'baz'" - ) + const rows = await client("test_table").where({ name: "baz" }).select() expect(rows).toHaveLength(1) + for (const row of rows) { + expect(row).toMatchObject({ name: "baz" }) + } }) it("should not allow handlebars as parameters", async () => { @@ -490,11 +478,14 @@ describe.each( expect(result.data).toEqual([{ created: true }]) - const rows = await rawQuery( - datasource, - `SELECT * FROM test_table WHERE birthday = '${date.toISOString()}'` - ) + const rows = await client("test_table") + .where({ birthday: datetimeStr }) + .select() expect(rows).toHaveLength(1) + + for (const row of rows) { + expect(new Date(row.birthday)).toEqual(date) + } } ) @@ -522,10 +513,9 @@ describe.each( expect(result.data).toEqual([{ created: true }]) - const rows = await rawQuery( - datasource, - `SELECT * FROM test_table WHERE name = '${notDateStr}'` - ) + const rows = await client("test_table") + .where({ name: notDateStr }) + .select() expect(rows).toHaveLength(1) } ) @@ -660,10 +650,7 @@ describe.each( }, ]) - const rows = await rawQuery( - datasource, - "SELECT * FROM test_table WHERE id = 1" - ) + const rows = await client("test_table").where({ id: 1 }).select() expect(rows).toEqual([ { id: 1, name: "foo", birthday: null, number: null }, ]) @@ -731,10 +718,7 @@ describe.each( }, ]) - const rows = await rawQuery( - datasource, - "SELECT * FROM test_table WHERE id = 1" - ) + const rows = await client("test_table").where({ id: 1 }).select() expect(rows).toHaveLength(0) }) }) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index eb81b8920f..eb6c840abc 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -18,13 +18,12 @@ function uniqueTableName(length?: number): string { const config = setup.getConfig()! describe("mysql integrations", () => { - let rawDatasource: Datasource let datasource: Datasource let client: Knex beforeAll(async () => { await config.init() - rawDatasource = await getDatasource(DatabaseName.MYSQL) + const rawDatasource = await getDatasource(DatabaseName.MYSQL) datasource = await config.api.datasource.create(rawDatasource) client = await knexClient(rawDatasource) }) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 3e34ce097b..f2e0382deb 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -12,13 +12,12 @@ import { Knex } from "knex" const config = setup.getConfig()! describe("postgres integrations", () => { - let rawDatasource: Datasource let datasource: Datasource let client: Knex beforeAll(async () => { await config.init() - rawDatasource = await getDatasource(DatabaseName.POSTGRES) + const rawDatasource = await getDatasource(DatabaseName.POSTGRES) datasource = await config.api.datasource.create(rawDatasource) client = await knexClient(rawDatasource) }) @@ -91,17 +90,16 @@ describe("postgres integrations", () => { }) describe("Integration compatibility with postgres search_path", () => { - let rawDatasource: Datasource, - datasource: Datasource, - client: Knex, - schema1: string, - schema2: string + let datasource: Datasource + let client: Knex + let schema1: string + let schema2: string beforeEach(async () => { schema1 = generator.guid().replaceAll("-", "") schema2 = generator.guid().replaceAll("-", "") - rawDatasource = await getDatasource(DatabaseName.POSTGRES) + const rawDatasource = await getDatasource(DatabaseName.POSTGRES) client = await knexClient(rawDatasource) await client.schema.createSchema(schema1) diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index fd7d2373bb..b888f1adc1 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -48,23 +48,6 @@ export async function getDatasources( return Promise.all(sourceNames.map(sourceName => providers[sourceName]())) } -export async function rawQuery(ds: Datasource, sql: string): Promise { - switch (ds.source) { - case SourceName.POSTGRES: { - return postgres.rawQuery(ds, sql) - } - case SourceName.MYSQL: { - return mysql.rawQuery(ds, sql) - } - case SourceName.SQL_SERVER: { - return mssql.rawQuery(ds, sql) - } - default: { - throw new Error(`Unsupported source: ${ds.source}`) - } - } -} - export async function knexClient(ds: Datasource) { switch (ds.source) { case SourceName.POSTGRES: { diff --git a/packages/server/src/integrations/tests/utils/mariadb.ts b/packages/server/src/integrations/tests/utils/mariadb.ts index c4dd4cf43b..3a90b554ee 100644 --- a/packages/server/src/integrations/tests/utils/mariadb.ts +++ b/packages/server/src/integrations/tests/utils/mariadb.ts @@ -1,9 +1,9 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" -import { rawQuery } from "./mysql" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." +import { knexClient } from "./mysql" let ports: Promise @@ -55,7 +55,8 @@ export async function getDatasource(): Promise { } const database = generator.guid().replaceAll("-", "") - await rawQuery(datasource, `CREATE DATABASE \`${database}\``) + const client = await knexClient(datasource) + await client.raw(`CREATE DATABASE \`${database}\``) datasource.config.database = database return datasource } diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index 3f661f4914..a4bd5448f4 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -1,13 +1,12 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" -import mssql from "mssql" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import knex from "knex" let ports: Promise -export async function GetDatasource(): Promise { +export async function getDatasource(): Promise { if (!ports) { ports = startContainer( new GenericContainer("mcr.microsoft.com/mssql/server:2022-latest") @@ -50,30 +49,13 @@ export async function GetDatasource(): Promise { } const database = generator.guid().replaceAll("-", "") - await rawQuery(datasource, `CREATE DATABASE "${database}"`) + const client = await knexClient(datasource) + await client.raw(`CREATE DATABASE "${database}"`) datasource.config!.database = database return datasource } -export async function rawQuery(ds: Datasource, sql: string) { - if (!ds.config) { - throw new Error("Datasource config is missing") - } - if (ds.source !== SourceName.SQL_SERVER) { - throw new Error("Datasource source is not SQL Server") - } - - const pool = new mssql.ConnectionPool(ds.config! as mssql.config) - const client = await pool.connect() - try { - const { recordset } = await client.query(sql) - return recordset - } finally { - await pool.close() - } -} - export async function knexClient(ds: Datasource) { if (!ds.config) { throw new Error("Datasource config is missing") diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts index 1975e07730..c35be0689e 100644 --- a/packages/server/src/integrations/tests/utils/mysql.ts +++ b/packages/server/src/integrations/tests/utils/mysql.ts @@ -1,7 +1,6 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" -import mysql from "mysql2/promise" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import knex from "knex" @@ -57,28 +56,12 @@ export async function getDatasource(): Promise { } const database = generator.guid().replaceAll("-", "") - await rawQuery(datasource, `CREATE DATABASE \`${database}\``) + const client = await knexClient(datasource) + await client.raw(`CREATE DATABASE \`${database}\``) datasource.config!.database = database return datasource } -export async function rawQuery(ds: Datasource, sql: string) { - if (!ds.config) { - throw new Error("Datasource config is missing") - } - if (ds.source !== SourceName.MYSQL) { - throw new Error("Datasource source is not MySQL") - } - - const connection = await mysql.createConnection(ds.config) - try { - const [rows] = await connection.query(sql) - return rows - } finally { - connection.end() - } -} - export async function knexClient(ds: Datasource) { if (!ds.config) { throw new Error("Datasource config is missing") diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index 20b56331c1..74f5722737 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -1,6 +1,5 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait } from "testcontainers" -import pg from "pg" import { generator, testContainerUtils } from "@budibase/backend-core/tests" import { startContainer } from "." import knex from "knex" @@ -44,30 +43,13 @@ export async function getDatasource(): Promise { } const database = generator.guid().replaceAll("-", "") - await rawQuery(datasource, `CREATE DATABASE "${database}"`) + const client = await knexClient(datasource) + await client.raw(`CREATE DATABASE "${database}"`) datasource.config!.database = database return datasource } -export async function rawQuery(ds: Datasource, sql: string) { - if (!ds.config) { - throw new Error("Datasource config is missing") - } - if (ds.source !== SourceName.POSTGRES) { - throw new Error("Datasource source is not Postgres") - } - - const client = new pg.Client(ds.config) - await client.connect() - try { - const { rows } = await client.query(sql) - return rows - } finally { - await client.end() - } -} - export async function knexClient(ds: Datasource) { if (!ds.config) { throw new Error("Datasource config is missing") From 903c3cf84d49f0001a0ba1a49b43de02eae624d5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 17 Jun 2024 16:17:00 +0100 Subject: [PATCH 6/6] Fix flake in generic-sql.spec.ts. --- .../server/src/api/routes/tests/queries/generic-sql.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index d44acb84d9..b060a099d8 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -63,6 +63,7 @@ describe.each( client = await knexClient(rawDatasource) + await client.schema.dropTableIfExists("test_table") await client.schema.createTable("test_table", table => { table.increments("id").primary() table.string("name") @@ -84,7 +85,6 @@ describe.each( afterEach(async () => { const ds = await config.api.datasource.get(datasource._id!) await config.api.datasource.delete(ds) - await client.schema.dropTable("test_table") }) afterAll(async () => {