Basic Postgres and Mongo query testcases.

packages/server/src/api/routes/tests/queries/mongodb.spec.ts

@@ -0,0 +1,109 @@
+import { Datasource, Query } from "@budibase/types"
+import * as setup from "../utilities"
+import { databaseTestProviders } from "../../../../integrations/tests/utils"
+import { MongoClient } from "mongodb"
+
+jest.unmock("mongodb")
+jest.setTimeout(3000)
+
+describe("/queries", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let datasource: Datasource
+
+  async function createQuery(query: Partial<Query>): Promise<Query> {
+    const defaultQuery: Query = {
+      datasourceId: datasource._id!,
+      name: "New Query",
+      parameters: [],
+      fields: {},
+      schema: {},
+      queryVerb: "read",
+      transformer: "return data",
+      readable: true,
+    }
+
+    const res = await request
+      .post(`/api/queries`)
+      .set(config.defaultHeaders())
+      .send({ ...defaultQuery, ...query })
+      .expect("Content-Type", /json/)
+
+    if (res.status !== 200) {
+      throw new Error(JSON.stringify(res.body))
+    }
+
+    return res.body as Query
+  }
+
+  afterAll(async () => {
+    await databaseTestProviders.mongodb.stop()
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+    datasource = await config.api.datasource.create(
+      await databaseTestProviders.mongodb.datasource()
+    )
+  })
+
+  beforeEach(async () => {
+    const ds = await databaseTestProviders.mongodb.datasource()
+    const client = new MongoClient(ds.config!.connectionString)
+    await client.connect()
+
+    const db = client.db(ds.config!.db)
+    const collection = db.collection("test_table")
+    await collection.insertMany([
+      { name: "one" },
+      { name: "two" },
+      { name: "three" },
+      { name: "four" },
+      { name: "five" },
+    ])
+    await client.close()
+  })
+
+  afterEach(async () => {
+    const ds = await databaseTestProviders.mongodb.datasource()
+    const client = new MongoClient(ds.config!.connectionString)
+    await client.connect()
+    const db = client.db(ds.config!.db)
+    await db.collection("test_table").drop()
+    await client.close()
+  })
+
+  it("should execute a query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: "{}",
+        extra: {
+          actionType: "count",
+          collection: "test_table",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([{ value: 5 }])
+  })
+
+  it("should execute a query with a transformer", async () => {
+    const query = await createQuery({
+      fields: {
+        json: "{}",
+        extra: {
+          actionType: "count",
+          collection: "test_table",
+        },
+      },
+      transformer: "return data + 1",
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([{ value: 6 }])
+  })
+})

packages/server/src/api/routes/tests/queries/postgres.spec.ts

@@ -0,0 +1,139 @@
+import { Datasource, Query } from "@budibase/types"
+import * as setup from "../utilities"
+import { databaseTestProviders } from "../../../../integrations/tests/utils"
+import { Client } from "pg"
+
+jest.unmock("pg")
+
+const createTableSQL = `
+CREATE TABLE test_table (
+    id serial PRIMARY KEY,
+    name VARCHAR ( 50 ) NOT NULL
+);
+`
+
+const insertSQL = `
+INSERT INTO test_table (name) VALUES ('one');
+INSERT INTO test_table (name) VALUES ('two');
+INSERT INTO test_table (name) VALUES ('three');
+INSERT INTO test_table (name) VALUES ('four');
+INSERT INTO test_table (name) VALUES ('five');
+`
+
+const dropTableSQL = `
+DROP TABLE test_table;
+`
+
+describe("/queries", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let datasource: Datasource
+
+  async function createQuery(query: Partial<Query>): Promise<Query> {
+    const defaultQuery: Query = {
+      datasourceId: datasource._id!,
+      name: "New Query",
+      parameters: [],
+      fields: {},
+      schema: {},
+      queryVerb: "read",
+      transformer: "return data",
+      readable: true,
+    }
+
+    const res = await request
+      .post(`/api/queries`)
+      .set(config.defaultHeaders())
+      .send({ ...defaultQuery, ...query })
+      .expect("Content-Type", /json/)
+
+    if (res.status !== 200) {
+      throw new Error(JSON.stringify(res.body))
+    }
+
+    return res.body as Query
+  }
+
+  afterAll(async () => {
+    await databaseTestProviders.postgres.stop()
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+    datasource = await config.api.datasource.create(
+      await databaseTestProviders.postgres.datasource()
+    )
+  })
+
+  beforeEach(async () => {
+    const ds = await databaseTestProviders.postgres.datasource()
+    const client = new Client(ds.config!)
+    await client.connect()
+    await client.query(createTableSQL)
+    await client.query(insertSQL)
+    await client.end()
+  })
+
+  afterEach(async () => {
+    const ds = await databaseTestProviders.postgres.datasource()
+    const client = new Client(ds.config!)
+    await client.connect()
+    await client.query(dropTableSQL)
+    await client.end()
+  })
+
+  it("should execute a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table ORDER BY id",
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 1,
+        name: "one",
+      },
+      {
+        id: 2,
+        name: "two",
+      },
+      {
+        id: 3,
+        name: "three",
+      },
+      {
+        id: 4,
+        name: "four",
+      },
+      {
+        id: 5,
+        name: "five",
+      },
+    ])
+  })
+
+  it("should be able to transform a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table WHERE id = 1",
+      },
+      transformer: `
+        data[0].id = data[0].id + 1;
+        return data;
+      `,
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 2,
+        name: "one",
+      },
+    ])
+  })
+})

packages/server/src/api/routes/tests/queries/query.seq.spec.ts

@@ -16,9 +16,9 @@ jest.mock("@budibase/backend-core", () => {
     },
   }
 })
-import * as setup from "./utilities"
-import { checkBuilderEndpoint } from "./utilities/TestFunctions"
-import { checkCacheForDynamicVariable } from "../../../threads/utils"
+import * as setup from "../utilities"
+import { checkBuilderEndpoint } from "../utilities/TestFunctions"
+import { checkCacheForDynamicVariable } from "../../../../threads/utils"
 
 const { basicQuery, basicDatasource } = setup.structures
 import { events, db as dbCore } from "@budibase/backend-core"

packages/server/src/api/routes/tests/row.spec.ts

@@ -12,7 +12,6 @@ import {
   FieldTypeSubtypes,
   FormulaType,
   INTERNAL_TABLE_SOURCE_ID,
-  MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
   RelationshipType,
@@ -53,7 +52,7 @@ describe.each([
 
   afterAll(async () => {
     if (dsProvider) {
-      await dsProvider.stopContainer()
+      await dsProvider.stop()
     }
     setup.afterAll()
   })
@@ -63,7 +62,7 @@ describe.each([
 
     if (dsProvider) {
       await config.createDatasource({
-        datasource: await dsProvider.getDsConfig(),
+        datasource: await dsProvider.datasource(),
       })
     }
   })
@@ -117,16 +116,6 @@ describe.each([
     return total
   }
 
-  const getQueryUsage = async () => {
-    const { total } = await config.doInContext(null, () =>
-      quotas.getCurrentUsageValues(
-        QuotaUsageType.MONTHLY,
-        MonthlyQuotaName.QUERIES
-      )
-    )
-    return total
-  }
-
   const assertRowUsage = async (expected: number) => {
     const usage = await getRowUsage()
     expect(usage).toBe(expected)
@@ -162,7 +151,6 @@ describe.each([
   describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await request
         .post(`/api/${tableId}/rows`)
@@ -180,7 +168,6 @@ describe.each([
 
     it("Increment row autoId per create row request", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const tableConfig = generateTableConfig()
       const newTable = await createTable(
@@ -231,7 +218,6 @@ describe.each([
     it("updates a row successfully", async () => {
       const existing = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.save(tableId, {
         _id: existing._id,
@@ -246,7 +232,6 @@ describe.each([
 
     it("should load a row", async () => {
       const existing = await config.createRow()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.get(tableId, existing._id!)
 
@@ -268,7 +253,6 @@ describe.each([
       }
       const firstRow = await config.createRow({ tableId })
       await config.createRow(newRow)
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.fetch(tableId)
 
@@ -279,7 +263,6 @@ describe.each([
 
     it("load should return 404 when row does not exist", async () => {
       await config.createRow()
-      const queryUsage = await getQueryUsage()
 
       await config.api.row.get(tableId, "1234567", {
         expectStatus: 404,
@@ -530,7 +513,6 @@ describe.each([
       const existing = await config.createRow()
 
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const row = await config.api.row.patch(table._id!, {
         _id: existing._id!,
@@ -552,7 +534,6 @@ describe.each([
     it("should throw an error when given improper types", async () => {
       const existing = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       await config.api.row.patch(
         table._id!,
@@ -650,7 +631,6 @@ describe.each([
     it("should be able to delete a row", async () => {
       const createdRow = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, [createdRow])
       expect(res.body[0]._id).toEqual(createdRow._id)
@@ -666,7 +646,6 @@ describe.each([
 
     it("should return no errors on valid row", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.validate(table._id!, { name: "ivan" })
 
@@ -677,7 +656,6 @@ describe.each([
 
     it("should errors on invalid row", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.validate(table._id!, { name: 1 })
 
@@ -703,7 +681,6 @@ describe.each([
       const row1 = await config.createRow()
       const row2 = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, [row1, row2])
 
@@ -719,7 +696,6 @@ describe.each([
         config.createRow(),
       ])
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, [
         row1,
@@ -735,7 +711,6 @@ describe.each([
     it("should accept a valid row object and delete the row", async () => {
       const row1 = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, row1)
 
@@ -746,7 +721,6 @@ describe.each([
 
     it("Should ignore malformed/invalid delete requests", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(
         table._id!,
@@ -782,7 +756,6 @@ describe.each([
       it("should be able to fetch tables contents via 'view'", async () => {
         const row = await config.createRow()
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         const res = await config.api.legacyView.get(table._id!)
         expect(res.body.length).toEqual(1)
@@ -792,7 +765,6 @@ describe.each([
 
       it("should throw an error if view doesn't exist", async () => {
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         await config.api.legacyView.get("derp", { expectStatus: 404 })
 
@@ -808,7 +780,6 @@ describe.each([
         })
         const row = await config.createRow()
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         const res = await config.api.legacyView.get(view.name)
         expect(res.body.length).toEqual(1)
@@ -864,7 +835,6 @@ describe.each([
         }
       )
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       // test basic enrichment
       const resBasic = await config.api.row.get(
@@ -1100,7 +1070,6 @@ describe.each([
 
         const createdRow = await config.createRow()
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         await config.api.row.delete(view.id, [createdRow])
 
@@ -1127,7 +1096,6 @@ describe.each([
           config.createRow(),
         ])
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         await config.api.row.delete(view.id, [rows[0], rows[2]])
 

packages/server/src/integration-test/postgres.spec.ts

@@ -41,12 +41,12 @@ describe("postgres integrations", () => {
     makeRequest = generateMakeRequest(apiKey, true)
 
     postgresDatasource = await config.api.datasource.create(
-      await databaseTestProviders.postgres.getDsConfig()
+      await databaseTestProviders.postgres.datasource()
     )
   })
 
   afterAll(async () => {
-    await databaseTestProviders.postgres.stopContainer()
+    await databaseTestProviders.postgres.stop()
   })
 
   beforeEach(async () => {
@@ -1041,14 +1041,14 @@ describe("postgres integrations", () => {
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
       const response = await config.api.datasource.verify({
-        datasource: await databaseTestProviders.postgres.getDsConfig(),
+        datasource: await databaseTestProviders.postgres.datasource(),
       })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const dbConfig = await databaseTestProviders.postgres.getDsConfig()
+      const dbConfig = await databaseTestProviders.postgres.datasource()
       const response = await config.api.datasource.verify({
         datasource: {
           ...dbConfig,
@@ -1082,7 +1082,7 @@ describe("postgres integrations", () => {
 
     beforeEach(async () => {
       client = new Client(
-        (await databaseTestProviders.postgres.getDsConfig()).config!
+        (await databaseTestProviders.postgres.datasource()).config!
       )
       await client.connect()
     })
@@ -1125,7 +1125,7 @@ describe("postgres integrations", () => {
       schema2 = "test-2"
 
     beforeAll(async () => {
-      const dsConfig = await databaseTestProviders.postgres.getDsConfig()
+      const dsConfig = await databaseTestProviders.postgres.datasource()
       const dbConfig = dsConfig.config!
 
       client = new Client(dbConfig)

packages/server/src/integrations/tests/utils/index.ts

@@ -1,14 +1,16 @@
 jest.unmock("pg")
 
 import { Datasource } from "@budibase/types"
-import * as pg from "./postgres"
+import * as postgres from "./postgres"
+import * as mongodb from "./mongodb"
+import { StartedTestContainer } from "testcontainers"
 
 jest.setTimeout(30000)
 
-export interface DatabasePlusTestProvider {
-  getDsConfig(): Promise<Datasource>
+export interface DatabaseProvider {
+  start(): Promise<StartedTestContainer>
+  stop(): Promise<void>
+  datasource(): Promise<Datasource>
 }
 
-export const databaseTestProviders = {
-  postgres: pg,
-}
+export const databaseTestProviders = { postgres, mongodb }

packages/server/src/integrations/tests/utils/mongodb.ts

@@ -0,0 +1,44 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+
+let container: StartedTestContainer | undefined
+
+export async function start(): Promise<StartedTestContainer> {
+  if (!container) {
+    container = await new GenericContainer("mongo:7.0-jammy")
+      .withExposedPorts(27017)
+      .withEnvironment({
+        MONGO_INITDB_ROOT_USERNAME: "mongo",
+        MONGO_INITDB_ROOT_PASSWORD: "password",
+      })
+      .withWaitStrategy(
+        Wait.forSuccessfulCommand(
+          `mongosh --eval "db.version()"`
+        ).withStartupTimeout(10000)
+      )
+      .start()
+  }
+  return container
+}
+
+export async function datasource(): Promise<Datasource> {
+  const container = await start()
+  const host = container.getHost()
+  const port = container.getMappedPort(27017)
+  return {
+    type: "datasource",
+    source: SourceName.MONGODB,
+    plus: false,
+    config: {
+      connectionString: `mongodb://mongo:password@${host}:${port}`,
+      db: "mongo",
+    },
+  }
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}

packages/server/src/integrations/tests/utils/postgres.ts

@@ -3,45 +3,45 @@ import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
 
 let container: StartedTestContainer | undefined
 
-export async function getDsConfig(): Promise<Datasource> {
-  try {
-    if (!container) {
-      container = await new GenericContainer("postgres:16.1-bullseye")
-        .withExposedPorts(5432)
-        .withEnvironment({ POSTGRES_PASSWORD: "password" })
-        .withWaitStrategy(
-          Wait.forLogMessage(
-            "database system is ready to accept connections",
-            2
-          )
-        )
-        .start()
-    }
-    const host = container.getHost()
-    const port = container.getMappedPort(5432)
+export async function start(): Promise<StartedTestContainer> {
+  if (!container) {
+    container = await new GenericContainer("postgres:16.1-bullseye")
+      .withExposedPorts(5432)
+      .withEnvironment({ POSTGRES_PASSWORD: "password" })
+      .withWaitStrategy(
+        Wait.forSuccessfulCommand(
+          "pg_isready -h localhost -p 5432"
+        ).withStartupTimeout(10000)
+      )
+      .start()
+  }
+  return container
+}
 
-    return {
-      type: "datasource_plus",
-      source: SourceName.POSTGRES,
-      plus: true,
-      config: {
-        host,
-        port,
-        database: "postgres",
-        user: "postgres",
-        password: "password",
-        schema: "public",
-        ssl: false,
-        rejectUnauthorized: false,
-        ca: false,
-      },
-    }
-  } catch (err) {
-    throw new Error("**UNABLE TO CREATE TO POSTGRES CONTAINER**")
+export async function datasource(): Promise<Datasource> {
+  const container = await start()
+  const host = container.getHost()
+  const port = container.getMappedPort(5432)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.POSTGRES,
+    plus: true,
+    config: {
+      host,
+      port,
+      database: "postgres",
+      user: "postgres",
+      password: "password",
+      schema: "public",
+      ssl: false,
+      rejectUnauthorized: false,
+      ca: false,
+    },
   }
 }
 
-export async function stopContainer() {
+export async function stop() {
   if (container) {
     await container.stop()
     container = undefined

packages/server/src/tests/utilities/api/index.ts

@@ -10,6 +10,7 @@ import { ApplicationAPI } from "./application"
 import { BackupAPI } from "./backup"
 import { AttachmentAPI } from "./attachment"
 import { UserAPI } from "./user"
+import { QueryAPI } from "./query"
 
 export default class API {
   table: TableAPI
@@ -23,6 +24,7 @@ export default class API {
   backup: BackupAPI
   attachment: AttachmentAPI
   user: UserAPI
+  query: QueryAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -36,5 +38,6 @@ export default class API {
     this.backup = new BackupAPI(config)
     this.attachment = new AttachmentAPI(config)
     this.user = new UserAPI(config)
+    this.query = new QueryAPI(config)
   }
 }

packages/server/src/tests/utilities/api/query.ts

@@ -0,0 +1,36 @@
+import TestConfiguration from "../TestConfiguration"
+import { Query } from "@budibase/types"
+import { TestAPI } from "./base"
+
+export class QueryAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  create = async (body: Query): Promise<Query> => {
+    const res = await this.request
+      .post(`/api/queries`)
+      .set(this.config.defaultHeaders())
+      .send(body)
+      .expect("Content-Type", /json/)
+
+    if (res.status !== 200) {
+      throw new Error(JSON.stringify(res.body))
+    }
+
+    return res.body as Query
+  }
+
+  execute = async (queryId: string): Promise<{ data: any }> => {
+    const res = await this.request
+      .post(`/api/v2/queries/${queryId}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (res.status !== 200) {
+      throw new Error(JSON.stringify(res.body))
+    }
+
+    return res.body
+  }
+}