Rebuild table schema when adding new column to get externalType (#13165)

* Rebuild table schema when adding new column to get externalType

* Added MySQL integration test suite

* Add test for emitting datasource on save new column

* Update packages/server/src/integration-test/mysql.spec.ts

Co-authored-by: Sam Rose <hello@samwho.dev>

* remove duplicate tests

* Use UUID

* update account portal

---------

Co-authored-by: Sam Rose <hello@samwho.dev>

packages/account-portal

@@ -1 +1 @@
-Subproject commit 19f7a5829f4d23cbc694136e45d94482a59a475a
+Subproject commit 806b6fd5c11c284ebf4a01627d75db939f0f8152

packages/builder/src/stores/builder/tables.js

@@ -147,6 +147,12 @@ export function createTablesStore() {
     if (indexes) {
       draft.indexes = indexes
     }
+    // Add object to indicate if column is being added
+    if (draft.schema[field.name] === undefined) {
+      draft._add = {
+        name: field.name,
+      }
+    }
     draft.schema = {
       ...draft.schema,
       [field.name]: cloneDeep(field),

packages/server/src/api/controllers/table/external.ts

@@ -28,6 +28,7 @@ function getDatasourceId(table: Table) {
 export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   const inputs = ctx.request.body
   const renaming = inputs?._rename
+  const adding = inputs?._add
   // can't do this right now
   delete inputs.rows
   const tableId = ctx.request.body._id
@@ -40,7 +41,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
     const { datasource, table } = await sdk.tables.external.save(
       datasourceId!,
       inputs,
-      { tableId, renaming }
+      { tableId, renaming, adding }
     )
     builderSocket?.emitDatasourceUpdate(ctx, datasource)
     return table

packages/server/src/integration-test/mysql.spec.ts

@@ -0,0 +1,309 @@
+import fetch from "node-fetch"
+import {
+  generateMakeRequest,
+  MakeRequestResponse,
+} from "../api/routes/public/tests/utils"
+import { v4 as uuidv4 } from "uuid"
+import * as setup from "../api/routes/tests/utilities"
+import {
+  Datasource,
+  FieldType,
+  Table,
+  TableRequest,
+  TableSourceType,
+} from "@budibase/types"
+import _ from "lodash"
+import { databaseTestProviders } from "../integrations/tests/utils"
+import mysql from "mysql2/promise"
+import { builderSocket } from "../websockets"
+// @ts-ignore
+fetch.mockSearch()
+
+const config = setup.getConfig()!
+
+jest.unmock("mysql2/promise")
+jest.mock("../websockets", () => ({
+  clientAppSocket: jest.fn(),
+  gridAppSocket: jest.fn(),
+  initialise: jest.fn(),
+  builderSocket: {
+    emitTableUpdate: jest.fn(),
+    emitTableDeletion: jest.fn(),
+    emitDatasourceUpdate: jest.fn(),
+    emitDatasourceDeletion: jest.fn(),
+    emitScreenUpdate: jest.fn(),
+    emitAppMetadataUpdate: jest.fn(),
+    emitAppPublish: jest.fn(),
+  },
+}))
+
+describe("mysql integrations", () => {
+  let makeRequest: MakeRequestResponse,
+    mysqlDatasource: Datasource,
+    primaryMySqlTable: Table
+
+  beforeAll(async () => {
+    await config.init()
+    const apiKey = await config.generateApiKey()
+
+    makeRequest = generateMakeRequest(apiKey, true)
+
+    mysqlDatasource = await config.api.datasource.create(
+      await databaseTestProviders.mysql.datasource()
+    )
+  })
+
+  afterAll(async () => {
+    await databaseTestProviders.mysql.stop()
+  })
+
+  beforeEach(async () => {
+    primaryMySqlTable = await config.createTable({
+      name: uuidv4(),
+      type: "table",
+      primary: ["id"],
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          autocolumn: true,
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
+        description: {
+          name: "description",
+          type: FieldType.STRING,
+        },
+        value: {
+          name: "value",
+          type: FieldType.NUMBER,
+        },
+      },
+      sourceId: mysqlDatasource._id,
+      sourceType: TableSourceType.EXTERNAL,
+    })
+  })
+
+  afterAll(config.end)
+
+  it("validate table schema", async () => {
+    const res = await makeRequest(
+      "get",
+      `/api/datasources/${mysqlDatasource._id}`
+    )
+
+    expect(res.status).toBe(200)
+    expect(res.body).toEqual({
+      config: {
+        database: "mysql",
+        host: mysqlDatasource.config!.host,
+        password: "--secret-value--",
+        port: mysqlDatasource.config!.port,
+        user: "root",
+      },
+      plus: true,
+      source: "MYSQL",
+      type: "datasource_plus",
+      _id: expect.any(String),
+      _rev: expect.any(String),
+      createdAt: expect.any(String),
+      updatedAt: expect.any(String),
+      entities: expect.any(Object),
+    })
+  })
+
+  describe("POST /api/datasources/verify", () => {
+    it("should be able to verify the connection", async () => {
+      const response = await config.api.datasource.verify({
+        datasource: await databaseTestProviders.mysql.datasource(),
+      })
+      expect(response.status).toBe(200)
+      expect(response.body.connected).toBe(true)
+    })
+
+    it("should state an invalid datasource cannot connect", async () => {
+      const dbConfig = await databaseTestProviders.mysql.datasource()
+      const response = await config.api.datasource.verify({
+        datasource: {
+          ...dbConfig,
+          config: {
+            ...dbConfig.config,
+            password: "wrongpassword",
+          },
+        },
+      })
+
+      expect(response.status).toBe(200)
+      expect(response.body.connected).toBe(false)
+      expect(response.body.error).toBeDefined()
+    })
+  })
+
+  describe("POST /api/datasources/info", () => {
+    it("should fetch information about mysql datasource", async () => {
+      const primaryName = primaryMySqlTable.name
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: mysqlDatasource,
+      })
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1)
+    })
+  })
+
+  describe("Integration compatibility with mysql search_path", () => {
+    let client: mysql.Connection, pathDatasource: Datasource
+    const database = "test1"
+    const database2 = "test-2"
+
+    beforeAll(async () => {
+      const dsConfig = await databaseTestProviders.mysql.datasource()
+      const dbConfig = dsConfig.config!
+
+      client = await mysql.createConnection(dbConfig)
+      await client.query(`CREATE DATABASE \`${database}\`;`)
+      await client.query(`CREATE DATABASE \`${database2}\`;`)
+
+      const pathConfig: any = {
+        ...dsConfig,
+        config: {
+          ...dbConfig,
+          database,
+        },
+      }
+      pathDatasource = await config.api.datasource.create(pathConfig)
+    })
+
+    afterAll(async () => {
+      await client.query(`DROP DATABASE \`${database}\`;`)
+      await client.query(`DROP DATABASE \`${database2}\`;`)
+      await client.end()
+    })
+
+    it("discovers tables from any schema in search path", async () => {
+      await client.query(
+        `CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);`
+      )
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: pathDatasource,
+      })
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames).toEqual(
+        expect.arrayContaining(["table1"])
+      )
+    })
+
+    it("does not mix columns from different tables", async () => {
+      const repeated_table_name = "table_same_name"
+      await client.query(
+        `CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
+      )
+      await client.query(
+        `CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
+      )
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${pathDatasource._id}/schema`,
+        {
+          tablesFilter: [repeated_table_name],
+        }
+      )
+      expect(response.status).toBe(200)
+      expect(
+        response.body.datasource.entities[repeated_table_name].schema
+      ).toBeDefined()
+      const schema =
+        response.body.datasource.entities[repeated_table_name].schema
+      expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
+    })
+  })
+
+  describe("POST /api/tables/", () => {
+    let client: mysql.Connection
+    const emitDatasourceUpdateMock = jest.fn()
+
+    beforeEach(async () => {
+      client = await mysql.createConnection(
+        (
+          await databaseTestProviders.mysql.datasource()
+        ).config!
+      )
+    })
+
+    afterEach(async () => {
+      await client.end()
+    })
+
+    it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => {
+      mysqlDatasource = (
+        await makeRequest(
+          "post",
+          `/api/datasources/${mysqlDatasource._id}/schema`
+        )
+      ).body.datasource
+
+      const addColumnToTable: TableRequest = {
+        type: "table",
+        sourceType: TableSourceType.EXTERNAL,
+        name: "table",
+        sourceId: mysqlDatasource._id!,
+        primary: ["id"],
+        schema: {
+          id: {
+            type: FieldType.AUTO,
+            name: "id",
+            autocolumn: true,
+          },
+          new_column: {
+            type: FieldType.NUMBER,
+            name: "new_column",
+          },
+        },
+        _add: {
+          name: "new_column",
+        },
+      }
+
+      jest
+        .spyOn(builderSocket!, "emitDatasourceUpdate")
+        .mockImplementation(emitDatasourceUpdateMock)
+
+      await makeRequest("post", "/api/tables/", addColumnToTable)
+
+      const expectedTable: TableRequest = {
+        ...addColumnToTable,
+        schema: {
+          id: {
+            type: FieldType.NUMBER,
+            name: "id",
+            autocolumn: true,
+            constraints: {
+              presence: false,
+            },
+            externalType: "int unsigned",
+          },
+          new_column: {
+            type: FieldType.NUMBER,
+            name: "new_column",
+            autocolumn: false,
+            constraints: {
+              presence: false,
+            },
+            externalType: "float(8,2)",
+          },
+        },
+        created: true,
+        _id: `${mysqlDatasource._id}__table`,
+      }
+      delete expectedTable._add
+
+      expect(emitDatasourceUpdateMock).toBeCalledTimes(1)
+      const emittedDatasource: Datasource =
+        emitDatasourceUpdateMock.mock.calls[0][1]
+      expect(emittedDatasource.entities!["table"]).toEqual(expectedTable)
+    })
+  })
+})

packages/server/src/sdk/app/tables/external/index.ts

@@ -3,6 +3,7 @@ import {
   Operation,
   RelationshipType,
   RenameColumn,
+  AddColumn,
   Table,
   TableRequest,
   ViewV2,
@@ -32,7 +33,7 @@ import * as viewSdk from "../../views"
 export async function save(
   datasourceId: string,
   update: Table,
-  opts?: { tableId?: string; renaming?: RenameColumn }
+  opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn }
 ) {
   let tableToSave: TableRequest = {
     ...update,
@@ -165,8 +166,17 @@ export async function save(
 
   // remove the rename prop
   delete tableToSave._rename
+
+  // if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column
+  if (opts?.adding) {
+    datasource.entities[tableToSave.name] = (
+      await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name])
+    ).tables[tableToSave.name]
+  } else {
+    datasource.entities[tableToSave.name] = tableToSave
+  }
+
   // store it into couch now for budibase reference
-  datasource.entities[tableToSave.name] = tableToSave
   await db.put(populateExternalTableSchemas(datasource))
 
   // Since tables are stored inside datasources, we need to notify clients

packages/types/src/documents/app/table/table.ts

@@ -1,6 +1,6 @@
 import { Document } from "../../document"
 import { View, ViewV2 } from "../view"
-import { RenameColumn } from "../../../sdk"
+import { AddColumn, RenameColumn } from "../../../sdk"
 import { TableSchema } from "./schema"
 
 export const INTERNAL_TABLE_SOURCE_ID = "bb_internal"
@@ -29,5 +29,6 @@ export interface Table extends Document {
 
 export interface TableRequest extends Table {
   _rename?: RenameColumn
+  _add?: AddColumn
   created?: boolean
 }

packages/types/src/sdk/search.ts

@@ -60,6 +60,10 @@ export interface RenameColumn {
   updated: string
 }
 
+export interface AddColumn {
+  name: string
+}
+
 export interface RelationshipsJson {
   through?: string
   from?: string