Merge pull request #4795 from Budibase/fix/sql-issues

Various SQL fixes

packages/server/scripts/integrations/mssql/data/setup.sql

@@ -1,5 +1,10 @@
 USE master;
-  
+
+IF NOT EXISTS(SELECT 1 FROM sys.schemas WHERE name = 'Chains')
+BEGIN
+    EXEC sys.sp_executesql N'CREATE SCHEMA Chains;'
+END
+
 IF OBJECT_ID ('dbo.products', 'U') IS NOT NULL  
   DROP TABLE products;  
 GO  
@@ -61,3 +66,15 @@ VALUES ('Bob', '30'),
   ('Bobert', '99'),
   ('Jan', '22'),
   ('Megan', '11');
+
+
+IF OBJECT_ID ('Chains.sizes', 'U') IS NOT NULL
+  DROP TABLE Chains.sizes;
+GO
+CREATE TABLE Chains.sizes
+(
+  sizeid int IDENTITY(1, 1),
+  name varchar(30),
+  CONSTRAINT pk_size PRIMARY KEY NONCLUSTERED (sizeid)
+);
+

packages/server/scripts/integrations/postgres/init.sql

@@ -1,5 +1,6 @@
 SELECT 'CREATE DATABASE main'
 WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+CREATE SCHEMA test;
 CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer');
 CREATE TABLE Persons (
     PersonID SERIAL PRIMARY KEY,
@@ -37,6 +38,10 @@ CREATE TABLE Products_Tasks (
             REFERENCES Tasks(TaskID),
     PRIMARY KEY (ProductID, TaskID)
 );
+CREATE TABLE test.table1 (
+  id SERIAL PRIMARY KEY,
+  Name varchar(255)
+);
 INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa');
 INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer');
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE);
@@ -48,3 +53,4 @@ INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 1);
 INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (2, 1);
 INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (3, 1);
 INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 2);
+INSERT INTO test.table1 (Name) VALUES ('Test');

packages/server/src/definitions/datasource.ts

@@ -153,6 +153,7 @@ export interface QueryJson {
     datasourceId: string
     entityId: string
     operation: Operation
+    schema?: string
   }
   resource: {
     fields: string[]

packages/server/src/integrations/base/sql.ts

@@ -249,6 +249,9 @@ class InternalBuilder {
   create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body } = json
     let query: KnexQuery = knex(endpoint.entityId)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     const parsedBody = parseBody(body)
     // make sure no null values in body for creation
     for (let [key, value] of Object.entries(parsedBody)) {
@@ -267,6 +270,9 @@ class InternalBuilder {
   bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
     const { endpoint, body } = json
     let query: KnexQuery = knex(endpoint.entityId)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     if (!Array.isArray(body)) {
       return query
     }
@@ -275,7 +281,7 @@ class InternalBuilder {
   }
 
   read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
-    let { endpoint, resource, filters, sort, paginate, relationships } = json
+    let { endpoint, resource, filters, paginate, relationships } = json
     const tableName = endpoint.entityId
     // select all if not specified
     if (!resource) {
@@ -302,6 +308,9 @@ class InternalBuilder {
     }
     // start building the query
     let query: KnexQuery = knex(tableName).limit(foundLimit)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
@@ -331,6 +340,9 @@ class InternalBuilder {
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     const parsedBody = parseBody(body)
     query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
@@ -344,6 +356,9 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
     if (opts.disableReturning) {

packages/server/src/integrations/base/sqlTable.ts

@@ -101,28 +101,28 @@ function generateSchema(
 }
 
 function buildCreateTable(
-  knex: Knex,
+  knex: SchemaBuilder,
   table: Table,
   tables: Record<string, Table>
 ): SchemaBuilder {
-  return knex.schema.createTable(table.name, schema => {
+  return knex.createTable(table.name, schema => {
     generateSchema(schema, table, tables)
   })
 }
 
 function buildUpdateTable(
-  knex: Knex,
+  knex: SchemaBuilder,
   table: Table,
   tables: Record<string, Table>,
   oldTable: Table
 ): SchemaBuilder {
-  return knex.schema.alterTable(table.name, schema => {
+  return knex.alterTable(table.name, schema => {
     generateSchema(schema, table, tables, oldTable)
   })
 }
 
-function buildDeleteTable(knex: Knex, table: Table): SchemaBuilder {
-  return knex.schema.dropTable(table.name)
+function buildDeleteTable(knex: SchemaBuilder, table: Table): SchemaBuilder {
+  return knex.dropTable(table.name)
 }
 
 class SqlTableQueryBuilder {
@@ -146,7 +146,11 @@ class SqlTableQueryBuilder {
   }
 
   _tableQuery(json: QueryJson): any {
-    const client = knex({ client: this.sqlClient })
+    let client = knex({ client: this.sqlClient }).schema
+    if (json?.endpoint?.schema) {
+      client = client.withSchema(json.endpoint.schema)
+    }
+
     let query
     if (!json.table || !json.meta || !json.meta.tables) {
       throw "Cannot execute without table being specified"

packages/server/src/integrations/microsoftSqlServer.ts

@@ -19,6 +19,7 @@ import { Table, TableSchema } from "../definitions/common"
 module MSSQLModule {
   const sqlServer = require("mssql")
   const Sql = require("./base/sql")
+  const DEFAULT_SCHEMA = "dbo"
 
   interface MSSQLConfig {
     user: string
@@ -26,9 +27,17 @@ module MSSQLModule {
     server: string
     port: number
     database: string
+    schema: string
     encrypt?: boolean
   }
 
+  interface TablesResponse {
+    TABLE_CATALOG: string
+    TABLE_SCHEMA: string
+    TABLE_NAME: string
+    TABLE_TYPE: string
+  }
+
   const SCHEMA: Integration = {
     docs: "https://github.com/tediousjs/node-mssql",
     plus: true,
@@ -58,6 +67,10 @@ module MSSQLModule {
         type: DatasourceFieldTypes.STRING,
         default: "root",
       },
+      schema: {
+        type: DatasourceFieldTypes.STRING,
+        default: DEFAULT_SCHEMA,
+      },
       encrypt: {
         type: DatasourceFieldTypes.BOOLEAN,
         default: true,
@@ -96,6 +109,35 @@ module MSSQLModule {
     TABLES_SQL =
       "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'"
 
+    constructor(config: MSSQLConfig) {
+      super(SqlClients.MS_SQL)
+      this.config = config
+      const clientCfg = {
+        ...this.config,
+        options: {
+          encrypt: this.config.encrypt,
+          enableArithAbort: true,
+        },
+      }
+      delete clientCfg.encrypt
+      if (!this.pool) {
+        this.pool = new sqlServer.ConnectionPool(clientCfg)
+      }
+    }
+
+    getBindingIdentifier(): string {
+      return `(@p${this.index++})`
+    }
+
+    async connect() {
+      try {
+        this.client = await this.pool.connect()
+      } catch (err) {
+        // @ts-ignore
+        throw new Error(err)
+      }
+    }
+
     async internalQuery(
       query: SqlQuery,
       operation: string | undefined = undefined
@@ -151,35 +193,6 @@ module MSSQLModule {
               WHERE TABLE_NAME='${tableName}'`
     }
 
-    constructor(config: MSSQLConfig) {
-      super(SqlClients.MS_SQL)
-      this.config = config
-      const clientCfg = {
-        ...this.config,
-        options: {
-          encrypt: this.config.encrypt,
-          enableArithAbort: true,
-        },
-      }
-      delete clientCfg.encrypt
-      if (!this.pool) {
-        this.pool = new sqlServer.ConnectionPool(clientCfg)
-      }
-    }
-
-    getBindingIdentifier(): string {
-      return `(@p${this.index++})`
-    }
-
-    async connect() {
-      try {
-        this.client = await this.pool.connect()
-      } catch (err) {
-        // @ts-ignore
-        throw new Error(err)
-      }
-    }
-
     async runSQL(sql: string) {
       return (await this.internalQuery(getSqlQuery(sql))).recordset
     }
@@ -191,11 +204,14 @@ module MSSQLModule {
      */
     async buildSchema(datasourceId: string, entities: Record<string, Table>) {
       await this.connect()
-      let tableNames = await this.runSQL(this.TABLES_SQL)
-      if (tableNames == null || !Array.isArray(tableNames)) {
+      let tableInfo: TablesResponse[] = await this.runSQL(this.TABLES_SQL)
+      if (tableInfo == null || !Array.isArray(tableInfo)) {
         throw "Unable to get list of tables in database"
       }
-      tableNames = tableNames
+
+      const schema = this.config.schema || DEFAULT_SCHEMA
+      const tableNames = tableInfo
+        .filter((record: any) => record.TABLE_SCHEMA === schema)
         .map((record: any) => record.TABLE_NAME)
         .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
 
@@ -267,7 +283,11 @@ module MSSQLModule {
     }
 
     async query(json: QueryJson) {
+      const schema = this.config.schema
       await this.connect()
+      if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) {
+        json.endpoint.schema = schema
+      }
       const operation = this._operation(json)
       const queryFn = (query: any, op: string) => this.internalQuery(query, op)
       const processFn = (result: any) =>

packages/server/src/integrations/utils.ts

@@ -6,35 +6,63 @@ import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 
-const SQL_TYPE_MAP = {
-  text: FieldTypes.LONGFORM,
-  varchar: FieldTypes.STRING,
+const SQL_NUMBER_TYPE_MAP = {
   integer: FieldTypes.NUMBER,
+  int: FieldTypes.NUMBER,
   bigint: FieldTypes.NUMBER,
   decimal: FieldTypes.NUMBER,
   smallint: FieldTypes.NUMBER,
   real: FieldTypes.NUMBER,
-  "double precision": FieldTypes.NUMBER,
-  timestamp: FieldTypes.DATETIME,
-  time: FieldTypes.DATETIME,
-  boolean: FieldTypes.BOOLEAN,
-  json: FieldTypes.JSON,
-  date: FieldTypes.DATETIME,
-  blob: FieldTypes.LONGFORM,
-  enum: FieldTypes.STRING,
   float: FieldTypes.NUMBER,
-  int: FieldTypes.NUMBER,
   numeric: FieldTypes.NUMBER,
   mediumint: FieldTypes.NUMBER,
   dec: FieldTypes.NUMBER,
   double: FieldTypes.NUMBER,
   fixed: FieldTypes.NUMBER,
-  datetime: FieldTypes.DATETIME,
-  tinyint: FieldTypes.BOOLEAN,
-  long: FieldTypes.LONGFORM,
+  "double precision": FieldTypes.NUMBER,
   number: FieldTypes.NUMBER,
   binary_float: FieldTypes.NUMBER,
   binary_double: FieldTypes.NUMBER,
+  money: FieldTypes.NUMBER,
+  smallmoney: FieldTypes.NUMBER,
+}
+
+const SQL_DATE_TYPE_MAP = {
+  timestamp: FieldTypes.DATETIME,
+  time: FieldTypes.DATETIME,
+  datetime: FieldTypes.DATETIME,
+  smalldatetime: FieldTypes.DATETIME,
+  date: FieldTypes.DATETIME,
+}
+
+const SQL_STRING_TYPE_MAP = {
+  varchar: FieldTypes.STRING,
+  char: FieldTypes.STRING,
+  nchar: FieldTypes.STRING,
+  nvarchar: FieldTypes.STRING,
+  ntext: FieldTypes.STRING,
+  enum: FieldTypes.STRING,
+  blob: FieldTypes.LONGFORM,
+  long: FieldTypes.LONGFORM,
+  text: FieldTypes.LONGFORM,
+}
+
+const SQL_BOOLEAN_TYPE_MAP = {
+  boolean: FieldTypes.BOOLEAN,
+  bit: FieldTypes.BOOLEAN,
+  tinyint: FieldTypes.BOOLEAN,
+}
+
+const SQL_MISC_TYPE_MAP = {
+  json: FieldTypes.JSON,
+}
+
+const SQL_TYPE_MAP = {
+  ...SQL_NUMBER_TYPE_MAP,
+  ...SQL_DATE_TYPE_MAP,
+  ...SQL_STRING_TYPE_MAP,
+  ...SQL_BOOLEAN_TYPE_MAP,
+  ...SQL_MISC_TYPE_MAP,
 }
 
 export enum SqlClients {