Merge branch 'master' into BUDI-8046/redis-increment

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.21.2",
+  "version": "2.21.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/redis/redis.ts

@@ -291,23 +291,16 @@ class RedisWrapper {
       return acc
     }, {} as Record<string, any>)
 
-    const luaScript = `
+    const pipeline = client.pipeline()
-        for i, key in ipairs(KEYS) do
+    pipeline.mset(dataToStore)
-            redis.call('MSET', key, ARGV[i])
+
-            ${
-              expirySeconds !== null
-                ? `redis.call('EXPIRE', key, ARGV[#ARGV])`
-                : ""
-            }
-        end
-        `
-    const keys = Object.keys(dataToStore)
-    const values = Object.values(dataToStore)
     if (expirySeconds !== null) {
-      values.push(expirySeconds)
+      for (const key of Object.keys(dataToStore)) {
+        pipeline.expire(key, expirySeconds)
+      }
     }
 
-    await client.eval(luaScript, keys.length, ...keys, ...values)
+    await pipeline.exec()
   }
 
   async getTTL(key: string) {

packages/server/src/api/controllers/row/alias.ts

@@ -1,12 +1,27 @@
 import {
-  QueryJson,
+  Datasource,
-  SearchFilters,
-  Table,
-  Row,
   DatasourcePlusQueryResponse,
+  Operation,
+  QueryJson,
+  Row,
+  SearchFilters,
 } from "@budibase/types"
-import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
+import { getSQLClient } from "../../../sdk/app/rows/utils"
 import { cloneDeep } from "lodash"
+import sdk from "../../../sdk"
+import { makeExternalQuery } from "../../../integrations/base/query"
+import { SqlClient } from "../../../integrations/utils"
+
+const WRITE_OPERATIONS: Operation[] = [
+  Operation.CREATE,
+  Operation.UPDATE,
+  Operation.DELETE,
+]
+const DISABLED_WRITE_CLIENTS: SqlClient[] = [
+  SqlClient.MY_SQL,
+  SqlClient.MS_SQL,
+  SqlClient.ORACLE,
+]
 
 class CharSequence {
   static alphabet = "abcdefghijklmnopqrstuvwxyz"
@@ -43,6 +58,25 @@ export default class AliasTables {
     this.charSeq = new CharSequence()
   }
 
+  isAliasingEnabled(json: QueryJson, datasource: Datasource) {
+    const fieldLength = json.resource?.fields?.length
+    if (!fieldLength || fieldLength <= 0) {
+      return false
+    }
+    try {
+      const sqlClient = getSQLClient(datasource)
+      const isWrite = WRITE_OPERATIONS.includes(json.endpoint.operation)
+      const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient)
+      if (isWrite && isDisabledClient) {
+        return false
+      }
+    } catch (err) {
+      // if we can't get an SQL client, we can't alias
+      return false
+    }
+    return true
+  }
+
   getAlias(tableName: string) {
     if (this.aliases[tableName]) {
       return this.aliases[tableName]
@@ -111,8 +145,10 @@ export default class AliasTables {
   }
 
   async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
-    const fieldLength = json.resource?.fields?.length
+    const datasourceId = json.endpoint.datasourceId
-    const aliasingEnabled = fieldLength && fieldLength > 0
+    const datasource = await sdk.datasources.get(datasourceId)
+
+    const aliasingEnabled = this.isAliasingEnabled(json, datasource)
     if (aliasingEnabled) {
       json = cloneDeep(json)
       // run through the query json to update anywhere a table may be used
@@ -158,7 +194,7 @@ export default class AliasTables {
       }
       json.tableAliases = invertedTableAliases
     }
-    const response = await getDatasourceAndQuery(json)
+    const response = await makeExternalQuery(datasource, json)
     if (Array.isArray(response) && aliasingEnabled) {
       return this.reverse(response)
     } else {

packages/server/src/integrations/base/sql.ts

@@ -435,10 +435,12 @@ class InternalBuilder {
     aliases?: QueryJson["tableAliases"]
   ): Knex.QueryBuilder {
     const tableName = endpoint.entityId
-    const tableAliased = aliases?.[tableName]
+    const tableAlias = aliases?.[tableName]
-      ? `${tableName} as ${aliases?.[tableName]}`
+    let table: string | Record<string, string> = tableName
-      : tableName
+    if (tableAlias) {
-    let query = knex(tableAliased)
+      table = { [tableAlias]: tableName }
+    }
+    let query = knex(table)
     if (endpoint.schema) {
       query = query.withSchema(endpoint.schema)
     }

packages/server/src/integrations/index.ts

@@ -14,7 +14,12 @@ import firebase from "./firebase"
 import redis from "./redis"
 import snowflake from "./snowflake"
 import oracle from "./oracle"
-import { SourceName, Integration, PluginType } from "@budibase/types"
+import {
+  SourceName,
+  Integration,
+  PluginType,
+  IntegrationBase,
+} from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
 import env from "../environment"
 import cloneDeep from "lodash/cloneDeep"
@@ -40,25 +45,28 @@ const DEFINITIONS: Record<SourceName, Integration | undefined> = {
   [SourceName.BUDIBASE]: undefined,
 }
 
-const INTEGRATIONS: Record<SourceName, any> = {
+type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase
-  [SourceName.POSTGRES]: postgres.integration,
+
-  [SourceName.DYNAMODB]: dynamodb.integration,
+const INTEGRATIONS: Record<SourceName, IntegrationBaseConstructor | undefined> =
-  [SourceName.MONGODB]: mongodb.integration,
+  {
-  [SourceName.ELASTICSEARCH]: elasticsearch.integration,
+    [SourceName.POSTGRES]: postgres.integration,
-  [SourceName.COUCHDB]: couchdb.integration,
+    [SourceName.DYNAMODB]: dynamodb.integration,
-  [SourceName.SQL_SERVER]: sqlServer.integration,
+    [SourceName.MONGODB]: mongodb.integration,
-  [SourceName.S3]: s3.integration,
+    [SourceName.ELASTICSEARCH]: elasticsearch.integration,
-  [SourceName.AIRTABLE]: airtable.integration,
+    [SourceName.COUCHDB]: couchdb.integration,
-  [SourceName.MYSQL]: mysql.integration,
+    [SourceName.SQL_SERVER]: sqlServer.integration,
-  [SourceName.ARANGODB]: arangodb.integration,
+    [SourceName.S3]: s3.integration,
-  [SourceName.REST]: rest.integration,
+    [SourceName.AIRTABLE]: airtable.integration,
-  [SourceName.FIRESTORE]: firebase.integration,
+    [SourceName.MYSQL]: mysql.integration,
-  [SourceName.GOOGLE_SHEETS]: googlesheets.integration,
+    [SourceName.ARANGODB]: arangodb.integration,
-  [SourceName.REDIS]: redis.integration,
+    [SourceName.REST]: rest.integration,
-  [SourceName.SNOWFLAKE]: snowflake.integration,
+    [SourceName.FIRESTORE]: firebase.integration,
-  [SourceName.ORACLE]: undefined,
+    [SourceName.GOOGLE_SHEETS]: googlesheets.integration,
-  [SourceName.BUDIBASE]: undefined,
+    [SourceName.REDIS]: redis.integration,
-}
+    [SourceName.SNOWFLAKE]: snowflake.integration,
+    [SourceName.ORACLE]: undefined,
+    [SourceName.BUDIBASE]: undefined,
+  }
 
 // optionally add oracle integration if the oracle binary can be installed
 if (

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -1,4 +1,4 @@
-import { QueryJson } from "@budibase/types"
+import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types"
 import { join } from "path"
 import Sql from "../base/sql"
 import { SqlClient } from "../utils"
@@ -198,6 +198,114 @@ describe("Captures of real examples", () => {
     })
   })
 
+  describe("check aliasing is disabled/enabled", () => {
+    const tables = ["tableA", "tableB"]
+
+    function getDatasource(source: SourceName): Datasource {
+      return {
+        source,
+        type: "datasource",
+        isSQL: true,
+      }
+    }
+
+    function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson {
+      return {
+        endpoint: { datasourceId: "", entityId: "", operation: op },
+        resource: {
+          fields,
+        },
+      }
+    }
+
+    it("should check for Postgres aliased status", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.POSTGRES)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
+      ).toEqual(true)
+    })
+
+    it("should check for MS-SQL aliased status", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.SQL_SERVER)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
+      ).toEqual(false)
+    })
+
+    it("should check for MySQL aliased status", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.MYSQL)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
+      ).toEqual(false)
+    })
+
+    it("should check for Oracle aliased status", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.ORACLE)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
+      ).toEqual(false)
+    })
+
+    it("should disable aliasing for non-SQL datasources", () => {
+      const aliasing = new AliasTables(tables)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), {
+          source: SourceName.GOOGLE_SHEETS,
+          type: "datasource",
+          isSQL: false,
+        })
+      )
+    })
+
+    it("should disable when no fields", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.POSTGRES)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource)
+      ).toEqual(false)
+    })
+  })
+
   describe("check some edge cases", () => {
     const tableNames = ["hello", "world"]
 

packages/server/src/sdk/app/rows/utils.ts

@@ -1,17 +1,51 @@
 import cloneDeep from "lodash/cloneDeep"
 import validateJs from "validate.js"
 import {
+  Datasource,
+  DatasourcePlusQueryResponse,
   FieldType,
   QueryJson,
   Row,
+  SourceName,
   Table,
   TableSchema,
-  DatasourcePlusQueryResponse,
 } from "@budibase/types"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { isRelationshipColumn } from "../../../db/utils"
+import { SqlClient } from "../../../integrations/utils"
+
+const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
+  [SourceName.POSTGRES]: SqlClient.POSTGRES,
+  [SourceName.MYSQL]: SqlClient.MY_SQL,
+  [SourceName.SQL_SERVER]: SqlClient.MS_SQL,
+  [SourceName.ORACLE]: SqlClient.ORACLE,
+  [SourceName.DYNAMODB]: undefined,
+  [SourceName.MONGODB]: undefined,
+  [SourceName.ELASTICSEARCH]: undefined,
+  [SourceName.COUCHDB]: undefined,
+  [SourceName.S3]: undefined,
+  [SourceName.AIRTABLE]: undefined,
+  [SourceName.ARANGODB]: undefined,
+  [SourceName.REST]: undefined,
+  [SourceName.FIRESTORE]: undefined,
+  [SourceName.GOOGLE_SHEETS]: undefined,
+  [SourceName.REDIS]: undefined,
+  [SourceName.SNOWFLAKE]: undefined,
+  [SourceName.BUDIBASE]: undefined,
+}
+
+export function getSQLClient(datasource: Datasource): SqlClient {
+  if (!datasource.isSQL) {
+    throw new Error("Cannot get SQL Client for non-SQL datasource")
+  }
+  const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source]
+  if (lookup) {
+    return lookup
+  }
+  throw new Error("Unable to determine client for SQL datasource")
+}
 
 export async function getDatasourceAndQuery(
   json: QueryJson