diff --git a/lerna.json b/lerna.json index 6fb032ac77..a77a16a24e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.2", + "version": "2.21.3", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index a5e1597c9f..204a96d236 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -291,23 +291,16 @@ class RedisWrapper { return acc }, {} as Record) - const luaScript = ` - for i, key in ipairs(KEYS) do - redis.call('MSET', key, ARGV[i]) - ${ - expirySeconds !== null - ? `redis.call('EXPIRE', key, ARGV[#ARGV])` - : "" - } - end - ` - const keys = Object.keys(dataToStore) - const values = Object.values(dataToStore) + const pipeline = client.pipeline() + pipeline.mset(dataToStore) + if (expirySeconds !== null) { - values.push(expirySeconds) + for (const key of Object.keys(dataToStore)) { + pipeline.expire(key, expirySeconds) + } } - await client.eval(luaScript, keys.length, ...keys, ...values) + await pipeline.exec() } async getTTL(key: string) { diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 46b090bb97..1d586c54fd 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -1,12 +1,27 @@ import { - QueryJson, - SearchFilters, - Table, - Row, + Datasource, DatasourcePlusQueryResponse, + Operation, + QueryJson, + Row, + SearchFilters, } from "@budibase/types" -import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" +import { getSQLClient } from "../../../sdk/app/rows/utils" import { cloneDeep } from "lodash" +import sdk from "../../../sdk" +import { makeExternalQuery } from "../../../integrations/base/query" +import { SqlClient } from "../../../integrations/utils" + +const WRITE_OPERATIONS: Operation[] = [ + Operation.CREATE, + Operation.UPDATE, + Operation.DELETE, +] +const DISABLED_WRITE_CLIENTS: SqlClient[] = [ + SqlClient.MY_SQL, + SqlClient.MS_SQL, + SqlClient.ORACLE, +] class CharSequence { static alphabet = "abcdefghijklmnopqrstuvwxyz" @@ -43,6 +58,25 @@ export default class AliasTables { this.charSeq = new CharSequence() } + isAliasingEnabled(json: QueryJson, datasource: Datasource) { + const fieldLength = json.resource?.fields?.length + if (!fieldLength || fieldLength <= 0) { + return false + } + try { + const sqlClient = getSQLClient(datasource) + const isWrite = WRITE_OPERATIONS.includes(json.endpoint.operation) + const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient) + if (isWrite && isDisabledClient) { + return false + } + } catch (err) { + // if we can't get an SQL client, we can't alias + return false + } + return true + } + getAlias(tableName: string) { if (this.aliases[tableName]) { return this.aliases[tableName] @@ -111,8 +145,10 @@ export default class AliasTables { } async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse { - const fieldLength = json.resource?.fields?.length - const aliasingEnabled = fieldLength && fieldLength > 0 + const datasourceId = json.endpoint.datasourceId + const datasource = await sdk.datasources.get(datasourceId) + + const aliasingEnabled = this.isAliasingEnabled(json, datasource) if (aliasingEnabled) { json = cloneDeep(json) // run through the query json to update anywhere a table may be used @@ -158,7 +194,7 @@ export default class AliasTables { } json.tableAliases = invertedTableAliases } - const response = await getDatasourceAndQuery(json) + const response = await makeExternalQuery(datasource, json) if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index c8acb606b3..be1883c8ec 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -435,10 +435,12 @@ class InternalBuilder { aliases?: QueryJson["tableAliases"] ): Knex.QueryBuilder { const tableName = endpoint.entityId - const tableAliased = aliases?.[tableName] - ? `${tableName} as ${aliases?.[tableName]}` - : tableName - let query = knex(tableAliased) + const tableAlias = aliases?.[tableName] + let table: string | Record = tableName + if (tableAlias) { + table = { [tableAlias]: tableName } + } + let query = knex(table) if (endpoint.schema) { query = query.withSchema(endpoint.schema) } diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index ee2bb23f23..8cbc29251b 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -14,7 +14,12 @@ import firebase from "./firebase" import redis from "./redis" import snowflake from "./snowflake" import oracle from "./oracle" -import { SourceName, Integration, PluginType } from "@budibase/types" +import { + SourceName, + Integration, + PluginType, + IntegrationBase, +} from "@budibase/types" import { getDatasourcePlugin } from "../utilities/fileSystem" import env from "../environment" import cloneDeep from "lodash/cloneDeep" @@ -40,25 +45,28 @@ const DEFINITIONS: Record = { [SourceName.BUDIBASE]: undefined, } -const INTEGRATIONS: Record = { - [SourceName.POSTGRES]: postgres.integration, - [SourceName.DYNAMODB]: dynamodb.integration, - [SourceName.MONGODB]: mongodb.integration, - [SourceName.ELASTICSEARCH]: elasticsearch.integration, - [SourceName.COUCHDB]: couchdb.integration, - [SourceName.SQL_SERVER]: sqlServer.integration, - [SourceName.S3]: s3.integration, - [SourceName.AIRTABLE]: airtable.integration, - [SourceName.MYSQL]: mysql.integration, - [SourceName.ARANGODB]: arangodb.integration, - [SourceName.REST]: rest.integration, - [SourceName.FIRESTORE]: firebase.integration, - [SourceName.GOOGLE_SHEETS]: googlesheets.integration, - [SourceName.REDIS]: redis.integration, - [SourceName.SNOWFLAKE]: snowflake.integration, - [SourceName.ORACLE]: undefined, - [SourceName.BUDIBASE]: undefined, -} +type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase + +const INTEGRATIONS: Record = + { + [SourceName.POSTGRES]: postgres.integration, + [SourceName.DYNAMODB]: dynamodb.integration, + [SourceName.MONGODB]: mongodb.integration, + [SourceName.ELASTICSEARCH]: elasticsearch.integration, + [SourceName.COUCHDB]: couchdb.integration, + [SourceName.SQL_SERVER]: sqlServer.integration, + [SourceName.S3]: s3.integration, + [SourceName.AIRTABLE]: airtable.integration, + [SourceName.MYSQL]: mysql.integration, + [SourceName.ARANGODB]: arangodb.integration, + [SourceName.REST]: rest.integration, + [SourceName.FIRESTORE]: firebase.integration, + [SourceName.GOOGLE_SHEETS]: googlesheets.integration, + [SourceName.REDIS]: redis.integration, + [SourceName.SNOWFLAKE]: snowflake.integration, + [SourceName.ORACLE]: undefined, + [SourceName.BUDIBASE]: undefined, + } // optionally add oracle integration if the oracle binary can be installed if ( diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index fe9798aaa0..dd82dadac0 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -1,4 +1,4 @@ -import { QueryJson } from "@budibase/types" +import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types" import { join } from "path" import Sql from "../base/sql" import { SqlClient } from "../utils" @@ -198,6 +198,114 @@ describe("Captures of real examples", () => { }) }) + describe("check aliasing is disabled/enabled", () => { + const tables = ["tableA", "tableB"] + + function getDatasource(source: SourceName): Datasource { + return { + source, + type: "datasource", + isSQL: true, + } + } + + function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson { + return { + endpoint: { datasourceId: "", entityId: "", operation: op }, + resource: { + fields, + }, + } + } + + it("should check for Postgres aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.POSTGRES) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(true) + }) + + it("should check for MS-SQL aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.SQL_SERVER) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(false) + }) + + it("should check for MySQL aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.MYSQL) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(false) + }) + + it("should check for Oracle aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.ORACLE) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(false) + }) + + it("should disable aliasing for non-SQL datasources", () => { + const aliasing = new AliasTables(tables) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), { + source: SourceName.GOOGLE_SHEETS, + type: "datasource", + isSQL: false, + }) + ) + }) + + it("should disable when no fields", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.POSTGRES) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource) + ).toEqual(false) + }) + }) + describe("check some edge cases", () => { const tableNames = ["hello", "world"] diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index a8052462a9..6e3e25364e 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -1,17 +1,51 @@ import cloneDeep from "lodash/cloneDeep" import validateJs from "validate.js" import { + Datasource, + DatasourcePlusQueryResponse, FieldType, QueryJson, Row, + SourceName, Table, TableSchema, - DatasourcePlusQueryResponse, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { isRelationshipColumn } from "../../../db/utils" +import { SqlClient } from "../../../integrations/utils" + +const SQL_CLIENT_SOURCE_MAP: Record = { + [SourceName.POSTGRES]: SqlClient.POSTGRES, + [SourceName.MYSQL]: SqlClient.MY_SQL, + [SourceName.SQL_SERVER]: SqlClient.MS_SQL, + [SourceName.ORACLE]: SqlClient.ORACLE, + [SourceName.DYNAMODB]: undefined, + [SourceName.MONGODB]: undefined, + [SourceName.ELASTICSEARCH]: undefined, + [SourceName.COUCHDB]: undefined, + [SourceName.S3]: undefined, + [SourceName.AIRTABLE]: undefined, + [SourceName.ARANGODB]: undefined, + [SourceName.REST]: undefined, + [SourceName.FIRESTORE]: undefined, + [SourceName.GOOGLE_SHEETS]: undefined, + [SourceName.REDIS]: undefined, + [SourceName.SNOWFLAKE]: undefined, + [SourceName.BUDIBASE]: undefined, +} + +export function getSQLClient(datasource: Datasource): SqlClient { + if (!datasource.isSQL) { + throw new Error("Cannot get SQL Client for non-SQL datasource") + } + const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source] + if (lookup) { + return lookup + } + throw new Error("Unable to determine client for SQL datasource") +} export async function getDatasourceAndQuery( json: QueryJson