From a332c058ce8f1fb371c84c7beb2aff44c1590354 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 16:19:21 +0000 Subject: [PATCH 1/9] Disabling aliasing on writes (create, update, delete) for MySQL/MS-SQL datasources. --- .../server/src/api/controllers/row/alias.ts | 48 +++++++++++++++---- packages/server/src/integrations/base/sql.ts | 10 ++-- packages/server/src/integrations/index.ts | 11 +++-- packages/server/src/sdk/app/rows/utils.ts | 20 ++++++++ 4 files changed, 74 insertions(+), 15 deletions(-) diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 46b090bb97..0adcfaa582 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -1,12 +1,16 @@ import { - QueryJson, - SearchFilters, - Table, - Row, + Datasource, DatasourcePlusQueryResponse, + Operation, + QueryJson, + Row, + SearchFilters, } from "@budibase/types" -import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" +import { getSQLClient } from "../../../sdk/app/rows/utils" import { cloneDeep } from "lodash" +import sdk from "../../../sdk" +import { makeExternalQuery } from "../../../integrations/base/query" +import { SqlClient } from "../../../integrations/utils" class CharSequence { static alphabet = "abcdefghijklmnopqrstuvwxyz" @@ -43,6 +47,32 @@ export default class AliasTables { this.charSeq = new CharSequence() } + isAliasingEnabled(json: QueryJson, datasource: Datasource) { + const fieldLength = json.resource?.fields?.length + if (!fieldLength || fieldLength <= 0) { + return false + } + const writeOperations = [ + Operation.CREATE, + Operation.UPDATE, + Operation.DELETE, + ] + try { + const sqlClient = getSQLClient(datasource) + const isWrite = writeOperations.includes(json.endpoint.operation) + if ( + isWrite && + (sqlClient === SqlClient.MY_SQL || sqlClient === SqlClient.MS_SQL) + ) { + return false + } + } catch (err) { + // if we can't get an SQL client, we can't alias + return false + } + return true + } + getAlias(tableName: string) { if (this.aliases[tableName]) { return this.aliases[tableName] @@ -111,8 +141,10 @@ export default class AliasTables { } async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse { - const fieldLength = json.resource?.fields?.length - const aliasingEnabled = fieldLength && fieldLength > 0 + const datasourceId = json.endpoint.datasourceId + const datasource = await sdk.datasources.get(datasourceId) + + const aliasingEnabled = this.isAliasingEnabled(json, datasource) if (aliasingEnabled) { json = cloneDeep(json) // run through the query json to update anywhere a table may be used @@ -158,7 +190,7 @@ export default class AliasTables { } json.tableAliases = invertedTableAliases } - const response = await getDatasourceAndQuery(json) + const response = await makeExternalQuery(datasource, json) if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index c8acb606b3..be1883c8ec 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -435,10 +435,12 @@ class InternalBuilder { aliases?: QueryJson["tableAliases"] ): Knex.QueryBuilder { const tableName = endpoint.entityId - const tableAliased = aliases?.[tableName] - ? `${tableName} as ${aliases?.[tableName]}` - : tableName - let query = knex(tableAliased) + const tableAlias = aliases?.[tableName] + let table: string | Record = tableName + if (tableAlias) { + table = { [tableAlias]: tableName } + } + let query = knex(table) if (endpoint.schema) { query = query.withSchema(endpoint.schema) } diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index ee2bb23f23..18c46b9260 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -14,13 +14,18 @@ import firebase from "./firebase" import redis from "./redis" import snowflake from "./snowflake" import oracle from "./oracle" -import { SourceName, Integration, PluginType } from "@budibase/types" +import { + SourceName, + Integration, + PluginType, + IntegrationBase, +} from "@budibase/types" import { getDatasourcePlugin } from "../utilities/fileSystem" import env from "../environment" import cloneDeep from "lodash/cloneDeep" import sdk from "../sdk" -const DEFINITIONS: Record = { +const DEFINITIONS: { [key: SourceName]: Integration | undefined } = { [SourceName.POSTGRES]: postgres.schema, [SourceName.DYNAMODB]: dynamodb.schema, [SourceName.MONGODB]: mongodb.schema, @@ -40,7 +45,7 @@ const DEFINITIONS: Record = { [SourceName.BUDIBASE]: undefined, } -const INTEGRATIONS: Record = { +const INTEGRATIONS: { [key: SourceName]: IntegrationBase | undefined } = { [SourceName.POSTGRES]: postgres.integration, [SourceName.DYNAMODB]: dynamodb.integration, [SourceName.MONGODB]: mongodb.integration, diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index a8052462a9..e090045925 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -7,11 +7,31 @@ import { Table, TableSchema, DatasourcePlusQueryResponse, + Datasource, + SourceName, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { isRelationshipColumn } from "../../../db/utils" +import { SqlClient } from "../../../integrations/utils" + +export function getSQLClient(datasource: Datasource): SqlClient { + if (!datasource.isSQL) { + throw new Error("Cannot get SQL Client for non-SQL datasource") + } + switch (datasource.source) { + case SourceName.POSTGRES: + return SqlClient.POSTGRES + case SourceName.MYSQL: + return SqlClient.MY_SQL + case SourceName.ORACLE: + return SqlClient.ORACLE + case SourceName.SQL_SERVER: + return SqlClient.MS_SQL + } + throw new Error("Unable to find a valid SQL client") +} export async function getDatasourceAndQuery( json: QueryJson From 0520c0c54083eed0900f12017dfcee2b50257bec Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 17:27:35 +0000 Subject: [PATCH 2/9] Adding tests to confirm when aliasing should be used. --- .../server/src/api/controllers/row/alias.ts | 29 +++----- .../src/integrations/tests/sqlAlias.spec.ts | 74 ++++++++++++++++++- 2 files changed, 84 insertions(+), 19 deletions(-) diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 0adcfaa582..28eaa95bf8 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -1,17 +1,17 @@ -import { - Datasource, - DatasourcePlusQueryResponse, - Operation, - QueryJson, - Row, - SearchFilters, -} from "@budibase/types" +import { Datasource, DatasourcePlusQueryResponse, Operation, QueryJson, Row, SearchFilters } from "@budibase/types" import { getSQLClient } from "../../../sdk/app/rows/utils" import { cloneDeep } from "lodash" import sdk from "../../../sdk" import { makeExternalQuery } from "../../../integrations/base/query" import { SqlClient } from "../../../integrations/utils" +const WRITE_OPERATIONS: Operation[] = [ + Operation.CREATE, + Operation.UPDATE, + Operation.DELETE, +] +const DISABLED_WRITE_CLIENTS: SqlClient[] = [SqlClient.MY_SQL, SqlClient.MS_SQL, SqlClient.ORACLE] + class CharSequence { static alphabet = "abcdefghijklmnopqrstuvwxyz" counters: number[] @@ -52,18 +52,11 @@ export default class AliasTables { if (!fieldLength || fieldLength <= 0) { return false } - const writeOperations = [ - Operation.CREATE, - Operation.UPDATE, - Operation.DELETE, - ] try { const sqlClient = getSQLClient(datasource) - const isWrite = writeOperations.includes(json.endpoint.operation) - if ( - isWrite && - (sqlClient === SqlClient.MY_SQL || sqlClient === SqlClient.MS_SQL) - ) { + const isWrite = WRITE_OPERATIONS.includes(json.endpoint.operation) + const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient) + if (isWrite && isDisabledClient) { return false } } catch (err) { diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index fe9798aaa0..70dda8c335 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -1,4 +1,4 @@ -import { QueryJson } from "@budibase/types" +import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types" import { join } from "path" import Sql from "../base/sql" import { SqlClient } from "../utils" @@ -198,6 +198,78 @@ describe("Captures of real examples", () => { }) }) + describe("check aliasing is disabled/enabled", () => { + const tables = ["tableA", "tableB"] + + function getDatasource(source: SourceName): Datasource { + return { + source, + type: "datasource", + isSQL: true, + } + } + + function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson { + return { + endpoint: { datasourceId: "", entityId: "", operation: op }, + resource: { + fields, + } + } + } + + it("should check for Postgres aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.POSTGRES) + expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(true) + }) + + it("should check for MS-SQL aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.SQL_SERVER) + expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + }) + + it("should check for MySQL aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.MYSQL) + expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + }) + + it("should check for Oracle aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.ORACLE) + expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + }) + + it("should disable aliasing for non-SQL datasources", () => { + const aliasing = new AliasTables(tables) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), { + source: SourceName.GOOGLE_SHEETS, + type: "datasource", + isSQL: false, + })) + }) + + it("should disable when no fields", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.POSTGRES) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource)).toEqual(false) + }) + }) + describe("check some edge cases", () => { const tableNames = ["hello", "world"] From edda776b14c09fe97325ded56eebca8970a1f537 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 17:42:44 +0000 Subject: [PATCH 3/9] PR comments. --- packages/server/src/sdk/app/rows/utils.ts | 40 +++++++++++++++-------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index e090045925..6e3e25364e 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -1,14 +1,14 @@ import cloneDeep from "lodash/cloneDeep" import validateJs from "validate.js" import { + Datasource, + DatasourcePlusQueryResponse, FieldType, QueryJson, Row, + SourceName, Table, TableSchema, - DatasourcePlusQueryResponse, - Datasource, - SourceName, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" @@ -16,21 +16,35 @@ import sdk from "../.." import { isRelationshipColumn } from "../../../db/utils" import { SqlClient } from "../../../integrations/utils" +const SQL_CLIENT_SOURCE_MAP: Record = { + [SourceName.POSTGRES]: SqlClient.POSTGRES, + [SourceName.MYSQL]: SqlClient.MY_SQL, + [SourceName.SQL_SERVER]: SqlClient.MS_SQL, + [SourceName.ORACLE]: SqlClient.ORACLE, + [SourceName.DYNAMODB]: undefined, + [SourceName.MONGODB]: undefined, + [SourceName.ELASTICSEARCH]: undefined, + [SourceName.COUCHDB]: undefined, + [SourceName.S3]: undefined, + [SourceName.AIRTABLE]: undefined, + [SourceName.ARANGODB]: undefined, + [SourceName.REST]: undefined, + [SourceName.FIRESTORE]: undefined, + [SourceName.GOOGLE_SHEETS]: undefined, + [SourceName.REDIS]: undefined, + [SourceName.SNOWFLAKE]: undefined, + [SourceName.BUDIBASE]: undefined, +} + export function getSQLClient(datasource: Datasource): SqlClient { if (!datasource.isSQL) { throw new Error("Cannot get SQL Client for non-SQL datasource") } - switch (datasource.source) { - case SourceName.POSTGRES: - return SqlClient.POSTGRES - case SourceName.MYSQL: - return SqlClient.MY_SQL - case SourceName.ORACLE: - return SqlClient.ORACLE - case SourceName.SQL_SERVER: - return SqlClient.MS_SQL + const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source] + if (lookup) { + return lookup } - throw new Error("Unable to find a valid SQL client") + throw new Error("Unable to determine client for SQL datasource") } export async function getDatasourceAndQuery( From b72edc21ecde8fb9564520d94c9a6046430886d5 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 17:46:09 +0000 Subject: [PATCH 4/9] Linting. --- .../server/src/api/controllers/row/alias.ts | 15 +++- .../src/integrations/tests/sqlAlias.spec.ts | 82 +++++++++++++------ 2 files changed, 72 insertions(+), 25 deletions(-) diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 28eaa95bf8..1d586c54fd 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -1,4 +1,11 @@ -import { Datasource, DatasourcePlusQueryResponse, Operation, QueryJson, Row, SearchFilters } from "@budibase/types" +import { + Datasource, + DatasourcePlusQueryResponse, + Operation, + QueryJson, + Row, + SearchFilters, +} from "@budibase/types" import { getSQLClient } from "../../../sdk/app/rows/utils" import { cloneDeep } from "lodash" import sdk from "../../../sdk" @@ -10,7 +17,11 @@ const WRITE_OPERATIONS: Operation[] = [ Operation.UPDATE, Operation.DELETE, ] -const DISABLED_WRITE_CLIENTS: SqlClient[] = [SqlClient.MY_SQL, SqlClient.MS_SQL, SqlClient.ORACLE] +const DISABLED_WRITE_CLIENTS: SqlClient[] = [ + SqlClient.MY_SQL, + SqlClient.MS_SQL, + SqlClient.ORACLE, +] class CharSequence { static alphabet = "abcdefghijklmnopqrstuvwxyz" diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 70dda8c335..dd82dadac0 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -214,59 +214,95 @@ describe("Captures of real examples", () => { endpoint: { datasourceId: "", entityId: "", operation: op }, resource: { fields, - } + }, } } it("should check for Postgres aliased status", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.POSTGRES) - expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(true) }) it("should check for MS-SQL aliased status", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.SQL_SERVER) - expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(false) }) it("should check for MySQL aliased status", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.MYSQL) - expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(false) }) it("should check for Oracle aliased status", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.ORACLE) - expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(false) }) it("should disable aliasing for non-SQL datasources", () => { const aliasing = new AliasTables(tables) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), { - source: SourceName.GOOGLE_SHEETS, - type: "datasource", - isSQL: false, - })) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), { + source: SourceName.GOOGLE_SHEETS, + type: "datasource", + isSQL: false, + }) + ) }) it("should disable when no fields", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.POSTGRES) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource)).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource) + ).toEqual(false) }) }) From 1918ec6c68f6979d41852061cf8437a7599f9511 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 18:00:15 +0000 Subject: [PATCH 5/9] Reverting type changes. --- packages/server/src/integrations/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 18c46b9260..92067d1918 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -25,7 +25,7 @@ import env from "../environment" import cloneDeep from "lodash/cloneDeep" import sdk from "../sdk" -const DEFINITIONS: { [key: SourceName]: Integration | undefined } = { +const DEFINITIONS: Record = { [SourceName.POSTGRES]: postgres.schema, [SourceName.DYNAMODB]: dynamodb.schema, [SourceName.MONGODB]: mongodb.schema, @@ -45,7 +45,7 @@ const DEFINITIONS: { [key: SourceName]: Integration | undefined } = { [SourceName.BUDIBASE]: undefined, } -const INTEGRATIONS: { [key: SourceName]: IntegrationBase | undefined } = { +const INTEGRATIONS: Record = { [SourceName.POSTGRES]: postgres.integration, [SourceName.DYNAMODB]: dynamodb.integration, [SourceName.MONGODB]: mongodb.integration, From b58b0d3b40320ae551b3e8c82028b18ce236f8d5 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 18:15:19 +0000 Subject: [PATCH 6/9] Fixing integration base types. --- packages/server/src/integrations/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 92067d1918..747a717278 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -45,7 +45,9 @@ const DEFINITIONS: Record = { [SourceName.BUDIBASE]: undefined, } -const INTEGRATIONS: Record = { +type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase + +const INTEGRATIONS: Record = { [SourceName.POSTGRES]: postgres.integration, [SourceName.DYNAMODB]: dynamodb.integration, [SourceName.MONGODB]: mongodb.integration, From de56324a4b24fd41414369435654c4833fc4dca7 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 18:16:27 +0000 Subject: [PATCH 7/9] Linting --- packages/server/src/integrations/index.ts | 39 ++++++++++++----------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 747a717278..8cbc29251b 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -47,25 +47,26 @@ const DEFINITIONS: Record = { type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase -const INTEGRATIONS: Record = { - [SourceName.POSTGRES]: postgres.integration, - [SourceName.DYNAMODB]: dynamodb.integration, - [SourceName.MONGODB]: mongodb.integration, - [SourceName.ELASTICSEARCH]: elasticsearch.integration, - [SourceName.COUCHDB]: couchdb.integration, - [SourceName.SQL_SERVER]: sqlServer.integration, - [SourceName.S3]: s3.integration, - [SourceName.AIRTABLE]: airtable.integration, - [SourceName.MYSQL]: mysql.integration, - [SourceName.ARANGODB]: arangodb.integration, - [SourceName.REST]: rest.integration, - [SourceName.FIRESTORE]: firebase.integration, - [SourceName.GOOGLE_SHEETS]: googlesheets.integration, - [SourceName.REDIS]: redis.integration, - [SourceName.SNOWFLAKE]: snowflake.integration, - [SourceName.ORACLE]: undefined, - [SourceName.BUDIBASE]: undefined, -} +const INTEGRATIONS: Record = + { + [SourceName.POSTGRES]: postgres.integration, + [SourceName.DYNAMODB]: dynamodb.integration, + [SourceName.MONGODB]: mongodb.integration, + [SourceName.ELASTICSEARCH]: elasticsearch.integration, + [SourceName.COUCHDB]: couchdb.integration, + [SourceName.SQL_SERVER]: sqlServer.integration, + [SourceName.S3]: s3.integration, + [SourceName.AIRTABLE]: airtable.integration, + [SourceName.MYSQL]: mysql.integration, + [SourceName.ARANGODB]: arangodb.integration, + [SourceName.REST]: rest.integration, + [SourceName.FIRESTORE]: firebase.integration, + [SourceName.GOOGLE_SHEETS]: googlesheets.integration, + [SourceName.REDIS]: redis.integration, + [SourceName.SNOWFLAKE]: snowflake.integration, + [SourceName.ORACLE]: undefined, + [SourceName.BUDIBASE]: undefined, + } // optionally add oracle integration if the oracle binary can be installed if ( From bed813da77c2ef5c0eb7a4bdc290f21ae2fd4e4c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 5 Mar 2024 18:29:11 +0000 Subject: [PATCH 8/9] Bump version to 2.21.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 6fb032ac77..a77a16a24e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.2", + "version": "2.21.3", "npmClient": "yarn", "packages": [ "packages/*", From 4baadadaa8d2572617236ebb26d66cdbde88f611 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Mar 2024 12:22:20 +0100 Subject: [PATCH 9/9] Use pipeline instead of eval --- packages/backend-core/src/redis/redis.ts | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 8cfa3db5c1..6124f5f447 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -291,23 +291,16 @@ class RedisWrapper { return acc }, {} as Record) - const luaScript = ` - for i, key in ipairs(KEYS) do - redis.call('MSET', key, ARGV[i]) - ${ - expirySeconds !== null - ? `redis.call('EXPIRE', key, ARGV[#ARGV])` - : "" - } - end - ` - const keys = Object.keys(dataToStore) - const values = Object.values(dataToStore) + const pipeline = client.pipeline() + pipeline.mset(dataToStore) + if (expirySeconds !== null) { - values.push(expirySeconds) + for (const key of Object.keys(dataToStore)) { + pipeline.expire(key, expirySeconds) + } } - await client.eval(luaScript, keys.length, ...keys, ...values) + await pipeline.exec() } async getTTL(key: string) {