CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-06 15:00:49 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into BUDI-8046/redis-increment

Browse source
This commit is contained in:
Adria Navarro 2024-03-06 12:32:51 +01:00 committed by GitHub
parent c198a5ae03 c19a852ee6
commit baa58990c8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 230 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lerna.json
View file

@ -1,5 +1,5 @@
{
"version": "2.21.2",
"version": "2.21.3",
"npmClient": "yarn",
"packages": [
"packages/*",

21
packages/backend-core/src/redis/redis.ts
View file

@ -291,23 +291,16 @@ class RedisWrapper {
return acc
}, {} as Record<string, any>)
const luaScript = `
for i, key in ipairs(KEYS) do
redis.call('MSET', key, ARGV[i])
${
expirySeconds !== null
? `redis.call('EXPIRE', key, ARGV[#ARGV])`
: ""
}
end
`
const keys = Object.keys(dataToStore)
const values = Object.values(dataToStore)
const pipeline = client.pipeline()
pipeline.mset(dataToStore)
if (expirySeconds !== null) {
values.push(expirySeconds)
for (const key of Object.keys(dataToStore)) {
pipeline.expire(key, expirySeconds)
}
}
await client.eval(luaScript, keys.length, ...keys, ...values)
await pipeline.exec()
}
async getTTL(key: string) {

52
packages/server/src/api/controllers/row/alias.ts
View file

@ -1,12 +1,27 @@
import {
QueryJson,
SearchFilters,
Table,
Row,
Datasource,
DatasourcePlusQueryResponse,
Operation,
QueryJson,
Row,
SearchFilters,
} from "@budibase/types"
import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { getSQLClient } from "../../../sdk/app/rows/utils"
import { cloneDeep } from "lodash"
import sdk from "../../../sdk"
import { makeExternalQuery } from "../../../integrations/base/query"
import { SqlClient } from "../../../integrations/utils"
const WRITE_OPERATIONS: Operation[] = [
Operation.CREATE,
Operation.UPDATE,
Operation.DELETE,
]
const DISABLED_WRITE_CLIENTS: SqlClient[] = [
SqlClient.MY_SQL,
SqlClient.MS_SQL,
SqlClient.ORACLE,
]
class CharSequence {
static alphabet = "abcdefghijklmnopqrstuvwxyz"
@ -43,6 +58,25 @@ export default class AliasTables {
this.charSeq = new CharSequence()
}
isAliasingEnabled(json: QueryJson, datasource: Datasource) {
const fieldLength = json.resource?.fields?.length
if (!fieldLength || fieldLength <= 0) {
return false
}
try {
const sqlClient = getSQLClient(datasource)
const isWrite = WRITE_OPERATIONS.includes(json.endpoint.operation)
const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient)
if (isWrite && isDisabledClient) {
return false
}
} catch (err) {
// if we can't get an SQL client, we can't alias
return false
}
return true
}
getAlias(tableName: string) {
if (this.aliases[tableName]) {
return this.aliases[tableName]
@ -111,8 +145,10 @@ export default class AliasTables {
}
async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
const fieldLength = json.resource?.fields?.length
const aliasingEnabled = fieldLength && fieldLength > 0
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
const aliasingEnabled = this.isAliasingEnabled(json, datasource)
if (aliasingEnabled) {
json = cloneDeep(json)
// run through the query json to update anywhere a table may be used
@ -158,7 +194,7 @@ export default class AliasTables {
}
json.tableAliases = invertedTableAliases
}
const response = await getDatasourceAndQuery(json)
const response = await makeExternalQuery(datasource, json)
if (Array.isArray(response) && aliasingEnabled) {
return this.reverse(response)
} else {

10
packages/server/src/integrations/base/sql.ts
View file

@ -435,10 +435,12 @@ class InternalBuilder {
aliases?: QueryJson["tableAliases"]
): Knex.QueryBuilder {
const tableName = endpoint.entityId
const tableAliased = aliases?.[tableName]
? `${tableName} as ${aliases?.[tableName]}`
: tableName
let query = knex(tableAliased)
const tableAlias = aliases?.[tableName]
let table: string | Record<string, string> = tableName
if (tableAlias) {
table = { [tableAlias]: tableName }
}
let query = knex(table)
if (endpoint.schema) {
query = query.withSchema(endpoint.schema)
}

48
packages/server/src/integrations/index.ts
View file

@ -14,7 +14,12 @@ import firebase from "./firebase"
import redis from "./redis"
import snowflake from "./snowflake"
import oracle from "./oracle"
import { SourceName, Integration, PluginType } from "@budibase/types"
import {
SourceName,
Integration,
PluginType,
IntegrationBase,
} from "@budibase/types"
import { getDatasourcePlugin } from "../utilities/fileSystem"
import env from "../environment"
import cloneDeep from "lodash/cloneDeep"
@ -40,25 +45,28 @@ const DEFINITIONS: Record<SourceName, Integration | undefined> = {
[SourceName.BUDIBASE]: undefined,
}
const INTEGRATIONS: Record<SourceName, any> = {
[SourceName.POSTGRES]: postgres.integration,
[SourceName.DYNAMODB]: dynamodb.integration,
[SourceName.MONGODB]: mongodb.integration,
[SourceName.ELASTICSEARCH]: elasticsearch.integration,
[SourceName.COUCHDB]: couchdb.integration,
[SourceName.SQL_SERVER]: sqlServer.integration,
[SourceName.S3]: s3.integration,
[SourceName.AIRTABLE]: airtable.integration,
[SourceName.MYSQL]: mysql.integration,
[SourceName.ARANGODB]: arangodb.integration,
[SourceName.REST]: rest.integration,
[SourceName.FIRESTORE]: firebase.integration,
[SourceName.GOOGLE_SHEETS]: googlesheets.integration,
[SourceName.REDIS]: redis.integration,
[SourceName.SNOWFLAKE]: snowflake.integration,
[SourceName.ORACLE]: undefined,
[SourceName.BUDIBASE]: undefined,
}
type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase
const INTEGRATIONS: Record<SourceName, IntegrationBaseConstructor | undefined> =
{
[SourceName.POSTGRES]: postgres.integration,
[SourceName.DYNAMODB]: dynamodb.integration,
[SourceName.MONGODB]: mongodb.integration,
[SourceName.ELASTICSEARCH]: elasticsearch.integration,
[SourceName.COUCHDB]: couchdb.integration,
[SourceName.SQL_SERVER]: sqlServer.integration,
[SourceName.S3]: s3.integration,
[SourceName.AIRTABLE]: airtable.integration,
[SourceName.MYSQL]: mysql.integration,
[SourceName.ARANGODB]: arangodb.integration,
[SourceName.REST]: rest.integration,
[SourceName.FIRESTORE]: firebase.integration,
[SourceName.GOOGLE_SHEETS]: googlesheets.integration,
[SourceName.REDIS]: redis.integration,
[SourceName.SNOWFLAKE]: snowflake.integration,
[SourceName.ORACLE]: undefined,
[SourceName.BUDIBASE]: undefined,
}
// optionally add oracle integration if the oracle binary can be installed
if (

110
packages/server/src/integrations/tests/sqlAlias.spec.ts
View file

@ -1,4 +1,4 @@
import { QueryJson } from "@budibase/types"
import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types"
import { join } from "path"
import Sql from "../base/sql"
import { SqlClient } from "../utils"
@ -198,6 +198,114 @@ describe("Captures of real examples", () => {
})
})
describe("check aliasing is disabled/enabled", () => {
const tables = ["tableA", "tableB"]
function getDatasource(source: SourceName): Datasource {
return {
source,
type: "datasource",
isSQL: true,
}
}
function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson {
return {
endpoint: { datasourceId: "", entityId: "", operation: op },
resource: {
fields,
},
}
}
it("should check for Postgres aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.POSTGRES)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(true)
})
it("should check for MS-SQL aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.SQL_SERVER)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(false)
})
it("should check for MySQL aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.MYSQL)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(false)
})
it("should check for Oracle aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.ORACLE)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(false)
})
it("should disable aliasing for non-SQL datasources", () => {
const aliasing = new AliasTables(tables)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), {
source: SourceName.GOOGLE_SHEETS,
type: "datasource",
isSQL: false,
})
)
})
it("should disable when no fields", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.POSTGRES)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource)
).toEqual(false)
})
})
describe("check some edge cases", () => {
const tableNames = ["hello", "world"]

36
packages/server/src/sdk/app/rows/utils.ts
View file

@ -1,17 +1,51 @@
import cloneDeep from "lodash/cloneDeep"
import validateJs from "validate.js"
import {
Datasource,
DatasourcePlusQueryResponse,
FieldType,
QueryJson,
Row,
SourceName,
Table,
TableSchema,
DatasourcePlusQueryResponse,
} from "@budibase/types"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Format } from "../../../api/controllers/view/exporters"
import sdk from "../.."
import { isRelationshipColumn } from "../../../db/utils"
import { SqlClient } from "../../../integrations/utils"
const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
[SourceName.POSTGRES]: SqlClient.POSTGRES,
[SourceName.MYSQL]: SqlClient.MY_SQL,
[SourceName.SQL_SERVER]: SqlClient.MS_SQL,
[SourceName.ORACLE]: SqlClient.ORACLE,
[SourceName.DYNAMODB]: undefined,
[SourceName.MONGODB]: undefined,
[SourceName.ELASTICSEARCH]: undefined,
[SourceName.COUCHDB]: undefined,
[SourceName.S3]: undefined,
[SourceName.AIRTABLE]: undefined,
[SourceName.ARANGODB]: undefined,
[SourceName.REST]: undefined,
[SourceName.FIRESTORE]: undefined,
[SourceName.GOOGLE_SHEETS]: undefined,
[SourceName.REDIS]: undefined,
[SourceName.SNOWFLAKE]: undefined,
[SourceName.BUDIBASE]: undefined,
}
export function getSQLClient(datasource: Datasource): SqlClient {
if (!datasource.isSQL) {
throw new Error("Cannot get SQL Client for non-SQL datasource")
}
const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source]
if (lookup) {
return lookup
}
throw new Error("Unable to determine client for SQL datasource")
}
export async function getDatasourceAndQuery(
json: QueryJson