Merge branch 'master' into BUDI-8508/sql-support-for-logical-operators
This commit is contained in:
Adria Navarro
GitHub
commit
9743aca715
26 changed files with 1017 additions and 733 deletions
1
.github/workflows/budibase_ci.yml
vendored
1
.github/workflows/budibase_ci.yml
vendored
|
|
@ -175,6 +175,7 @@ jobs:
|
|||
docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} &
|
||||
docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} &
|
||||
docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} &
|
||||
docker pull budibase/oracle-database:23.2-slim-faststart &
|
||||
docker pull minio/minio &
|
||||
docker pull redis &
|
||||
docker pull testcontainers/ryuk:0.5.1 &
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
|
||||
"version": "2.29.27",
|
||||
"version": "2.29.29",
|
||||
"npmClient": "yarn",
|
||||
"packages": [
|
||||
"packages/*",
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,87 +0,0 @@
|
|||
import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types"
|
||||
import { Knex } from "knex"
|
||||
|
||||
export class SqlStatements {
|
||||
client: string
|
||||
table: Table
|
||||
allOr: boolean | undefined
|
||||
columnPrefix: string | undefined
|
||||
|
||||
constructor(
|
||||
client: string,
|
||||
table: Table,
|
||||
{ allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {}
|
||||
) {
|
||||
this.client = client
|
||||
this.table = table
|
||||
this.allOr = allOr
|
||||
this.columnPrefix = columnPrefix
|
||||
}
|
||||
|
||||
getField(key: string): FieldSchema | undefined {
|
||||
const fieldName = key.split(".")[1]
|
||||
let found = this.table.schema[fieldName]
|
||||
if (!found && this.columnPrefix) {
|
||||
const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "")
|
||||
found = this.table.schema[prefixRemovedFieldName]
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
between(
|
||||
query: Knex.QueryBuilder,
|
||||
key: string,
|
||||
low: number | string,
|
||||
high: number | string
|
||||
) {
|
||||
// Use a between operator if we have 2 valid range values
|
||||
const field = this.getField(key)
|
||||
if (
|
||||
field?.type === FieldType.BIGINT &&
|
||||
this.client === SqlClient.SQL_LITE
|
||||
) {
|
||||
query = query.whereRaw(
|
||||
`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
|
||||
[low, high]
|
||||
)
|
||||
} else {
|
||||
const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
|
||||
query = query[fnc](key, [low, high])
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
lte(query: Knex.QueryBuilder, key: string, low: number | string) {
|
||||
// Use just a single greater than operator if we only have a low
|
||||
const field = this.getField(key)
|
||||
if (
|
||||
field?.type === FieldType.BIGINT &&
|
||||
this.client === SqlClient.SQL_LITE
|
||||
) {
|
||||
query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
|
||||
low,
|
||||
])
|
||||
} else {
|
||||
const fnc = this.allOr ? "orWhere" : "where"
|
||||
query = query[fnc](key, ">=", low)
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
gte(query: Knex.QueryBuilder, key: string, high: number | string) {
|
||||
const field = this.getField(key)
|
||||
// Use just a single less than operator if we only have a high
|
||||
if (
|
||||
field?.type === FieldType.BIGINT &&
|
||||
this.client === SqlClient.SQL_LITE
|
||||
) {
|
||||
query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
|
||||
high,
|
||||
])
|
||||
} else {
|
||||
const fnc = this.allOr ? "orWhere" : "where"
|
||||
query = query[fnc](key, "<=", high)
|
||||
}
|
||||
return query
|
||||
}
|
||||
}
|
||||
|
|
@ -22,6 +22,7 @@ export function getNativeSql(
|
|||
query: Knex.SchemaBuilder | Knex.QueryBuilder
|
||||
): SqlQuery | SqlQuery[] {
|
||||
let sql = query.toSQL()
|
||||
|
||||
if (Array.isArray(sql)) {
|
||||
return sql as SqlQuery[]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
const executeMock = jest.fn(() => ({
|
||||
rows: [
|
||||
{
|
||||
a: "string",
|
||||
b: 1,
|
||||
},
|
||||
],
|
||||
}))
|
||||
|
||||
const closeMock = jest.fn()
|
||||
|
||||
class Connection {
|
||||
execute = executeMock
|
||||
close = closeMock
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getConnection: jest.fn(() => new Connection()),
|
||||
executeMock,
|
||||
closeMock,
|
||||
}
|
||||
|
|
@ -6,9 +6,9 @@ services:
|
|||
db:
|
||||
restart: unless-stopped
|
||||
platform: linux/x86_64
|
||||
image: container-registry.oracle.com/database/express:18.4.0-xe
|
||||
image: gvenzl/oracle-free:23.2-slim-faststart
|
||||
environment:
|
||||
ORACLE_PWD: oracle
|
||||
ORACLE_PWD: Password1
|
||||
ports:
|
||||
- 1521:1521
|
||||
- 5500:5500
|
||||
|
|
|
|||
|
|
@ -22,9 +22,13 @@ describe.each(
|
|||
DatabaseName.MYSQL,
|
||||
DatabaseName.SQL_SERVER,
|
||||
DatabaseName.MARIADB,
|
||||
DatabaseName.ORACLE,
|
||||
].map(name => [name, getDatasource(name)])
|
||||
)("queries (%s)", (dbName, dsProvider) => {
|
||||
const config = setup.getConfig()
|
||||
const isOracle = dbName === DatabaseName.ORACLE
|
||||
const isMsSQL = dbName === DatabaseName.SQL_SERVER
|
||||
|
||||
let rawDatasource: Datasource
|
||||
let datasource: Datasource
|
||||
let client: Knex
|
||||
|
|
@ -97,7 +101,7 @@ describe.each(
|
|||
const query = await createQuery({
|
||||
name: "New Query",
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table",
|
||||
sql: client("test_table").select("*").toString(),
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -106,7 +110,7 @@ describe.each(
|
|||
name: "New Query",
|
||||
parameters: [],
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table",
|
||||
sql: client("test_table").select("*").toString(),
|
||||
},
|
||||
schema: {},
|
||||
queryVerb: "read",
|
||||
|
|
@ -125,7 +129,7 @@ describe.each(
|
|||
it("should be able to update a query", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table",
|
||||
sql: client("test_table").select("*").toString(),
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -135,7 +139,7 @@ describe.each(
|
|||
...query,
|
||||
name: "Updated Query",
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table WHERE id = 1",
|
||||
sql: client("test_table").where({ id: 1 }).toString(),
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -144,7 +148,7 @@ describe.each(
|
|||
name: "Updated Query",
|
||||
parameters: [],
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table WHERE id = 1",
|
||||
sql: client("test_table").where({ id: 1 }).toString(),
|
||||
},
|
||||
schema: {},
|
||||
queryVerb: "read",
|
||||
|
|
@ -161,7 +165,7 @@ describe.each(
|
|||
it("should be able to delete a query", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table",
|
||||
sql: client("test_table").select("*").toString(),
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -180,7 +184,7 @@ describe.each(
|
|||
it("should be able to list queries", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table",
|
||||
sql: client("test_table").select("*").toString(),
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -191,7 +195,7 @@ describe.each(
|
|||
it("should strip sensitive fields for prod apps", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table",
|
||||
sql: client("test_table").select("*").toString(),
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -212,7 +216,7 @@ describe.each(
|
|||
datasourceId: datasource._id!,
|
||||
queryVerb: "read",
|
||||
fields: {
|
||||
sql: `SELECT * FROM test_table WHERE id = 1`,
|
||||
sql: client("test_table").where({ id: 1 }).toString(),
|
||||
},
|
||||
parameters: [],
|
||||
transformer: "return data",
|
||||
|
|
@ -270,7 +274,7 @@ describe.each(
|
|||
name: "Test Query",
|
||||
queryVerb: "read",
|
||||
fields: {
|
||||
sql: `SELECT * FROM ${tableName}`,
|
||||
sql: client(tableName).select("*").toString(),
|
||||
},
|
||||
parameters: [],
|
||||
transformer: "return data",
|
||||
|
|
@ -284,11 +288,13 @@ describe.each(
|
|||
})
|
||||
)
|
||||
|
||||
await client(tableName).delete()
|
||||
await client.schema.alterTable(tableName, table => {
|
||||
table.string("data").alter()
|
||||
})
|
||||
|
||||
await client(tableName).update({
|
||||
await client(tableName).insert({
|
||||
name: "test",
|
||||
data: "string value",
|
||||
})
|
||||
|
||||
|
|
@ -297,7 +303,7 @@ describe.each(
|
|||
name: "Test Query",
|
||||
queryVerb: "read",
|
||||
fields: {
|
||||
sql: `SELECT * FROM ${tableName}`,
|
||||
sql: client(tableName).select("*").toString(),
|
||||
},
|
||||
parameters: [],
|
||||
transformer: "return data",
|
||||
|
|
@ -311,6 +317,7 @@ describe.each(
|
|||
})
|
||||
)
|
||||
})
|
||||
|
||||
it("should work with static variables", async () => {
|
||||
await config.api.datasource.update({
|
||||
...datasource,
|
||||
|
|
@ -326,7 +333,7 @@ describe.each(
|
|||
datasourceId: datasource._id!,
|
||||
queryVerb: "read",
|
||||
fields: {
|
||||
sql: `SELECT '{{ foo }}' as foo`,
|
||||
sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
|
||||
},
|
||||
parameters: [],
|
||||
transformer: "return data",
|
||||
|
|
@ -337,16 +344,17 @@ describe.each(
|
|||
|
||||
const response = await config.api.query.preview(request)
|
||||
|
||||
let key = isOracle ? "FOO" : "foo"
|
||||
expect(response.schema).toEqual({
|
||||
foo: {
|
||||
name: "foo",
|
||||
[key]: {
|
||||
name: key,
|
||||
type: "string",
|
||||
},
|
||||
})
|
||||
|
||||
expect(response.rows).toEqual([
|
||||
{
|
||||
foo: "bar",
|
||||
[key]: "bar",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
|
@ -354,7 +362,7 @@ describe.each(
|
|||
it("should work with dynamic variables", async () => {
|
||||
const basedOnQuery = await createQuery({
|
||||
fields: {
|
||||
sql: "SELECT name FROM test_table WHERE id = 1",
|
||||
sql: client("test_table").select("name").where({ id: 1 }).toString(),
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -376,7 +384,7 @@ describe.each(
|
|||
datasourceId: datasource._id!,
|
||||
queryVerb: "read",
|
||||
fields: {
|
||||
sql: `SELECT '{{ foo }}' as foo`,
|
||||
sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
|
||||
},
|
||||
parameters: [],
|
||||
transformer: "return data",
|
||||
|
|
@ -385,16 +393,17 @@ describe.each(
|
|||
readable: true,
|
||||
})
|
||||
|
||||
let key = isOracle ? "FOO" : "foo"
|
||||
expect(preview.schema).toEqual({
|
||||
foo: {
|
||||
name: "foo",
|
||||
[key]: {
|
||||
name: key,
|
||||
type: "string",
|
||||
},
|
||||
})
|
||||
|
||||
expect(preview.rows).toEqual([
|
||||
{
|
||||
foo: "one",
|
||||
[key]: "one",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
|
@ -402,7 +411,7 @@ describe.each(
|
|||
it("should handle the dynamic base query being deleted", async () => {
|
||||
const basedOnQuery = await createQuery({
|
||||
fields: {
|
||||
sql: "SELECT name FROM test_table WHERE id = 1",
|
||||
sql: client("test_table").select("name").where({ id: 1 }).toString(),
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -426,7 +435,7 @@ describe.each(
|
|||
datasourceId: datasource._id!,
|
||||
queryVerb: "read",
|
||||
fields: {
|
||||
sql: `SELECT '{{ foo }}' as foo`,
|
||||
sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
|
||||
},
|
||||
parameters: [],
|
||||
transformer: "return data",
|
||||
|
|
@ -435,16 +444,17 @@ describe.each(
|
|||
readable: true,
|
||||
})
|
||||
|
||||
let key = isOracle ? "FOO" : "foo"
|
||||
expect(preview.schema).toEqual({
|
||||
foo: {
|
||||
name: "foo",
|
||||
[key]: {
|
||||
name: key,
|
||||
type: "string",
|
||||
},
|
||||
})
|
||||
|
||||
expect(preview.rows).toEqual([
|
||||
{
|
||||
foo: datasource.source === SourceName.SQL_SERVER ? "" : null,
|
||||
[key]: datasource.source === SourceName.SQL_SERVER ? "" : null,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
|
@ -455,7 +465,7 @@ describe.each(
|
|||
it("should be able to insert with bindings", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
|
||||
sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
|
||||
},
|
||||
parameters: [
|
||||
{
|
||||
|
|
@ -488,7 +498,7 @@ describe.each(
|
|||
it("should not allow handlebars as parameters", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
|
||||
sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
|
||||
},
|
||||
parameters: [
|
||||
{
|
||||
|
|
@ -516,46 +526,55 @@ describe.each(
|
|||
)
|
||||
})
|
||||
|
||||
it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
|
||||
"should coerce %s into a date",
|
||||
async datetimeStr => {
|
||||
const date = new Date(datetimeStr)
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: `INSERT INTO test_table (name, birthday) VALUES ('foo', {{ birthday }})`,
|
||||
},
|
||||
parameters: [
|
||||
{
|
||||
name: "birthday",
|
||||
default: "",
|
||||
// Oracle doesn't automatically coerce strings into dates.
|
||||
!isOracle &&
|
||||
it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
|
||||
"should coerce %s into a date",
|
||||
async datetimeStr => {
|
||||
const date = new Date(datetimeStr)
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: client("test_table")
|
||||
.insert({
|
||||
name: "foo",
|
||||
birthday: client.raw("{{ birthday }}"),
|
||||
})
|
||||
.toString(),
|
||||
},
|
||||
],
|
||||
queryVerb: "create",
|
||||
})
|
||||
parameters: [
|
||||
{
|
||||
name: "birthday",
|
||||
default: "",
|
||||
},
|
||||
],
|
||||
queryVerb: "create",
|
||||
})
|
||||
|
||||
const result = await config.api.query.execute(query._id!, {
|
||||
parameters: { birthday: datetimeStr },
|
||||
})
|
||||
const result = await config.api.query.execute(query._id!, {
|
||||
parameters: { birthday: datetimeStr },
|
||||
})
|
||||
|
||||
expect(result.data).toEqual([{ created: true }])
|
||||
expect(result.data).toEqual([{ created: true }])
|
||||
|
||||
const rows = await client("test_table")
|
||||
.where({ birthday: datetimeStr })
|
||||
.select()
|
||||
expect(rows).toHaveLength(1)
|
||||
const rows = await client("test_table")
|
||||
.where({ birthday: datetimeStr })
|
||||
.select()
|
||||
expect(rows).toHaveLength(1)
|
||||
|
||||
for (const row of rows) {
|
||||
expect(new Date(row.birthday)).toEqual(date)
|
||||
for (const row of rows) {
|
||||
expect(new Date(row.birthday)).toEqual(date)
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
it.each(["2021,02,05", "202205-1500"])(
|
||||
"should not coerce %s as a date",
|
||||
async notDateStr => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "INSERT INTO test_table (name) VALUES ({{ name }})",
|
||||
sql: client("test_table")
|
||||
.insert({ name: client.raw("{{ name }}") })
|
||||
.toString(),
|
||||
},
|
||||
parameters: [
|
||||
{
|
||||
|
|
@ -586,7 +605,7 @@ describe.each(
|
|||
it("should execute a query", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table ORDER BY id",
|
||||
sql: client("test_table").select("*").orderBy("id").toString(),
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -629,7 +648,7 @@ describe.each(
|
|||
it("should be able to transform a query", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table WHERE id = 1",
|
||||
sql: client("test_table").where({ id: 1 }).select("*").toString(),
|
||||
},
|
||||
transformer: `
|
||||
data[0].id = data[0].id + 1;
|
||||
|
|
@ -652,7 +671,10 @@ describe.each(
|
|||
it("should coerce numeric bindings", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "SELECT * FROM test_table WHERE id = {{ id }}",
|
||||
sql: client("test_table")
|
||||
.where({ id: client.raw("{{ id }}") })
|
||||
.select("*")
|
||||
.toString(),
|
||||
},
|
||||
parameters: [
|
||||
{
|
||||
|
|
@ -683,7 +705,10 @@ describe.each(
|
|||
it("should be able to update rows", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
|
||||
sql: client("test_table")
|
||||
.update({ name: client.raw("{{ name }}") })
|
||||
.where({ id: client.raw("{{ id }}") })
|
||||
.toString(),
|
||||
},
|
||||
parameters: [
|
||||
{
|
||||
|
|
@ -698,19 +723,13 @@ describe.each(
|
|||
queryVerb: "update",
|
||||
})
|
||||
|
||||
const result = await config.api.query.execute(query._id!, {
|
||||
await config.api.query.execute(query._id!, {
|
||||
parameters: {
|
||||
id: "1",
|
||||
name: "foo",
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.data).toEqual([
|
||||
{
|
||||
updated: true,
|
||||
},
|
||||
])
|
||||
|
||||
const rows = await client("test_table").where({ id: 1 }).select()
|
||||
expect(rows).toEqual([
|
||||
{ id: 1, name: "foo", birthday: null, number: null },
|
||||
|
|
@ -720,35 +739,34 @@ describe.each(
|
|||
it("should be able to execute an update that updates no rows", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
|
||||
sql: client("test_table")
|
||||
.update({ name: "updated" })
|
||||
.where({ id: 100 })
|
||||
.toString(),
|
||||
},
|
||||
queryVerb: "update",
|
||||
})
|
||||
|
||||
const result = await config.api.query.execute(query._id!)
|
||||
await config.api.query.execute(query._id!)
|
||||
|
||||
expect(result.data).toEqual([
|
||||
{
|
||||
updated: true,
|
||||
},
|
||||
])
|
||||
const rows = await client("test_table").select()
|
||||
for (const row of rows) {
|
||||
expect(row.name).not.toEqual("updated")
|
||||
}
|
||||
})
|
||||
|
||||
it("should be able to execute a delete that deletes no rows", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "DELETE FROM test_table WHERE id = 100",
|
||||
sql: client("test_table").where({ id: 100 }).delete().toString(),
|
||||
},
|
||||
queryVerb: "delete",
|
||||
})
|
||||
|
||||
const result = await config.api.query.execute(query._id!)
|
||||
await config.api.query.execute(query._id!)
|
||||
|
||||
expect(result.data).toEqual([
|
||||
{
|
||||
deleted: true,
|
||||
},
|
||||
])
|
||||
const rows = await client("test_table").select()
|
||||
expect(rows).toHaveLength(5)
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -756,7 +774,10 @@ describe.each(
|
|||
it("should be able to delete rows", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "DELETE FROM test_table WHERE id = {{ id }}",
|
||||
sql: client("test_table")
|
||||
.where({ id: client.raw("{{ id }}") })
|
||||
.delete()
|
||||
.toString(),
|
||||
},
|
||||
parameters: [
|
||||
{
|
||||
|
|
@ -767,18 +788,12 @@ describe.each(
|
|||
queryVerb: "delete",
|
||||
})
|
||||
|
||||
const result = await config.api.query.execute(query._id!, {
|
||||
await config.api.query.execute(query._id!, {
|
||||
parameters: {
|
||||
id: "1",
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.data).toEqual([
|
||||
{
|
||||
deleted: true,
|
||||
},
|
||||
])
|
||||
|
||||
const rows = await client("test_table").where({ id: 1 }).select()
|
||||
expect(rows).toHaveLength(0)
|
||||
})
|
||||
|
|
@ -823,72 +838,63 @@ describe.each(
|
|||
})
|
||||
})
|
||||
|
||||
it("should be able to execute an update that updates no rows", async () => {
|
||||
const query = await createQuery({
|
||||
fields: {
|
||||
sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
|
||||
},
|
||||
queryVerb: "update",
|
||||
// this parameter really only impacts SQL queries
|
||||
describe("confirm nullDefaultSupport", () => {
|
||||
let queryParams: Partial<Query>
|
||||
beforeAll(async () => {
|
||||
queryParams = {
|
||||
fields: {
|
||||
sql: client("test_table")
|
||||
.insert({
|
||||
name: client.raw("{{ bindingName }}"),
|
||||
number: client.raw("{{ bindingNumber }}"),
|
||||
})
|
||||
.toString(),
|
||||
},
|
||||
parameters: [
|
||||
{
|
||||
name: "bindingName",
|
||||
default: "",
|
||||
},
|
||||
{
|
||||
name: "bindingNumber",
|
||||
default: "",
|
||||
},
|
||||
],
|
||||
queryVerb: "create",
|
||||
}
|
||||
})
|
||||
|
||||
const result = await config.api.query.execute(query._id!, {})
|
||||
it("should error for old queries", async () => {
|
||||
const query = await createQuery(queryParams)
|
||||
await config.api.query.save({ ...query, nullDefaultSupport: false })
|
||||
let error: string | undefined
|
||||
try {
|
||||
await config.api.query.execute(query._id!, {
|
||||
parameters: {
|
||||
bindingName: "testing",
|
||||
},
|
||||
})
|
||||
} catch (err: any) {
|
||||
error = err.message
|
||||
}
|
||||
if (isMsSQL || isOracle) {
|
||||
expect(error).toBeUndefined()
|
||||
} else {
|
||||
expect(error).toBeDefined()
|
||||
expect(error).toContain("integer")
|
||||
}
|
||||
})
|
||||
|
||||
expect(result.data).toEqual([
|
||||
{
|
||||
updated: true,
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
// this parameter really only impacts SQL queries
|
||||
describe("confirm nullDefaultSupport", () => {
|
||||
const queryParams = {
|
||||
fields: {
|
||||
sql: "INSERT INTO test_table (name, number) VALUES ({{ bindingName }}, {{ bindingNumber }})",
|
||||
},
|
||||
parameters: [
|
||||
{
|
||||
name: "bindingName",
|
||||
default: "",
|
||||
},
|
||||
{
|
||||
name: "bindingNumber",
|
||||
default: "",
|
||||
},
|
||||
],
|
||||
queryVerb: "create",
|
||||
}
|
||||
|
||||
it("should error for old queries", async () => {
|
||||
const query = await createQuery(queryParams)
|
||||
await config.api.query.save({ ...query, nullDefaultSupport: false })
|
||||
let error: string | undefined
|
||||
try {
|
||||
await config.api.query.execute(query._id!, {
|
||||
it("should not error for new queries", async () => {
|
||||
const query = await createQuery(queryParams)
|
||||
const results = await config.api.query.execute(query._id!, {
|
||||
parameters: {
|
||||
bindingName: "testing",
|
||||
},
|
||||
})
|
||||
} catch (err: any) {
|
||||
error = err.message
|
||||
}
|
||||
if (dbName === "mssql") {
|
||||
expect(error).toBeUndefined()
|
||||
} else {
|
||||
expect(error).toBeDefined()
|
||||
expect(error).toContain("integer")
|
||||
}
|
||||
})
|
||||
|
||||
it("should not error for new queries", async () => {
|
||||
const query = await createQuery(queryParams)
|
||||
const results = await config.api.query.execute(query._id!, {
|
||||
parameters: {
|
||||
bindingName: "testing",
|
||||
},
|
||||
expect(results).toEqual({ data: [{ created: true }] })
|
||||
})
|
||||
expect(results).toEqual({ data: [{ created: true }] })
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -72,9 +72,11 @@ describe.each([
|
|||
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
|
||||
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
|
||||
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
|
||||
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
|
||||
])("/rows (%s)", (providerType, dsProvider) => {
|
||||
const isInternal = dsProvider === undefined
|
||||
const isMSSQL = providerType === DatabaseName.SQL_SERVER
|
||||
const isOracle = providerType === DatabaseName.ORACLE
|
||||
const config = setup.getConfig()
|
||||
|
||||
let table: Table
|
||||
|
|
@ -129,7 +131,8 @@ describe.each([
|
|||
primary: ["id"],
|
||||
schema: defaultSchema,
|
||||
}
|
||||
return merge(req, ...overrides)
|
||||
const merged = merge(req, ...overrides)
|
||||
return merged
|
||||
}
|
||||
|
||||
function defaultTable(
|
||||
|
|
@ -1406,9 +1409,10 @@ describe.each([
|
|||
await assertRowUsage(rowUsage + 3)
|
||||
})
|
||||
|
||||
// Upserting isn't yet supported in MSSQL, see:
|
||||
// Upserting isn't yet supported in MSSQL / Oracle, see:
|
||||
// https://github.com/knex/knex/pull/6050
|
||||
!isMSSQL &&
|
||||
!isOracle &&
|
||||
it("should be able to update existing rows with bulkImport", async () => {
|
||||
const table = await config.api.table.save(
|
||||
saveTableRequest({
|
||||
|
|
@ -1478,9 +1482,10 @@ describe.each([
|
|||
expect(rows[2].description).toEqual("Row 3 description")
|
||||
})
|
||||
|
||||
// Upserting isn't yet supported in MSSQL, see:
|
||||
// Upserting isn't yet supported in MSSQL or Oracle, see:
|
||||
// https://github.com/knex/knex/pull/6050
|
||||
!isMSSQL &&
|
||||
!isOracle &&
|
||||
!isInternal &&
|
||||
it("should be able to update existing rows with composite primary keys with bulkImport", async () => {
|
||||
const tableName = uuid.v4()
|
||||
|
|
@ -1547,9 +1552,10 @@ describe.each([
|
|||
expect(rows[2].description).toEqual("Row 3 description")
|
||||
})
|
||||
|
||||
// Upserting isn't yet supported in MSSQL, see:
|
||||
// Upserting isn't yet supported in MSSQL/Oracle, see:
|
||||
// https://github.com/knex/knex/pull/6050
|
||||
!isMSSQL &&
|
||||
!isOracle &&
|
||||
!isInternal &&
|
||||
it("should be able to update existing rows an autoID primary key", async () => {
|
||||
const tableName = uuid.v4()
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ describe.each([
|
|||
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
|
||||
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
|
||||
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
|
||||
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
|
||||
])("search (%s)", (name, dsProvider) => {
|
||||
const isSqs = name === "sqs"
|
||||
const isLucene = name === "lucene"
|
||||
|
|
@ -1594,7 +1595,10 @@ describe.each([
|
|||
const MEDIUM = "10000000"
|
||||
|
||||
// Our bigints are int64s in most datasources.
|
||||
const BIG = "9223372036854775807"
|
||||
let BIG = "9223372036854775807"
|
||||
if (name === DatabaseName.ORACLE) {
|
||||
// BIG = "9223372036854775808"
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
table = await createTable({
|
||||
|
|
|
|||
|
|
@ -40,7 +40,8 @@ describe.each([
|
|||
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
|
||||
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
|
||||
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
|
||||
])("/tables (%s)", (_, dsProvider) => {
|
||||
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
|
||||
])("/tables (%s)", (name, dsProvider) => {
|
||||
const isInternal: boolean = !dsProvider
|
||||
let datasource: Datasource | undefined
|
||||
let config = setup.getConfig()
|
||||
|
|
@ -59,15 +60,20 @@ describe.each([
|
|||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it.each([
|
||||
let names = [
|
||||
"alphanum",
|
||||
"with spaces",
|
||||
"with-dashes",
|
||||
"with_underscores",
|
||||
'with "double quotes"',
|
||||
"with 'single quotes'",
|
||||
"with `backticks`",
|
||||
])("creates a table with name: %s", async name => {
|
||||
]
|
||||
|
||||
if (name !== DatabaseName.ORACLE) {
|
||||
names.push(`with "double quotes"`)
|
||||
names.push(`with 'single quotes'`)
|
||||
}
|
||||
|
||||
it.each(names)("creates a table with name: %s", async name => {
|
||||
const table = await config.api.table.save(
|
||||
tableForDatasource(datasource, { name })
|
||||
)
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ describe.each([
|
|||
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
|
||||
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
|
||||
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
|
||||
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
|
||||
])("/v2/views (%s)", (name, dsProvider) => {
|
||||
const config = setup.getConfig()
|
||||
const isSqs = name === "sqs"
|
||||
|
|
|
|||
|
|
@ -1,27 +1,20 @@
|
|||
import { Datasource, Query, SourceName } from "@budibase/types"
|
||||
import { Datasource, Query } from "@budibase/types"
|
||||
import * as setup from "./utilities"
|
||||
import { DatabaseName, getDatasource } from "../../integrations/tests/utils"
|
||||
import knex, { Knex } from "knex"
|
||||
import {
|
||||
DatabaseName,
|
||||
getDatasource,
|
||||
knexClient,
|
||||
} from "../../integrations/tests/utils"
|
||||
import { Knex } from "knex"
|
||||
import { generator } from "@budibase/backend-core/tests"
|
||||
|
||||
function getKnexClientName(source: SourceName) {
|
||||
switch (source) {
|
||||
case SourceName.MYSQL:
|
||||
return "mysql2"
|
||||
case SourceName.SQL_SERVER:
|
||||
return "mssql"
|
||||
case SourceName.POSTGRES:
|
||||
return "pg"
|
||||
}
|
||||
throw new Error(`Unsupported source: ${source}`)
|
||||
}
|
||||
|
||||
describe.each(
|
||||
[
|
||||
DatabaseName.POSTGRES,
|
||||
DatabaseName.MYSQL,
|
||||
DatabaseName.SQL_SERVER,
|
||||
DatabaseName.MARIADB,
|
||||
DatabaseName.ORACLE,
|
||||
].map(name => [name, getDatasource(name)])
|
||||
)("execute query action (%s)", (_, dsProvider) => {
|
||||
let tableName: string
|
||||
|
|
@ -35,10 +28,7 @@ describe.each(
|
|||
|
||||
const ds = await dsProvider
|
||||
datasource = await config.api.datasource.create(ds)
|
||||
client = knex({
|
||||
client: getKnexClientName(ds.source),
|
||||
connection: ds.config,
|
||||
})
|
||||
client = await knexClient(ds)
|
||||
})
|
||||
|
||||
beforeEach(async () => {
|
||||
|
|
|
|||
|
|
@ -104,6 +104,37 @@ export interface OracleColumnsResponse {
|
|||
SEARCH_CONDITION: null | string
|
||||
}
|
||||
|
||||
export enum TriggeringEvent {
|
||||
INSERT = "INSERT",
|
||||
DELETE = "DELETE",
|
||||
UPDATE = "UPDATE",
|
||||
LOGON = "LOGON",
|
||||
LOGOFF = "LOGOFF",
|
||||
STARTUP = "STARTUP",
|
||||
SHUTDOWN = "SHUTDOWN",
|
||||
SERVERERROR = "SERVERERROR",
|
||||
SCHEMA = "SCHEMA",
|
||||
ALTER = "ALTER",
|
||||
DROP = "DROP",
|
||||
}
|
||||
|
||||
export enum TriggerType {
|
||||
BEFORE_EACH_ROW = "BEFORE EACH ROW",
|
||||
AFTER_EACH_ROW = "AFTER EACH ROW",
|
||||
BEFORE_STATEMENT = "BEFORE STATEMENT",
|
||||
AFTER_STATEMENT = "AFTER STATEMENT",
|
||||
INSTEAD_OF = "INSTEAD OF",
|
||||
COMPOUND = "COMPOUND",
|
||||
}
|
||||
|
||||
export interface OracleTriggersResponse {
|
||||
TABLE_NAME: string
|
||||
TRIGGER_NAME: string
|
||||
TRIGGER_TYPE: TriggerType
|
||||
TRIGGERING_EVENT: TriggeringEvent
|
||||
TRIGGER_BODY: string
|
||||
}
|
||||
|
||||
/**
|
||||
* An oracle constraint
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -31,7 +31,14 @@ import oracledb, {
|
|||
ExecuteOptions,
|
||||
Result,
|
||||
} from "oracledb"
|
||||
import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
|
||||
import {
|
||||
OracleTable,
|
||||
OracleColumn,
|
||||
OracleColumnsResponse,
|
||||
OracleTriggersResponse,
|
||||
TriggeringEvent,
|
||||
TriggerType,
|
||||
} from "./base/types"
|
||||
import { sql } from "@budibase/backend-core"
|
||||
|
||||
const Sql = sql.Sql
|
||||
|
|
@ -98,7 +105,7 @@ const SCHEMA: Integration = {
|
|||
},
|
||||
}
|
||||
|
||||
const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
|
||||
const UNSUPPORTED_TYPES = ["BLOB", "NCLOB"]
|
||||
|
||||
const OracleContraintTypes = {
|
||||
PRIMARY: "P",
|
||||
|
|
@ -111,7 +118,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
|
|||
private readonly config: OracleConfig
|
||||
private index: number = 1
|
||||
|
||||
private readonly COLUMNS_SQL = `
|
||||
private static readonly COLUMNS_SQL = `
|
||||
SELECT
|
||||
tabs.table_name,
|
||||
cols.column_name,
|
||||
|
|
@ -139,6 +146,19 @@ class OracleIntegration extends Sql implements DatasourcePlus {
|
|||
(cons.status = 'ENABLED'
|
||||
OR cons.status IS NULL)
|
||||
`
|
||||
|
||||
private static readonly TRIGGERS_SQL = `
|
||||
SELECT
|
||||
table_name,
|
||||
trigger_name,
|
||||
trigger_type,
|
||||
triggering_event,
|
||||
trigger_body
|
||||
FROM
|
||||
all_triggers
|
||||
WHERE status = 'ENABLED'
|
||||
`
|
||||
|
||||
constructor(config: OracleConfig) {
|
||||
super(SqlClient.ORACLE)
|
||||
this.config = config
|
||||
|
|
@ -211,6 +231,75 @@ class OracleIntegration extends Sql implements DatasourcePlus {
|
|||
return oracleTables
|
||||
}
|
||||
|
||||
private getTriggersFor(
|
||||
tableName: string,
|
||||
triggersResponse: Result<OracleTriggersResponse>,
|
||||
opts?: { event?: TriggeringEvent; type?: TriggerType }
|
||||
): OracleTriggersResponse[] {
|
||||
const triggers: OracleTriggersResponse[] = []
|
||||
for (const trigger of triggersResponse.rows || []) {
|
||||
if (trigger.TABLE_NAME !== tableName) {
|
||||
continue
|
||||
}
|
||||
if (opts?.event && opts.event !== trigger.TRIGGERING_EVENT) {
|
||||
continue
|
||||
}
|
||||
if (opts?.type && opts.type !== trigger.TRIGGER_TYPE) {
|
||||
continue
|
||||
}
|
||||
triggers.push(trigger)
|
||||
}
|
||||
return triggers
|
||||
}
|
||||
|
||||
private markAutoIncrementColumns(
|
||||
triggersResponse: Result<OracleTriggersResponse>,
|
||||
tables: Record<string, Table>
|
||||
) {
|
||||
for (const table of Object.values(tables)) {
|
||||
const triggers = this.getTriggersFor(table.name, triggersResponse, {
|
||||
type: TriggerType.BEFORE_EACH_ROW,
|
||||
event: TriggeringEvent.INSERT,
|
||||
})
|
||||
|
||||
// This is the trigger body Knex generates for an auto increment column
|
||||
// called "id" on a table called "foo":
|
||||
//
|
||||
// declare checking number := 1;
|
||||
// begin if (:new. "id" is null) then while checking >= 1 loop
|
||||
// select
|
||||
// "foo_seq".nextval into :new. "id"
|
||||
// from
|
||||
// dual;
|
||||
// select
|
||||
// count("id") into checking
|
||||
// from
|
||||
// "foo"
|
||||
// where
|
||||
// "id" = :new. "id";
|
||||
// end loop;
|
||||
// end if;
|
||||
// end;
|
||||
for (const [columnName, schema] of Object.entries(table.schema)) {
|
||||
const autoIncrementTriggers = triggers.filter(
|
||||
trigger =>
|
||||
// This is a bit heuristic, but I think it's the best we can do with
|
||||
// the information we have. We're looking for triggers that run
|
||||
// before each row is inserted, and that have a body that contains a
|
||||
// call to a function that generates a new value for the column. We
|
||||
// also check that the column name is in the trigger body, to make
|
||||
// sure we're not picking up triggers that don't affect the column.
|
||||
trigger.TRIGGER_BODY.includes(`"${columnName}"`) &&
|
||||
trigger.TRIGGER_BODY.includes(`.nextval`)
|
||||
)
|
||||
|
||||
if (autoIncrementTriggers.length > 0) {
|
||||
schema.autocolumn = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static isSupportedColumn(column: OracleColumn) {
|
||||
return !UNSUPPORTED_TYPES.includes(column.type)
|
||||
}
|
||||
|
|
@ -255,7 +344,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
|
|||
entities: Record<string, Table>
|
||||
): Promise<Schema> {
|
||||
const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
|
||||
sql: this.COLUMNS_SQL,
|
||||
sql: OracleIntegration.COLUMNS_SQL,
|
||||
})
|
||||
const triggersResponse = await this.internalQuery<OracleTriggersResponse>({
|
||||
sql: OracleIntegration.TRIGGERS_SQL,
|
||||
})
|
||||
const oracleTables = this.mapColumns(columnsResponse)
|
||||
|
||||
|
|
@ -318,6 +410,8 @@ class OracleIntegration extends Sql implements DatasourcePlus {
|
|||
})
|
||||
})
|
||||
|
||||
this.markAutoIncrementColumns(triggersResponse, tables)
|
||||
|
||||
let externalTables = finaliseExternalTables(tables, entities)
|
||||
let errors = checkExternalTables(externalTables)
|
||||
return { tables: externalTables, errors }
|
||||
|
|
@ -325,7 +419,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
|
|||
|
||||
async getTableNames() {
|
||||
const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
|
||||
sql: this.COLUMNS_SQL,
|
||||
sql: OracleIntegration.COLUMNS_SQL,
|
||||
})
|
||||
return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
|
||||
}
|
||||
|
|
@ -360,11 +454,32 @@ class OracleIntegration extends Sql implements DatasourcePlus {
|
|||
this.index = 1
|
||||
connection = await this.getConnection()
|
||||
|
||||
const options: ExecuteOptions = { autoCommit: true }
|
||||
const options: ExecuteOptions = {
|
||||
autoCommit: true,
|
||||
fetchTypeHandler: function (metaData) {
|
||||
if (metaData.dbType === oracledb.CLOB) {
|
||||
return { type: oracledb.STRING }
|
||||
} else if (
|
||||
// When we create a new table in OracleDB from Budibase, bigints get
|
||||
// created as NUMBER(20,0). Budibase expects bigints to be returned
|
||||
// as strings, which is what we're doing here. However, this is
|
||||
// likely to be brittle if we connect to externally created
|
||||
// databases that have used different precisions and scales.
|
||||
// We shold find a way to do better.
|
||||
metaData.dbType === oracledb.NUMBER &&
|
||||
metaData.precision === 20 &&
|
||||
metaData.scale === 0
|
||||
) {
|
||||
return { type: oracledb.STRING }
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
}
|
||||
const bindings: BindParameters = query.bindings || []
|
||||
|
||||
this.log(query.sql, bindings)
|
||||
return await connection.execute<T>(query.sql, bindings, options)
|
||||
const result = await connection.execute(query.sql, bindings, options)
|
||||
return result as Result<T>
|
||||
} finally {
|
||||
if (connection) {
|
||||
try {
|
||||
|
|
@ -377,7 +492,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
|
|||
}
|
||||
|
||||
private getConnection = async (): Promise<Connection> => {
|
||||
//connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))"
|
||||
const connectString = `${this.config.host}:${this.config.port || 1521}/${
|
||||
this.config.database
|
||||
}`
|
||||
|
|
@ -386,7 +500,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
|
|||
password: this.config.password,
|
||||
connectString,
|
||||
}
|
||||
return oracledb.getConnection(attributes)
|
||||
const tz = Intl.DateTimeFormat().resolvedOptions().timeZone
|
||||
const connection = await oracledb.getConnection(attributes)
|
||||
await connection.execute(`ALTER SESSION SET TIME_ZONE = '${tz}'`)
|
||||
return connection
|
||||
}
|
||||
|
||||
async create(query: SqlQuery | string): Promise<any[]> {
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
const oracledb = require("oracledb")
|
||||
|
||||
import { default as OracleIntegration } from "../oracle"
|
||||
|
||||
jest.mock("oracledb")
|
||||
|
||||
class TestConfiguration {
|
||||
integration: any
|
||||
|
||||
constructor(config: any = {}) {
|
||||
this.integration = new OracleIntegration.integration(config)
|
||||
}
|
||||
}
|
||||
|
||||
const options = { autoCommit: true }
|
||||
|
||||
describe("Oracle Integration", () => {
|
||||
let config: any
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks()
|
||||
config = new TestConfiguration()
|
||||
})
|
||||
|
||||
it("calls the create method with the correct params", async () => {
|
||||
const sql = "insert into users (name, age) values ('Joe', 123);"
|
||||
await config.integration.create({
|
||||
sql,
|
||||
})
|
||||
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
|
||||
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
|
||||
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("calls the read method with the correct params", async () => {
|
||||
const sql = "select * from users;"
|
||||
await config.integration.read({
|
||||
sql,
|
||||
})
|
||||
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
|
||||
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
|
||||
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("calls the update method with the correct params", async () => {
|
||||
const sql = "update table users set name = 'test';"
|
||||
await config.integration.update({
|
||||
sql,
|
||||
})
|
||||
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
|
||||
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
|
||||
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("calls the delete method with the correct params", async () => {
|
||||
const sql = "delete from users where name = 'todelete';"
|
||||
await config.integration.delete({
|
||||
sql,
|
||||
})
|
||||
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
|
||||
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
|
||||
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
describe("no rows returned", () => {
|
||||
beforeEach(() => {
|
||||
oracledb.executeMock.mockImplementation(() => ({ rows: [] }))
|
||||
})
|
||||
|
||||
it("returns the correct response when the create response has no rows", async () => {
|
||||
const sql = "insert into users (name, age) values ('Joe', 123);"
|
||||
const response = await config.integration.create({
|
||||
sql,
|
||||
})
|
||||
expect(response).toEqual([{ created: true }])
|
||||
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
|
||||
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("returns the correct response when the update response has no rows", async () => {
|
||||
const sql = "update table users set name = 'test';"
|
||||
const response = await config.integration.update({
|
||||
sql,
|
||||
})
|
||||
expect(response).toEqual([{ updated: true }])
|
||||
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
|
||||
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("returns the correct response when the delete response has no rows", async () => {
|
||||
const sql = "delete from users where name = 'todelete';"
|
||||
const response = await config.integration.delete({
|
||||
sql,
|
||||
})
|
||||
expect(response).toEqual([{ deleted: true }])
|
||||
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
|
||||
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,12 +1,16 @@
|
|||
import {
|
||||
FieldType,
|
||||
Operation,
|
||||
PaginationJson,
|
||||
QueryJson,
|
||||
SearchFilters,
|
||||
SortJson,
|
||||
SqlClient,
|
||||
Table,
|
||||
TableSourceType,
|
||||
SqlClient,
|
||||
} from "@budibase/types"
|
||||
import { sql } from "@budibase/backend-core"
|
||||
import { merge } from "lodash"
|
||||
|
||||
const Sql = sql.Sql
|
||||
|
||||
|
|
@ -25,7 +29,16 @@ const TABLE: Table = {
|
|||
primary: ["id"],
|
||||
}
|
||||
|
||||
function endpoint(table: any, operation: any) {
|
||||
const ORACLE_TABLE: Partial<Table> = {
|
||||
schema: {
|
||||
name: {
|
||||
name: "name",
|
||||
type: FieldType.STRING,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
function endpoint(table: string, operation: Operation) {
|
||||
return {
|
||||
datasourceId: "Postgres",
|
||||
operation: operation,
|
||||
|
|
@ -39,19 +52,25 @@ function generateReadJson({
|
|||
filters,
|
||||
sort,
|
||||
paginate,
|
||||
}: any = {}): QueryJson {
|
||||
const tableObj = { ...TABLE }
|
||||
}: {
|
||||
table?: Partial<Table>
|
||||
fields?: string[]
|
||||
filters?: SearchFilters
|
||||
sort?: SortJson
|
||||
paginate?: PaginationJson
|
||||
} = {}): QueryJson {
|
||||
let tableObj: Table = { ...TABLE }
|
||||
if (table) {
|
||||
tableObj.name = table
|
||||
tableObj = merge(TABLE, table)
|
||||
}
|
||||
return {
|
||||
endpoint: endpoint(table || TABLE_NAME, "READ"),
|
||||
endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ),
|
||||
resource: {
|
||||
fields: fields || [],
|
||||
},
|
||||
filters: filters || {},
|
||||
sort: sort || {},
|
||||
paginate: paginate || {},
|
||||
paginate: paginate || undefined,
|
||||
meta: {
|
||||
table: tableObj,
|
||||
},
|
||||
|
|
@ -191,7 +210,7 @@ describe("SQL query builder", () => {
|
|||
)
|
||||
expect(query).toEqual({
|
||||
bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit, 5000],
|
||||
sql: `select * from (select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4) order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
|
||||
sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
|
||||
})
|
||||
|
||||
query = new Sql(SqlClient.ORACLE, limit)._query(
|
||||
|
|
@ -212,6 +231,7 @@ describe("SQL query builder", () => {
|
|||
it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => {
|
||||
let query = new Sql(SqlClient.ORACLE, limit)._query(
|
||||
generateReadJson({
|
||||
table: ORACLE_TABLE,
|
||||
filters: {
|
||||
equal: {
|
||||
name: "John",
|
||||
|
|
@ -222,13 +242,14 @@ describe("SQL query builder", () => {
|
|||
|
||||
expect(query).toEqual({
|
||||
bindings: ["John", limit, 5000],
|
||||
sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) = :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
|
||||
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
|
||||
})
|
||||
})
|
||||
|
||||
it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => {
|
||||
let query = new Sql(SqlClient.ORACLE, limit)._query(
|
||||
generateReadJson({
|
||||
table: ORACLE_TABLE,
|
||||
filters: {
|
||||
notEqual: {
|
||||
name: "John",
|
||||
|
|
@ -239,7 +260,7 @@ describe("SQL query builder", () => {
|
|||
|
||||
expect(query).toEqual({
|
||||
bindings: ["John", limit, 5000],
|
||||
sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) != :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
|
||||
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import * as mongodb from "./mongodb"
|
|||
import * as mysql from "./mysql"
|
||||
import * as mssql from "./mssql"
|
||||
import * as mariadb from "./mariadb"
|
||||
import * as oracle from "./oracle"
|
||||
import { GenericContainer, StartedTestContainer } from "testcontainers"
|
||||
import { testContainerUtils } from "@budibase/backend-core/tests"
|
||||
import cloneDeep from "lodash/cloneDeep"
|
||||
|
|
@ -17,6 +18,7 @@ export enum DatabaseName {
|
|||
MYSQL = "mysql",
|
||||
SQL_SERVER = "mssql",
|
||||
MARIADB = "mariadb",
|
||||
ORACLE = "oracle",
|
||||
}
|
||||
|
||||
const providers: Record<DatabaseName, DatasourceProvider> = {
|
||||
|
|
@ -25,6 +27,7 @@ const providers: Record<DatabaseName, DatasourceProvider> = {
|
|||
[DatabaseName.MYSQL]: mysql.getDatasource,
|
||||
[DatabaseName.SQL_SERVER]: mssql.getDatasource,
|
||||
[DatabaseName.MARIADB]: mariadb.getDatasource,
|
||||
[DatabaseName.ORACLE]: oracle.getDatasource,
|
||||
}
|
||||
|
||||
export function getDatasourceProviders(
|
||||
|
|
@ -60,6 +63,9 @@ export async function knexClient(ds: Datasource) {
|
|||
case SourceName.SQL_SERVER: {
|
||||
return mssql.knexClient(ds)
|
||||
}
|
||||
case SourceName.ORACLE: {
|
||||
return oracle.knexClient(ds)
|
||||
}
|
||||
default: {
|
||||
throw new Error(`Unsupported source: ${ds.source}`)
|
||||
}
|
||||
|
|
|
|||
78
packages/server/src/integrations/tests/utils/oracle.ts
Normal file
78
packages/server/src/integrations/tests/utils/oracle.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import { Datasource, SourceName } from "@budibase/types"
|
||||
import { GenericContainer, Wait } from "testcontainers"
|
||||
import { generator, testContainerUtils } from "@budibase/backend-core/tests"
|
||||
import { startContainer } from "."
|
||||
import knex from "knex"
|
||||
|
||||
let ports: Promise<testContainerUtils.Port[]>
|
||||
|
||||
export async function getDatasource(): Promise<Datasource> {
|
||||
// password needs to conform to Oracle standards
|
||||
const password = "password"
|
||||
if (!ports) {
|
||||
// couldn't build 19.3.0 for X64
|
||||
let image = "budibase/oracle-database:23.2-slim-faststart"
|
||||
if (process.arch.startsWith("arm")) {
|
||||
// there isn't an ARM compatible 23.2 build
|
||||
image = "budibase/oracle-database:19.3.0-ee-slim-faststart"
|
||||
}
|
||||
|
||||
ports = startContainer(
|
||||
new GenericContainer(image)
|
||||
.withExposedPorts(1521)
|
||||
.withEnvironment({
|
||||
ORACLE_PASSWORD: password,
|
||||
})
|
||||
.withWaitStrategy(Wait.forLogMessage("DATABASE IS READY TO USE!"))
|
||||
)
|
||||
}
|
||||
|
||||
const port = (await ports).find(x => x.container === 1521)?.host
|
||||
if (!port) {
|
||||
throw new Error("Oracle port not found")
|
||||
}
|
||||
|
||||
const host = "127.0.0.1"
|
||||
const user = "SYSTEM"
|
||||
|
||||
const datasource: Datasource = {
|
||||
type: "datasource_plus",
|
||||
source: SourceName.ORACLE,
|
||||
plus: true,
|
||||
config: { host, port, user, password, database: "FREEPDB1" },
|
||||
}
|
||||
|
||||
const newUser = "a" + generator.guid().replaceAll("-", "")
|
||||
const client = await knexClient(datasource)
|
||||
await client.raw(`CREATE USER ${newUser} IDENTIFIED BY password`)
|
||||
await client.raw(
|
||||
`GRANT CONNECT, RESOURCE, CREATE VIEW, CREATE SESSION TO ${newUser}`
|
||||
)
|
||||
await client.raw(`GRANT UNLIMITED TABLESPACE TO ${newUser}`)
|
||||
datasource.config!.user = newUser
|
||||
|
||||
return datasource
|
||||
}
|
||||
|
||||
export async function knexClient(ds: Datasource) {
|
||||
if (!ds.config) {
|
||||
throw new Error("Datasource config is missing")
|
||||
}
|
||||
if (ds.source !== SourceName.ORACLE) {
|
||||
throw new Error("Datasource source is not Oracle")
|
||||
}
|
||||
|
||||
const db = ds.config.database || "FREEPDB1"
|
||||
const connectString = `${ds.config.host}:${ds.config.port}/${db}`
|
||||
|
||||
const c = knex({
|
||||
client: "oracledb",
|
||||
connection: {
|
||||
connectString,
|
||||
user: ds.config.user,
|
||||
password: ds.config.password,
|
||||
},
|
||||
})
|
||||
|
||||
return c
|
||||
}
|
||||
|
|
@ -147,7 +147,8 @@ export async function search(
|
|||
} catch (err: any) {
|
||||
if (err.message && err.message.includes("does not exist")) {
|
||||
throw new Error(
|
||||
`Table updated externally, please re-fetch - ${err.message}`
|
||||
`Table updated externally, please re-fetch - ${err.message}`,
|
||||
{ cause: err }
|
||||
)
|
||||
} else {
|
||||
throw err
|
||||
|
|
|
|||
|
|
@ -20,12 +20,11 @@ import {
|
|||
buildInternalRelationships,
|
||||
sqlOutputProcessing,
|
||||
} from "../../../../../api/controllers/row/utils"
|
||||
import sdk from "../../../../index"
|
||||
import {
|
||||
decodeNonAscii,
|
||||
mapToUserColumn,
|
||||
USER_COLUMN_PREFIX,
|
||||
} from "../../../tables/internal/sqs"
|
||||
import sdk from "../../../../index"
|
||||
import {
|
||||
context,
|
||||
sql,
|
||||
|
|
@ -42,7 +41,11 @@ import {
|
|||
getRelationshipColumns,
|
||||
getTableIDList,
|
||||
} from "../filters"
|
||||
import { dataFilters, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
|
||||
import {
|
||||
dataFilters,
|
||||
helpers,
|
||||
PROTECTED_INTERNAL_COLUMNS,
|
||||
} from "@budibase/shared-core"
|
||||
import { isSearchingByRowID } from "../utils"
|
||||
import tracer from "dd-trace"
|
||||
|
||||
|
|
@ -191,7 +194,7 @@ function reverseUserColumnMapping(rows: Row[]) {
|
|||
if (index !== -1) {
|
||||
// cut out the prefix
|
||||
const newKey = key.slice(0, index) + key.slice(index + prefixLength)
|
||||
const decoded = decodeNonAscii(newKey)
|
||||
const decoded = helpers.schema.decodeNonAscii(newKey)
|
||||
finalRow[decoded] = row[key]
|
||||
} else {
|
||||
finalRow[key] = row[key]
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import tablesSdk from "../"
|
|||
import { generateJunctionTableID } from "../../../../db/utils"
|
||||
import { isEqual } from "lodash"
|
||||
import { DEFAULT_TABLES } from "../../../../db/defaultData/datasource_bb_default"
|
||||
import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
|
||||
import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
|
||||
|
||||
const FieldTypeMap: Record<FieldType, SQLiteType> = {
|
||||
[FieldType.BOOLEAN]: SQLiteType.NUMERIC,
|
||||
|
|
@ -63,29 +63,10 @@ function buildRelationshipDefinitions(
|
|||
|
||||
export const USER_COLUMN_PREFIX = "data_"
|
||||
|
||||
// SQS does not support non-ASCII characters in column names, so we need to
|
||||
// replace them with unicode escape sequences.
|
||||
function encodeNonAscii(str: string): string {
|
||||
return str
|
||||
.split("")
|
||||
.map(char => {
|
||||
return char.charCodeAt(0) > 127
|
||||
? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
|
||||
: char
|
||||
})
|
||||
.join("")
|
||||
}
|
||||
|
||||
export function decodeNonAscii(str: string): string {
|
||||
return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
|
||||
String.fromCharCode(parseInt(p1, 16))
|
||||
)
|
||||
}
|
||||
|
||||
// utility function to denote that columns in SQLite are mapped to avoid overlap issues
|
||||
// the overlaps can occur due to case insensitivity and some of the columns which Budibase requires
|
||||
export function mapToUserColumn(key: string) {
|
||||
return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}`
|
||||
return `${USER_COLUMN_PREFIX}${helpers.schema.encodeNonAscii(key)}`
|
||||
}
|
||||
|
||||
// this can generate relationship tables as part of the mapping
|
||||
|
|
|
|||
|
|
@ -315,6 +315,21 @@ export async function outputProcessing<T extends Row[] | Row>(
|
|||
column.subtype
|
||||
)
|
||||
}
|
||||
} else if (column.type === FieldType.DATETIME && column.timeOnly) {
|
||||
for (let row of enriched) {
|
||||
if (row[property] instanceof Date) {
|
||||
const hours = row[property].getUTCHours().toString().padStart(2, "0")
|
||||
const minutes = row[property]
|
||||
.getUTCMinutes()
|
||||
.toString()
|
||||
.padStart(2, "0")
|
||||
const seconds = row[property]
|
||||
.getUTCSeconds()
|
||||
.toString()
|
||||
.padStart(2, "0")
|
||||
row[property] = `${hours}:${minutes}:${seconds}`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import dayjs from "dayjs"
|
|||
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
|
||||
import { deepGet, schema } from "./helpers"
|
||||
import { isPlainObject, isEmpty } from "lodash"
|
||||
import { decodeNonAscii } from "./helpers/schema"
|
||||
|
||||
const HBS_REGEX = /{{([^{].*?)}}/g
|
||||
|
||||
|
|
@ -181,8 +182,16 @@ export class ColumnSplitter {
|
|||
tableIds: string[]
|
||||
relationshipColumnNames: string[]
|
||||
relationships: string[]
|
||||
aliases?: Record<string, string>
|
||||
columnPrefix?: string
|
||||
|
||||
constructor(tables: Table[]) {
|
||||
constructor(
|
||||
tables: Table[],
|
||||
opts?: {
|
||||
aliases?: Record<string, string>
|
||||
columnPrefix?: string
|
||||
}
|
||||
) {
|
||||
this.tableNames = tables.map(table => table.name)
|
||||
this.tableIds = tables.map(table => table._id!)
|
||||
this.relationshipColumnNames = tables.flatMap(table =>
|
||||
|
|
@ -195,16 +204,38 @@ export class ColumnSplitter {
|
|||
.concat(this.relationshipColumnNames)
|
||||
// sort by length - makes sure there's no mis-matches due to similarities (sub column names)
|
||||
.sort((a, b) => b.length - a.length)
|
||||
|
||||
if (opts?.aliases) {
|
||||
this.aliases = {}
|
||||
for (const [key, value] of Object.entries(opts.aliases)) {
|
||||
this.aliases[value] = key
|
||||
}
|
||||
}
|
||||
|
||||
this.columnPrefix = opts?.columnPrefix
|
||||
}
|
||||
|
||||
run(key: string): {
|
||||
numberPrefix?: string
|
||||
relationshipPrefix?: string
|
||||
tableName?: string
|
||||
column: string
|
||||
} {
|
||||
let { prefix, key: splitKey } = getKeyNumbering(key)
|
||||
|
||||
let tableName: string | undefined = undefined
|
||||
if (this.aliases) {
|
||||
for (const possibleAlias of Object.keys(this.aliases || {})) {
|
||||
const withDot = `${possibleAlias}.`
|
||||
if (splitKey.startsWith(withDot)) {
|
||||
tableName = this.aliases[possibleAlias]!
|
||||
splitKey = splitKey.slice(withDot.length)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let relationship: string | undefined
|
||||
for (let possibleRelationship of this.relationships) {
|
||||
for (const possibleRelationship of this.relationships) {
|
||||
const withDot = `${possibleRelationship}.`
|
||||
if (splitKey.startsWith(withDot)) {
|
||||
const finalKeyParts = splitKey.split(withDot)
|
||||
|
|
@ -214,7 +245,15 @@ export class ColumnSplitter {
|
|||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (this.columnPrefix) {
|
||||
if (splitKey.startsWith(this.columnPrefix)) {
|
||||
splitKey = decodeNonAscii(splitKey.slice(this.columnPrefix.length))
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
tableName,
|
||||
numberPrefix: prefix,
|
||||
relationshipPrefix: relationship,
|
||||
column: splitKey,
|
||||
|
|
|
|||
|
|
@ -26,3 +26,22 @@ export function isRequired(constraints: FieldConstraints | undefined) {
|
|||
constraints.presence === true)
|
||||
return isRequired
|
||||
}
|
||||
|
||||
// SQS does not support non-ASCII characters in column names, so we need to
|
||||
// replace them with unicode escape sequences.
|
||||
export function encodeNonAscii(str: string): string {
|
||||
return str
|
||||
.split("")
|
||||
.map(char => {
|
||||
return char.charCodeAt(0) > 127
|
||||
? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
|
||||
: char
|
||||
})
|
||||
.join("")
|
||||
}
|
||||
|
||||
export function decodeNonAscii(str: string): string {
|
||||
return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
|
||||
String.fromCharCode(parseInt(p1, 16))
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,10 +48,13 @@ async function removeTenantUsers(tenantId: string) {
|
|||
try {
|
||||
const allUsers = await getTenantUsers(tenantId)
|
||||
const allEmails = allUsers.rows.map((row: any) => row.doc.email)
|
||||
const allSsoIds = allUsers.rows
|
||||
.map((row: any) => row.doc.ssoId)
|
||||
.filter(id => !!id)
|
||||
|
||||
// get the id and email doc ids
|
||||
let keys = allUsers.rows.map((row: any) => row.id)
|
||||
keys = keys.concat(allEmails)
|
||||
keys = keys.concat(allEmails).concat(allSsoIds)
|
||||
|
||||
const platformDb = platform.getPlatformDB()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue