diff --git a/.vscode/launch.json b/.vscode/launch.json index 8cb49d5825..6c0089bb6b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,42 +1,32 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Budibase Server", - "type": "node", - "request": "launch", - "runtimeArgs": [ - "--nolazy", - "-r", - "ts-node/register/transpile-only" - ], - "args": [ - "${workspaceFolder}/packages/server/src/index.ts" - ], - "cwd": "${workspaceFolder}/packages/server" - }, - { - "name": "Budibase Worker", - "type": "node", - "request": "launch", - "runtimeArgs": [ - "--nolazy", - "-r", - "ts-node/register/transpile-only" - ], - "args": [ - "${workspaceFolder}/packages/worker/src/index.ts" - ], - "cwd": "${workspaceFolder}/packages/worker" - }, - ], - "compounds": [ - { - "name": "Start Budibase", - "configurations": ["Budibase Server", "Budibase Worker"] - } - ] -} \ No newline at end of file + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Budibase Server", + "type": "node", + "request": "launch", + "runtimeVersion": "14.20.1", + "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], + "args": ["${workspaceFolder}/packages/server/src/index.ts"], + "cwd": "${workspaceFolder}/packages/server" + }, + { + "name": "Budibase Worker", + "type": "node", + "request": "launch", + "runtimeVersion": "14.20.1", + "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], + "args": ["${workspaceFolder}/packages/worker/src/index.ts"], + "cwd": "${workspaceFolder}/packages/worker" + } + ], + "compounds": [ + { + "name": "Start Budibase", + "configurations": ["Budibase Server", "Budibase Worker"] + } + ] +} diff --git a/lerna.json b/lerna.json index 7dc6b51562..721efe58a4 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.9.24-alpha.1", + "version": "2.9.24-alpha.2", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index 4b1ab0d68a..e9f6e5629f 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -17,7 +17,7 @@ import { generate } from "shortid" import { LuceneUtils, Constants } from "@budibase/frontend-core" import { getFields } from "helpers/searchFields" - import { createEventDispatcher } from "svelte" + import { createEventDispatcher, onMount } from "svelte" export let schemaFields export let filters = [] @@ -64,6 +64,15 @@ }) } + onMount(() => { + parseFilters(filters) + rawFilters.forEach(filter => { + filter.type = + schemaFields.find(field => field.name === filter.field)?.type || + filter.type + }) + }) + // Add field key prefixes and a special metadata filter object to indicate // whether to use the "match all" or "match any" behaviour const enrichFilters = (rawFilters, matchAny) => { diff --git a/packages/server/package.json b/packages/server/package.json index 7d0d8f5feb..0baec08436 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -100,7 +100,7 @@ "memorystream": "0.3.1", "mongodb": "5.7", "mssql": "9.1.1", - "mysql2": "2.3.3", + "mysql2": "3.5.2", "node-fetch": "2.6.7", "object-sizeof": "2.6.1", "open": "8.4.0", diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index ee8e1bb022..b9a69f3988 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -127,7 +127,7 @@ export async function preview(ctx: any) { const query = ctx.request.body // preview may not have a queryId as it hasn't been saved, but if it does // this stops dynamic variables from calling the same query - const { fields, parameters, queryVerb, transformer, queryId } = query + const { fields, parameters, queryVerb, transformer, queryId, schema } = query const authConfigCtx: any = getAuthConfig(ctx) @@ -140,6 +140,7 @@ export async function preview(ctx: any) { parameters, transformer, queryId, + schema, // have to pass down to the thread runner - can't put into context now environmentVariables: envVars, ctx: { @@ -235,6 +236,7 @@ async function execute( user: ctx.user, auth: { ...authConfigCtx }, }, + schema: query.schema, } const runFn = () => Runner.run(inputs) diff --git a/packages/server/src/automations/tests/filter.spec.js b/packages/server/src/automations/tests/filter.spec.ts similarity index 76% rename from packages/server/src/automations/tests/filter.spec.js rename to packages/server/src/automations/tests/filter.spec.ts index 5fa5e0e202..c2c5d13e2d 100644 --- a/packages/server/src/automations/tests/filter.spec.js +++ b/packages/server/src/automations/tests/filter.spec.ts @@ -1,11 +1,18 @@ -const setup = require("./utilities") -const { FilterConditions } = require("../steps/filter") +import * as setup from "./utilities" +import { FilterConditions } from "../steps/filter" describe("test the filter logic", () => { - async function checkFilter(field, condition, value, pass = true) { - let res = await setup.runStep(setup.actions.FILTER.stepId, - { field, condition, value } - ) + async function checkFilter( + field: any, + condition: string, + value: any, + pass = true + ) { + let res = await setup.runStep(setup.actions.FILTER.stepId, { + field, + condition, + value, + }) expect(res.result).toEqual(pass) expect(res.success).toEqual(true) } @@ -36,9 +43,9 @@ describe("test the filter logic", () => { it("check date coercion", async () => { await checkFilter( - (new Date()).toISOString(), + new Date().toISOString(), FilterConditions.GREATER_THAN, - (new Date(-10000)).toISOString(), + new Date(-10000).toISOString(), true ) }) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index a01f6b71f9..b88162c59a 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -93,6 +93,21 @@ const SCHEMA: Integration = { }, } +const defaultTypeCasting = function (field: any, next: any) { + if ( + field.type == "DATETIME" || + field.type === "DATE" || + field.type === "TIMESTAMP" || + field.type === "LONGLONG" + ) { + return field.string() + } + if (field.type === "BIT" && field.length === 1) { + return field.buffer()?.[0] + } + return next() +} + export function bindingTypeCoerce(bindings: any[]) { for (let i = 0; i < bindings.length; i++) { const binding = bindings[i] @@ -147,21 +162,8 @@ class MySQLIntegration extends Sql implements DatasourcePlus { delete config.rejectUnauthorized this.config = { ...config, + typeCast: defaultTypeCasting, multipleStatements: true, - typeCast: function (field: any, next: any) { - if ( - field.type == "DATETIME" || - field.type === "DATE" || - field.type === "TIMESTAMP" || - field.type === "LONGLONG" - ) { - return field.string() - } - if (field.type === "BIT" && field.length === 1) { - return field.buffer()?.[0] - } - return next() - }, } } @@ -194,6 +196,37 @@ class MySQLIntegration extends Sql implements DatasourcePlus { return `concat(${parts.join(", ")})` } + defineTypeCastingFromSchema(schema: { + [key: string]: { name: string; type: string } + }): void { + if (!schema) { + return + } + this.config.typeCast = function (field: any, next: any) { + if (schema[field.name]?.name === field.name) { + if (["LONGLONG", "NEWDECIMAL", "DECIMAL"].includes(field.type)) { + if (schema[field.name]?.type === "number") { + const value = field.string() + return value ? Number(value) : null + } else { + return field.string() + } + } + } + if ( + field.type == "DATETIME" || + field.type === "DATE" || + field.type === "TIMESTAMP" + ) { + return field.string() + } + if (field.type === "BIT" && field.length === 1) { + return field.buffer()?.[0] + } + return next() + } + } + async connect() { this.client = await mysql.createConnection(this.config) } @@ -204,7 +237,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus { async internalQuery( query: SqlQuery, - opts: { connect?: boolean; disableCoercion?: boolean } = { + opts: { + connect?: boolean + disableCoercion?: boolean + } = { connect: true, disableCoercion: false, } diff --git a/packages/server/src/threads/definitions.ts b/packages/server/src/threads/definitions.ts index 21e7ce0b69..dd0891d34a 100644 --- a/packages/server/src/threads/definitions.ts +++ b/packages/server/src/threads/definitions.ts @@ -11,6 +11,12 @@ export interface QueryEvent { queryId: string environmentVariables?: Record ctx?: any + schema?: { + [key: string]: { + name: string + type: string + } + } } export interface QueryVariable { diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 135c533122..3d68895753 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -8,6 +8,7 @@ import { context, cache, auth } from "@budibase/backend-core" import { getGlobalIDFromUserMetadataID } from "../db/utils" import sdk from "../sdk" import { cloneDeep } from "lodash/fp" +import { SourceName } from "@budibase/types" import { isSQL } from "../integrations/utils" import { interpolateSQL } from "../integrations/queries/sql" @@ -28,6 +29,7 @@ class QueryRunner { hasRerun: boolean hasRefreshedOAuth: boolean hasDynamicVariables: boolean + schema: any constructor(input: QueryEvent, flags = { noRecursiveQuery: false }) { this.datasource = input.datasource @@ -37,6 +39,7 @@ class QueryRunner { this.pagination = input.pagination this.transformer = input.transformer this.queryId = input.queryId + this.schema = input.schema this.noRecursiveQuery = flags.noRecursiveQuery this.cachedVariables = [] // Additional context items for enrichment @@ -51,7 +54,7 @@ class QueryRunner { } async execute(): Promise { - let { datasource, fields, queryVerb, transformer } = this + let { datasource, fields, queryVerb, transformer, schema } = this let datasourceClone = cloneDeep(datasource) let fieldsClone = cloneDeep(fields) @@ -70,6 +73,9 @@ class QueryRunner { const integration = new Integration(datasourceClone.config) + // define the type casting from the schema + integration.defineTypeCastingFromSchema?.(schema) + // pre-query, make sure datasource variables are added to parameters const parameters = await this.addDatasourceVariables() diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 2391f6e878..d6a0d4a7c8 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -166,6 +166,12 @@ export interface IntegrationBase { delete?(query: any): Promise testConnection?(): Promise getExternalSchema?(): Promise + defineTypeCastingFromSchema?(schema: { + [key: string]: { + name: string + type: string + } + }): void } export interface DatasourcePlus extends IntegrationBase {