From 0ff9bfbe9991436a2ead82006120b23cccb73cc5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Jan 2022 17:55:28 +0000 Subject: [PATCH 1/9] Adding method for client/builder to detect that a table supports SQL filtering. --- packages/server/src/api/controllers/table/index.js | 3 ++- packages/server/src/api/controllers/table/utils.js | 5 ++++- packages/server/src/integrations/utils.ts | 11 ++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index abbb4d6ff9..15e869fc75 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -2,7 +2,7 @@ const CouchDB = require("../../../db") const internal = require("./internal") const external = require("./external") const csvParser = require("../../../utilities/csvParser") -const { isExternalTable } = require("../../../integrations/utils") +const { isExternalTable, isSQL } = require("../../../integrations/utils") const { getTableParams, getDatasourceParams, @@ -49,6 +49,7 @@ exports.fetch = async function (ctx) { ...entity, type: "external", sourceId: row.doc._id, + sql: isSQL(row.doc), })) }) diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index e4086e8071..459566ce91 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -12,6 +12,7 @@ const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants") const { isExternalTable, breakExternalTableId, + isSQL, } = require("../../../integrations/utils") const { getViews, saveView } = require("../view/utils") const viewTemplate = require("../view/viewBuilder") @@ -242,7 +243,9 @@ exports.getTable = async (appId, tableId) => { const db = new CouchDB(appId) if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) - return exports.getExternalTable(appId, datasourceId, tableName) + const datasource = await db.get(datasourceId) + const table = await exports.getExternalTable(appId, datasourceId, tableName) + return { ...table, sql: isSQL(datasource) } } else { return db.get(tableId) } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 97380b1b5b..b1b1faf776 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,6 +1,6 @@ -import { SqlQuery } from "../definitions/datasource" +import { SourceNames, SqlQuery } from "../definitions/datasource" import { Datasource, Table } from "../definitions/common" -import { SourceNames } from "../definitions/datasource" + const { DocumentTypes, SEPARATOR } = require("../db/utils") const { FieldTypes, @@ -131,7 +131,12 @@ export function isSQL(datasource: Datasource): boolean { if (!datasource || !datasource.source) { return false } - const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL] + const SQL = [ + SourceNames.POSTGRES, + SourceNames.SQL_SERVER, + SourceNames.MYSQL, + SourceNames.ORACLE, + ] return SQL.indexOf(datasource.source) !== -1 } From e70e923112d92a2d2b70d824ef763ac0e3d071a6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Jan 2022 15:59:04 +0000 Subject: [PATCH 2/9] Basic frontend work, getting it sending up search fields from tables with links. --- .../FilterEditor/FilterDrawer.svelte | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index d9425c961d..31a7307911 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -14,6 +14,7 @@ import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" import { generate } from "shortid" import { getValidOperatorsForType, OperatorOptions } from "constants/lucene" + import { tables } from "stores/backend" export let schemaFields export let filters = [] @@ -23,9 +24,34 @@ const BannedTypes = ["link", "attachment", "formula", "json", "jsonarray"] - $: fieldOptions = (schemaFields ?? []) - .filter(field => !BannedTypes.includes(field.type)) - .map(field => field.name) + function getTableFields(linkField) { + const table = $tables.list.find(table => table._id === linkField.tableId) + if (!table) { + return [] + } + const linkFields = getFields(Object.values(table.schema), { + allowLinks: false, + }) + return linkFields.map(field => ({ + ...field, + name: `${linkField.name}.${field.name}`, + })) + } + + function getFields(fields, { allowLinks } = { allowLinks: true }) { + let fieldNames = fields.filter(field => !BannedTypes.includes(field.type)) + if (allowLinks) { + const linkFields = fields.filter(field => field.type === "link") + for (let linkField of linkFields) { + // only allow one depth of SQL relationship filtering + fieldNames = fieldNames.concat(getTableFields(linkField)) + } + } + return fieldNames + } + + $: enrichedSchemaFields = getFields(schemaFields || []) + $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] const addFilter = () => { @@ -53,7 +79,7 @@ const onFieldChange = (expression, field) => { // Update the field type - expression.type = schemaFields.find(x => x.name === field)?.type + expression.type = enrichedSchemaFields.find(x => x.name === field)?.type // Ensure a valid operator is set const validOperators = getValidOperatorsForType(expression.type).map( @@ -85,7 +111,7 @@ } const getFieldOptions = field => { - const schema = schemaFields.find(x => x.name === field) + const schema = enrichedSchemaFields.find(x => x.name === field) return schema?.constraints?.inclusion || [] } From 13629a1204b77a1813ac13fb85ec7f90d4463be2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Jan 2022 16:48:19 +0000 Subject: [PATCH 3/9] Quick fix, only allow using SQL. --- .../PropertyControls/FilterEditor/FilterDrawer.svelte | 2 +- packages/server/src/integrations/utils.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index 31a7307911..a1cdc48806 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -26,7 +26,7 @@ function getTableFields(linkField) { const table = $tables.list.find(table => table._id === linkField.tableId) - if (!table) { + if (!table || !table.sql) { return [] } const linkFields = getFields(Object.values(table.schema), { diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index c0c5877c7a..46bec0e33e 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,6 +1,5 @@ import { SourceNames, SqlQuery } from "../definitions/datasource" import { Datasource, Table } from "../definitions/common" -import { SourceNames } from "../definitions/datasource" import { DocumentTypes, SEPARATOR } from "../db/utils" import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants" From ee29245b6a53ef1da6b2519fe327f3368e203609 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Jan 2022 17:40:11 +0000 Subject: [PATCH 4/9] Adjusting SQL system to handle relationship filtering. --- packages/server/src/integrations/base/sql.ts | 21 +++++++++++++------- packages/server/src/integrations/s3.ts | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index df4f2d511b..c1641e8626 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -72,16 +72,22 @@ class InternalBuilder { // right now we only do filters on the specific table being queried addFilters( - tableName: string, query: KnexQuery, - filters: SearchFilters | undefined + filters: SearchFilters | undefined, + opts: { relationship?: boolean; tableName?: string } ): KnexQuery { function iterate( structure: { [key: string]: any }, fn: (key: string, value: any) => void ) { for (let [key, value] of Object.entries(structure)) { - fn(`${tableName}.${key}`, value) + const isRelationshipField = key.includes(".") + if (!opts.relationship && !isRelationshipField) { + fn(`${opts.tableName}.${key}`, value) + } + if (opts.relationship && isRelationshipField) { + fn(key, value) + } } } if (!filters) { @@ -272,7 +278,7 @@ class InternalBuilder { if (foundOffset) { query = query.offset(foundOffset) } - query = this.addFilters(tableName, query, filters) + query = this.addFilters(query, filters, { tableName }) // add sorting to pre-query query = this.addSorting(query, json) // @ts-ignore @@ -285,20 +291,21 @@ class InternalBuilder { preQuery = this.addSorting(preQuery, json) } // handle joins - return this.addRelationships( + query = this.addRelationships( knex, preQuery, selectStatement, tableName, relationships ) + return this.addFilters(query, filters, { relationship: true }) } update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { const { endpoint, body, filters } = json let query: KnexQuery = knex(endpoint.entityId) const parsedBody = parseBody(body) - query = this.addFilters(endpoint.entityId, query, filters) + query = this.addFilters(query, filters, { tableName: endpoint.entityId }) // mysql can't use returning if (opts.disableReturning) { return query.update(parsedBody) @@ -310,7 +317,7 @@ class InternalBuilder { delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { const { endpoint, filters } = json let query: KnexQuery = knex(endpoint.entityId) - query = this.addFilters(endpoint.entityId, query, filters) + query = this.addFilters(query, filters, { tableName: endpoint.entityId }) // mysql can't use returning if (opts.disableReturning) { return query.delete() diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 25b439fd58..273f221575 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -38,7 +38,7 @@ module S3Module { signatureVersion: { type: "string", required: false, - default: "v4" + default: "v4", }, }, query: { From 018e0bd73bc8dcebad77698a23b85eb4d582b5d7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jan 2022 17:42:14 +0000 Subject: [PATCH 5/9] Getting client partially working, having an issue with search fields not updating for a table block. --- .../FilterEditor/FilterDrawer.svelte | 30 +---------- .../PropertyControls/SearchFieldSelect.svelte | 52 +++++++++++++++++++ .../PropertyControls/componentSettings.js | 2 + .../builder/src/constants/backend/index.js | 8 +++ packages/builder/src/helpers/searchFields.js | 31 +++++++++++ packages/client/manifest.json | 2 +- .../components/app/blocks/TableBlock.svelte | 44 +++++++++++++--- packages/string-templates/test/basic.spec.js | 11 ++++ 8 files changed, 144 insertions(+), 36 deletions(-) create mode 100644 packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte create mode 100644 packages/builder/src/helpers/searchFields.js diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index a1cdc48806..8d7f50a527 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -14,7 +14,7 @@ import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" import { generate } from "shortid" import { getValidOperatorsForType, OperatorOptions } from "constants/lucene" - import { tables } from "stores/backend" + import { getFields } from "helpers/searchFields" export let schemaFields export let filters = [] @@ -22,34 +22,6 @@ export let panel = ClientBindingPanel export let allowBindings = true - const BannedTypes = ["link", "attachment", "formula", "json", "jsonarray"] - - function getTableFields(linkField) { - const table = $tables.list.find(table => table._id === linkField.tableId) - if (!table || !table.sql) { - return [] - } - const linkFields = getFields(Object.values(table.schema), { - allowLinks: false, - }) - return linkFields.map(field => ({ - ...field, - name: `${linkField.name}.${field.name}`, - })) - } - - function getFields(fields, { allowLinks } = { allowLinks: true }) { - let fieldNames = fields.filter(field => !BannedTypes.includes(field.type)) - if (allowLinks) { - const linkFields = fields.filter(field => field.type === "link") - for (let linkField of linkFields) { - // only allow one depth of SQL relationship filtering - fieldNames = fieldNames.concat(getTableFields(linkField)) - } - } - return fieldNames - } - $: enrichedSchemaFields = getFields(schemaFields || []) $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte new file mode 100644 index 0000000000..ad8fad8a87 --- /dev/null +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte @@ -0,0 +1,52 @@ + + + diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js index e752240302..5e27cdce28 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js @@ -7,6 +7,7 @@ import ColorPicker from "./ColorPicker.svelte" import { IconSelect } from "./IconSelect" import FieldSelect from "./FieldSelect.svelte" import MultiFieldSelect from "./MultiFieldSelect.svelte" +import SearchFieldSelect from "./SearchFieldSelect.svelte" import SchemaSelect from "./SchemaSelect.svelte" import SectionSelect from "./SectionSelect.svelte" import NavigationEditor from "./NavigationEditor/NavigationEditor.svelte" @@ -30,6 +31,7 @@ const componentMap = { icon: IconSelect, field: FieldSelect, multifield: MultiFieldSelect, + searchfield: SearchFieldSelect, options: OptionsEditor, schema: SchemaSelect, section: SectionSelect, diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index c5fb294f80..c1eea5b0ef 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -229,3 +229,11 @@ export const PaginationLocations = [ { label: "Query parameters", value: "query" }, { label: "Request body", value: "body" }, ] + +export const BannedSearchTypes = [ + "link", + "attachment", + "formula", + "json", + "jsonarray", +] diff --git a/packages/builder/src/helpers/searchFields.js b/packages/builder/src/helpers/searchFields.js new file mode 100644 index 0000000000..3f3ca1ce7e --- /dev/null +++ b/packages/builder/src/helpers/searchFields.js @@ -0,0 +1,31 @@ +import { tables } from "../stores/backend" +import { BannedSearchTypes } from "../constants/backend" +import { get } from "svelte/store" + +export function getTableFields(linkField) { + const table = get(tables).list.find(table => table._id === linkField.tableId) + if (!table || !table.sql) { + return [] + } + const linkFields = getFields(Object.values(table.schema), { + allowLinks: false, + }) + return linkFields.map(field => ({ + ...field, + name: `${table.name}.${field.name}`, + })) +} + +export function getFields(fields, { allowLinks } = { allowLinks: true }) { + let fieldNames = fields.filter( + field => !BannedSearchTypes.includes(field.type) + ) + if (allowLinks) { + const linkFields = fields.filter(field => field.type === "link") + for (let linkField of linkFields) { + // only allow one depth of SQL relationship filtering + fieldNames = fieldNames.concat(getTableFields(linkField)) + } + } + return fieldNames +} diff --git a/packages/client/manifest.json b/packages/client/manifest.json index dfe9ae5a91..310cec7f4b 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -2811,7 +2811,7 @@ "key": "dataSource" }, { - "type": "multifield", + "type": "searchfield", "label": "Search Columns", "key": "searchColumns", "placeholder": "Choose search columns" diff --git a/packages/client/src/components/app/blocks/TableBlock.svelte b/packages/client/src/components/app/blocks/TableBlock.svelte index 936a8d1734..fef6be5c50 100644 --- a/packages/client/src/components/app/blocks/TableBlock.svelte +++ b/packages/client/src/components/app/blocks/TableBlock.svelte @@ -41,9 +41,11 @@ let dataProviderId let schema let schemaLoaded = false + let enrichedSearchColumns + let enrichedSearchColumnsLoaded = false $: fetchSchema(dataSource) - $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema) + $: enrichSearchColumns(searchColumns, schema) $: enrichedFilter = enrichFilter(filter, enrichedSearchColumns, formId) $: titleButtonAction = [ { @@ -71,9 +73,9 @@ } // Determine data types for search fields and only use those that are valid - const enrichSearchColumns = (searchColumns, schema) => { + const enrichSearchColumns = async (searchColumns, schema) => { let enrichedColumns = [] - searchColumns?.forEach(column => { + const addType = column => { const schemaType = schema?.[column]?.type const componentType = schemaComponentMap[schemaType] if (componentType) { @@ -82,9 +84,39 @@ componentType, type: schemaType, }) + return true } - }) - return enrichedColumns.slice(0, 3) + return false + } + for (let column of searchColumns || []) { + // if addType returns false, it didn't find one, look for SQL relationships + if (!addType(column) && column.includes(".")) { + const [tableName, linkColumn] = column.split(".") + for (let colSchema of Object.values(schema || {})) { + // found the related table + if ( + colSchema.type === "link" && + colSchema.tableId && + colSchema.tableId.endsWith(tableName) + ) { + try { + const linkSchema = await fetchDatasourceSchema({ + ...dataSource, + tableId: colSchema.tableId, + }) + if (linkSchema) { + schema[column] = linkSchema[linkColumn] + addType(column) + } + } catch (err) { + // ignore the error, couldn't get table + } + } + } + } + } + enrichedSearchColumns = enrichedColumns.slice(0, 3) + enrichedSearchColumnsLoaded = true } // Load the datasource schema so we can determine column types @@ -96,7 +128,7 @@ } -{#if schemaLoaded} +{#if schemaLoaded && enrichedSearchColumnsLoaded}
diff --git a/packages/string-templates/test/basic.spec.js b/packages/string-templates/test/basic.spec.js index b437e386fc..2fd6505410 100644 --- a/packages/string-templates/test/basic.spec.js +++ b/packages/string-templates/test/basic.spec.js @@ -146,3 +146,14 @@ describe("check manifest", () => { ) }) }) + +describe("check full stops that are safe", () => { + it("should allow using an escaped full stop", async () => { + const data = { + "c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI": { "persons.firstname": "1" } + } + const template = "{{ [c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI].[persons.firstname] }}" + const output = await processString(template, data) + expect(output).toEqual("1") + }) +}) From ec93406578aa9a34f793e1515b8d730a7b06c0e0 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 17 Jan 2022 14:03:47 +0000 Subject: [PATCH 6/9] Add optional enrichment of relationship fields when determining datasource schema and update block filters to properly reference relationship fields --- packages/client/manifest.json | 2 +- .../components/app/blocks/CardsBlock.svelte | 7 ++- .../components/app/blocks/TableBlock.svelte | 51 ++++--------------- packages/client/src/utils/fetch/DataFetch.js | 5 +- packages/client/src/utils/schema.js | 35 +++++++++++-- 5 files changed, 51 insertions(+), 49 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 310cec7f4b..9431129fa0 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -2958,7 +2958,7 @@ "key": "dataSource" }, { - "type": "multifield", + "type": "searchfield", "label": "Search Columns", "key": "searchColumns", "placeholder": "Choose search columns" diff --git a/packages/client/src/components/app/blocks/CardsBlock.svelte b/packages/client/src/components/app/blocks/CardsBlock.svelte index 301b440ab3..f0892ca447 100644 --- a/packages/client/src/components/app/blocks/CardsBlock.svelte +++ b/packages/client/src/components/app/blocks/CardsBlock.svelte @@ -71,12 +71,13 @@ const enrichFilter = (filter, columns, formId) => { let enrichedFilter = [...(filter || [])] columns?.forEach(column => { + const safePath = column.name.split(".").map(safe).join(".") enrichedFilter.push({ field: column.name, operator: column.type === "string" ? "string" : "equal", type: column.type === "string" ? "string" : "number", valueType: "Binding", - value: `{{ [${formId}].[${column.name}] }}`, + value: `{{ ${safe(formId)}.${safePath} }}`, }) }) return enrichedFilter @@ -112,7 +113,9 @@ // Load the datasource schema so we can determine column types const fetchSchema = async dataSource => { if (dataSource) { - schema = await fetchDatasourceSchema(dataSource) + schema = await fetchDatasourceSchema(dataSource, { + enrichRelationships: true, + }) } schemaLoaded = true } diff --git a/packages/client/src/components/app/blocks/TableBlock.svelte b/packages/client/src/components/app/blocks/TableBlock.svelte index fef6be5c50..3de4497731 100644 --- a/packages/client/src/components/app/blocks/TableBlock.svelte +++ b/packages/client/src/components/app/blocks/TableBlock.svelte @@ -41,11 +41,9 @@ let dataProviderId let schema let schemaLoaded = false - let enrichedSearchColumns - let enrichedSearchColumnsLoaded = false $: fetchSchema(dataSource) - $: enrichSearchColumns(searchColumns, schema) + $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema) $: enrichedFilter = enrichFilter(filter, enrichedSearchColumns, formId) $: titleButtonAction = [ { @@ -61,21 +59,22 @@ const enrichFilter = (filter, columns, formId) => { let enrichedFilter = [...(filter || [])] columns?.forEach(column => { + const safePath = column.name.split(".").map(safe).join(".") enrichedFilter.push({ field: column.name, operator: column.type === "string" ? "string" : "equal", type: column.type === "string" ? "string" : "number", valueType: "Binding", - value: `{{ ${safe(formId)}.${safe(column.name)} }}`, + value: `{{ ${safe(formId)}.${safePath} }}`, }) }) return enrichedFilter } // Determine data types for search fields and only use those that are valid - const enrichSearchColumns = async (searchColumns, schema) => { + const enrichSearchColumns = (searchColumns, schema) => { let enrichedColumns = [] - const addType = column => { + searchColumns?.forEach(column => { const schemaType = schema?.[column]?.type const componentType = schemaComponentMap[schemaType] if (componentType) { @@ -84,51 +83,23 @@ componentType, type: schemaType, }) - return true } - return false - } - for (let column of searchColumns || []) { - // if addType returns false, it didn't find one, look for SQL relationships - if (!addType(column) && column.includes(".")) { - const [tableName, linkColumn] = column.split(".") - for (let colSchema of Object.values(schema || {})) { - // found the related table - if ( - colSchema.type === "link" && - colSchema.tableId && - colSchema.tableId.endsWith(tableName) - ) { - try { - const linkSchema = await fetchDatasourceSchema({ - ...dataSource, - tableId: colSchema.tableId, - }) - if (linkSchema) { - schema[column] = linkSchema[linkColumn] - addType(column) - } - } catch (err) { - // ignore the error, couldn't get table - } - } - } - } - } - enrichedSearchColumns = enrichedColumns.slice(0, 3) - enrichedSearchColumnsLoaded = true + }) + return enrichedColumns.slice(0, 3) } // Load the datasource schema so we can determine column types const fetchSchema = async dataSource => { if (dataSource) { - schema = await fetchDatasourceSchema(dataSource) + schema = await fetchDatasourceSchema(dataSource, { + enrichRelationships: true, + }) } schemaLoaded = true } -{#if schemaLoaded && enrichedSearchColumnsLoaded} +{#if schemaLoaded}
diff --git a/packages/client/src/utils/fetch/DataFetch.js b/packages/client/src/utils/fetch/DataFetch.js index 2333991ac9..884e12feb1 100644 --- a/packages/client/src/utils/fetch/DataFetch.js +++ b/packages/client/src/utils/fetch/DataFetch.js @@ -67,7 +67,6 @@ export default class DataFetch { this.getPage = this.getPage.bind(this) this.getInitialData = this.getInitialData.bind(this) this.determineFeatureFlags = this.determineFeatureFlags.bind(this) - this.enrichSchema = this.enrichSchema.bind(this) this.refresh = this.refresh.bind(this) this.update = this.update.bind(this) this.hasNextPage = this.hasNextPage.bind(this) @@ -129,7 +128,7 @@ export default class DataFetch { // Fetch and enrich schema let schema = this.constructor.getSchema(datasource, definition) - schema = this.enrichSchema(schema) + schema = DataFetch.enrichSchema(schema) if (!schema) { return } @@ -248,7 +247,7 @@ export default class DataFetch { * @param schema the datasource schema * @return {object} the enriched datasource schema */ - enrichSchema(schema) { + static enrichSchema(schema) { if (schema == null) { return null } diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index e333bb616e..dc00430fcd 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -6,13 +6,19 @@ import RelationshipFetch from "./fetch/RelationshipFetch.js" import NestedProviderFetch from "./fetch/NestedProviderFetch.js" import FieldFetch from "./fetch/FieldFetch.js" import JSONArrayFetch from "./fetch/JSONArrayFetch.js" +import DataFetch from "./fetch/DataFetch.js" /** * Fetches the schema of any kind of datasource. * All datasource fetch classes implement their own functionality to get the * schema of a datasource of their respective types. + * @param datasource the datasource to fetch the schema for + * @param options options for enriching the schema */ -export const fetchDatasourceSchema = async datasource => { +export const fetchDatasourceSchema = async ( + datasource, + options = { enrichRelationships: false } +) => { const handler = { table: TableFetch, view: ViewFetch, @@ -28,7 +34,7 @@ export const fetchDatasourceSchema = async datasource => { // Get the datasource definition and then schema const definition = await handler.getDefinition(datasource) - const schema = handler.getSchema(datasource, definition) + let schema = handler.getSchema(datasource, definition) if (!schema) { return null } @@ -49,5 +55,28 @@ export const fetchDatasourceSchema = async datasource => { }) } }) - return { ...schema, ...jsonAdditions } + schema = { ...schema, ...jsonAdditions } + + // Check for any relationship fields if required + if (options?.enrichRelationships) { + let relationshipAdditions = {} + for (let fieldKey of Object.keys(schema)) { + const fieldSchema = schema[fieldKey] + if (fieldSchema?.type === "link") { + const linkSchema = await fetchDatasourceSchema({ + type: "table", + tableId: fieldSchema?.tableId, + }) + Object.keys(linkSchema || {}).forEach(linkKey => { + relationshipAdditions[`${fieldKey}.${linkKey}`] = { + type: linkSchema[linkKey].type, + } + }) + } + } + schema = { ...schema, ...relationshipAdditions } + } + + // Ensure schema structure is correct + return DataFetch.enrichSchema(schema) } From abfa80bb470924a46137ff26b88bb2ffeedb0de6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 17 Jan 2022 16:48:49 +0000 Subject: [PATCH 7/9] Adding dynamic filter capabilities, also updating search field select in builder to make sure it removes banned search field types. --- .../PropertyControls/SearchFieldSelect.svelte | 15 ++---- packages/builder/src/helpers/searchFields.js | 6 +-- .../src/components/app/DataProvider.svelte | 1 + .../app/dynamic-filter/DynamicFilter.svelte | 52 +++++++++++++------ packages/client/src/utils/schema.js | 2 +- 5 files changed, 45 insertions(+), 31 deletions(-) diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte index ad8fad8a87..474fbc676c 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte @@ -7,7 +7,7 @@ import { currentAsset } from "builderStore" import { tables } from "stores/backend" import { createEventDispatcher } from "svelte" - import { getTableFields } from "helpers/searchFields" + import { getFields } from "helpers/searchFields" export let componentInstance = {} export let value = "" @@ -20,19 +20,14 @@ $: boundValue = getSelectedOption(value, options) function getOptions(ds, dsSchema) { - let base = Object.keys(dsSchema) + let base = Object.values(dsSchema) if (!ds?.tableId) { return base } const currentTable = $tables.list.find(table => table._id === ds.tableId) - if (currentTable && currentTable.sql) { - for (let column of Object.values(currentTable.schema)) { - if (column.type === "link") { - base = base.concat(getTableFields(column).map(field => field.name)) - } - } - } - return base + return getFields(base, { allowLinks: currentTable.sql }).map( + field => field.name + ) } function getSelectedOption(selectedOptions, allOptions) { diff --git a/packages/builder/src/helpers/searchFields.js b/packages/builder/src/helpers/searchFields.js index 3f3ca1ce7e..650e04a680 100644 --- a/packages/builder/src/helpers/searchFields.js +++ b/packages/builder/src/helpers/searchFields.js @@ -17,15 +17,15 @@ export function getTableFields(linkField) { } export function getFields(fields, { allowLinks } = { allowLinks: true }) { - let fieldNames = fields.filter( + let filteredFields = fields.filter( field => !BannedSearchTypes.includes(field.type) ) if (allowLinks) { const linkFields = fields.filter(field => field.type === "link") for (let linkField of linkFields) { // only allow one depth of SQL relationship filtering - fieldNames = fieldNames.concat(getTableFields(linkField)) + filteredFields = filteredFields.concat(getTableFields(linkField)) } } - return fieldNames + return filteredFields } diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index 71c54db4da..2341eef3b2 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -67,6 +67,7 @@ $: dataContext = { rows: $fetch.rows, info: $fetch.info, + datasource: dataSource || {}, schema: $fetch.schema, rowsLength: $fetch.rows.length, diff --git a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte index 20909d011c..6a114afe3e 100644 --- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte +++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte @@ -11,11 +11,14 @@ export let size = "M" const component = getContext("component") - const { builderStore, ActionTypes, getAction } = getContext("sdk") + const { builderStore, ActionTypes, getAction, fetchDatasourceSchema } = + getContext("sdk") let modal let tmpFilters = [] let filters = [] + let schemaLoaded = false, + schema $: dataProviderId = dataProvider?.id $: addExtension = getAction( @@ -26,7 +29,7 @@ dataProviderId, ActionTypes.RemoveDataProviderQueryExtension ) - $: schema = dataProvider?.schema + $: fetchSchema(dataProvider || {}) $: schemaFields = getSchemaFields(schema, allowedFields) // Add query extension to data provider @@ -39,7 +42,20 @@ } } - const getSchemaFields = (schema, allowedFields) => { + async function fetchSchema(dataProvider) { + const datasource = dataProvider?.datasource + if (datasource) { + schema = await fetchDatasourceSchema(datasource, { + enrichRelationships: true, + }) + } + schemaLoaded = true + } + + function getSchemaFields(schema, allowedFields) { + if (!schemaLoaded) { + return {} + } let clonedSchema = {} if (!allowedFields?.length) { clonedSchema = schema @@ -68,18 +84,20 @@ }) -