From 98b7bff6786017461c2e220e0eda182d52cefff3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 29 Jun 2021 17:42:46 +0100 Subject: [PATCH] Implementing all return possibilities, now to implement creation. --- .../scripts/integrations/postgres/init.sql | 2 + .../src/api/controllers/row/external.js | 218 ++---------------- .../src/api/controllers/row/externalUtils.js | 189 +++++++++++++++ packages/server/src/definitions/common.ts | 4 + packages/server/src/integrations/base/sql.ts | 17 +- packages/server/src/integrations/postgres.ts | 60 +++-- 6 files changed, 277 insertions(+), 213 deletions(-) create mode 100644 packages/server/src/api/controllers/row/externalUtils.js diff --git a/packages/server/scripts/integrations/postgres/init.sql b/packages/server/scripts/integrations/postgres/init.sql index de8faa437a..37835af4a7 100644 --- a/packages/server/scripts/integrations/postgres/init.sql +++ b/packages/server/scripts/integrations/postgres/init.sql @@ -32,9 +32,11 @@ CREATE TABLE Products_Tasks ( ); INSERT INTO Persons (PersonID, FirstName, LastName, Address, City) VALUES (1, 'Mike', 'Hughes', '123 Fake Street', 'Belfast'); INSERT INTO Tasks (TaskID, PersonID, TaskName) VALUES (1, 1, 'assembling'); +INSERT INTO Tasks (TaskID, PersonID, TaskName) VALUES (2, 1, 'processing'); INSERT INTO Products (ProductID, ProductName) VALUES (1, 'Computers'); INSERT INTO Products (ProductID, ProductName) VALUES (2, 'Laptops'); INSERT INTO Products (ProductID, ProductName) VALUES (3, 'Chairs'); INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 1); INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (2, 1); INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (3, 1); +INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 2); diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 6c5eea9f9c..2da4ecfa0a 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -2,204 +2,20 @@ const { makeExternalQuery } = require("./utils") const { DataSourceOperation, SortDirection, - FieldTypes, } = require("../../../constants") const { getAllExternalTables } = require("../table/utils") const { breakExternalTableId, - generateRowIdField, breakRowIdField, } = require("../../../integrations/utils") -const { cloneDeep } = require("lodash/fp") - -function inputProcessing(row, table, allTables) { - if (!row) { - return row - } - let newRow = {}, manyRelationships = [] - for (let [key, field] of Object.entries(table.schema)) { - // currently excludes empty strings - if (!row[key]) { - continue - } - const isLink = field.type === FieldTypes.LINK - if (isLink && !field.through) { - // we don't really support composite keys for relationships, this is why [0] is used - newRow[key] = breakRowIdField(row[key][0])[0] - } else if (isLink && field.through) { - const linkTable = allTables.find(table => table._id === field.tableId) - // table has to exist for many to many - if (!linkTable) { - continue - } - row[key].map(relationship => { - // we don't really support composite keys for relationships, this is why [0] is used - manyRelationships.push({ - tableId: field.through, - [linkTable.primary]: breakRowIdField(relationship)[0], - // leave the ID for enrichment later - [table.primary]: `{{ id }}`, - }) - }) - } else { - newRow[key] = row[key] - } - } - return { row: newRow, manyRelationships } -} - -function generateIdForRow(row, table) { - if (!row) { - return - } - const primary = table.primary - // build id array - let idParts = [] - for (let field of primary) { - idParts.push(row[field]) - } - return generateRowIdField(idParts) -} - -function updateRelationshipColumns(rows, row, relationships, allTables) { - const columns = {} - for (let relationship of relationships) { - const linkedTable = allTables[relationship.tableName] - if (!linkedTable) { - continue - } - const display = linkedTable.primaryDisplay - const related = {} - if (display && row[display]) { - related.primaryDisplay = row[display] - } - related._id = row[relationship.to] - columns[relationship.from] = related - } - for (let [column, related] of Object.entries(columns)) { - if (!Array.isArray(rows[row._id][column])) { - rows[row._id][column] = [] - } - rows[row._id][column].push(related) - } - return rows -} - -async function insertManyRelationships(appId, json, relationships) { - const promises = [] - for (let relationship of relationships) { - const newJson = { - // copy over datasource stuff - endpoint: json.endpoint, - } - const { tableName } = breakExternalTableId(relationship.tableId) - delete relationship.tableId - newJson.endpoint.entityId = tableName - newJson.body = relationship - promises.push(makeExternalQuery(appId, newJson)) - } - await Promise.all(promises) -} - -function outputProcessing(rows, table, relationships, allTables) { - // if no rows this is what is returned? Might be PG only - if (rows[0].read === true) { - return [] - } - let finalRows = {} - for (let row of rows) { - row._id = generateIdForRow(row, table) - // this is a relationship of some sort - if (finalRows[row._id]) { - finalRows = updateRelationshipColumns( - finalRows, - row, - relationships, - allTables - ) - continue - } - const thisRow = {} - // filter the row down to what is actually the row (not joined) - for (let fieldName of Object.keys(table.schema)) { - thisRow[fieldName] = row[fieldName] - } - thisRow._id = row._id - thisRow.tableId = table._id - thisRow._rev = "rev" - finalRows[thisRow._id] = thisRow - // do this at end once its been added to the final rows - finalRows = updateRelationshipColumns( - finalRows, - row, - relationships, - allTables - ) - } - return Object.values(finalRows) -} - -function buildFilters(id, filters, table) { - const primary = table.primary - // if passed in array need to copy for shifting etc - let idCopy = cloneDeep(id) - if (filters) { - // need to map over the filters and make sure the _id field isn't present - for (let filter of Object.values(filters)) { - if (filter._id) { - const parts = breakRowIdField(filter._id) - for (let field of primary) { - filter[field] = parts.shift() - } - } - // make sure this field doesn't exist on any filter - delete filter._id - } - } - // there is no id, just use the user provided filters - if (!idCopy || !table) { - return filters - } - // if used as URL parameter it will have been joined - if (typeof idCopy === "string") { - idCopy = breakRowIdField(idCopy) - } - const equal = {} - for (let field of primary) { - // work through the ID and get the parts - equal[field] = idCopy.shift() - } - return { - equal, - } -} - -function buildRelationships(table, allTables) { - const relationships = [] - for (let [fieldName, field] of Object.entries(table.schema)) { - if (field.type !== FieldTypes.LINK) { - continue - } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - const linkTable = allTables.find(table => table._id === field.tableId) - // no table to link to, this is not a valid relationships - if (!linkTable) { - continue - } - const definition = { - from: fieldName || table.primary, - to: field.fieldName || linkTable.primary, - tableName: linkTableName, - through: undefined, - } - if (field.through) { - const { tableName: throughTableName } = breakExternalTableId(field.through) - definition.through = throughTableName - } - relationships.push(definition) - } - return relationships -} +const { + buildRelationships, + buildFilters, + inputProcessing, + outputProcessing, + generateIdForRow, +} = require("./externalUtils") +const { processObjectSync } = require("@budibase/string-templates") async function handleRequest( appId, @@ -207,7 +23,7 @@ async function handleRequest( tableId, { id, row, filters, sort, paginate } = {} ) { - let { datasourceId, tableName } = breakExternalTableId(tableId) + let {datasourceId, tableName} = breakExternalTableId(tableId) const tables = await getAllExternalTables(appId, datasourceId) const table = tables[tableName] if (!table) { @@ -247,7 +63,21 @@ async function handleRequest( // can't really use response right now const response = await makeExternalQuery(appId, json) // handle many to many relationships now if we know the ID (could be auto increment) - await insertManyRelationships(appId, json, processed.manyRelationships) + if (processed.manyRelationships) { + const promises = [] + for (let toInsert of processed.manyRelationships) { + const {tableName} = breakExternalTableId(toInsert.tableId) + delete toInsert.tableId + promises.push(makeExternalQuery(appId, { + endpoint: { + ...json.endpoint, + entityId: tableName, + }, + body: toInsert, + })) + } + await Promise.all(promises) + } // we searched for rows in someway if (operation === DataSourceOperation.READ && Array.isArray(response)) { return outputProcessing(response, table, relationships, tables) diff --git a/packages/server/src/api/controllers/row/externalUtils.js b/packages/server/src/api/controllers/row/externalUtils.js new file mode 100644 index 0000000000..46284ca99f --- /dev/null +++ b/packages/server/src/api/controllers/row/externalUtils.js @@ -0,0 +1,189 @@ +const { + breakExternalTableId, + generateRowIdField, + breakRowIdField, +} = require("../../../integrations/utils") +const { FieldTypes } = require("../../../constants") +const { cloneDeep } = require("lodash/fp") + +exports.inputProcessing = (row, table, allTables) => { + if (!row) { + return { row, manyRelationships: [] } + } + let newRow = {}, manyRelationships = [] + for (let [key, field] of Object.entries(table.schema)) { + // currently excludes empty strings + if (!row[key]) { + continue + } + const isLink = field.type === FieldTypes.LINK + if (isLink && !field.through) { + // we don't really support composite keys for relationships, this is why [0] is used + newRow[key] = breakRowIdField(row[key][0])[0] + } else if (isLink && field.through) { + const linkTable = allTables.find(table => table._id === field.tableId) + // table has to exist for many to many + if (!linkTable) { + continue + } + row[key].map(relationship => { + // we don't really support composite keys for relationships, this is why [0] is used + manyRelationships.push({ + tableId: field.through, + [linkTable.primary]: breakRowIdField(relationship)[0], + // leave the ID for enrichment later + [table.primary]: `{{ id }}`, + }) + }) + } else { + newRow[key] = row[key] + } + } + return { row: newRow, manyRelationships } +} + +exports.generateIdForRow = (row, table) => { + if (!row) { + return + } + const primary = table.primary + // build id array + let idParts = [] + for (let field of primary) { + idParts.push(row[field]) + } + return generateRowIdField(idParts) +} + +exports.updateRelationshipColumns = (rows, row, relationships, allTables) => { + const columns = {} + for (let relationship of relationships) { + const linkedTable = allTables[relationship.tableName] + if (!linkedTable) { + continue + } + const display = linkedTable.primaryDisplay + const related = {} + if (display && row[display]) { + related.primaryDisplay = row[display] + } + related._id = row[relationship.to] + columns[relationship.column] = related + } + for (let [column, related] of Object.entries(columns)) { + if (!Array.isArray(rows[row._id][column])) { + rows[row._id][column] = [] + } + // make sure relationship hasn't been found already + if (!rows[row._id][column].find(relation => relation._id === related._id)) { + rows[row._id][column].push(related) + } + } + return rows +} + +exports.outputProcessing = (rows, table, relationships, allTables) => { + // if no rows this is what is returned? Might be PG only + if (rows[0].read === true) { + return [] + } + let finalRows = {} + for (let row of rows) { + row._id = exports.generateIdForRow(row, table) + // this is a relationship of some sort + if (finalRows[row._id]) { + finalRows = exports.updateRelationshipColumns( + finalRows, + row, + relationships, + allTables + ) + continue + } + const thisRow = {} + // filter the row down to what is actually the row (not joined) + for (let fieldName of Object.keys(table.schema)) { + thisRow[fieldName] = row[fieldName] + } + thisRow._id = row._id + thisRow.tableId = table._id + thisRow._rev = "rev" + finalRows[thisRow._id] = thisRow + // do this at end once its been added to the final rows + finalRows = exports.updateRelationshipColumns( + finalRows, + row, + relationships, + allTables + ) + } + return Object.values(finalRows) +} + +exports.buildFilters = (id, filters, table) => { + const primary = table.primary + // if passed in array need to copy for shifting etc + let idCopy = cloneDeep(id) + if (filters) { + // need to map over the filters and make sure the _id field isn't present + for (let filter of Object.values(filters)) { + if (filter._id) { + const parts = breakRowIdField(filter._id) + for (let field of primary) { + filter[field] = parts.shift() + } + } + // make sure this field doesn't exist on any filter + delete filter._id + } + } + // there is no id, just use the user provided filters + if (!idCopy || !table) { + return filters + } + // if used as URL parameter it will have been joined + if (typeof idCopy === "string") { + idCopy = breakRowIdField(idCopy) + } + const equal = {} + for (let field of primary) { + // work through the ID and get the parts + equal[field] = idCopy.shift() + } + return { + equal, + } +} + +exports.buildRelationships = (table, allTables) => { + const relationships = [] + for (let [fieldName, field] of Object.entries(table.schema)) { + if (field.type !== FieldTypes.LINK) { + continue + } + const { tableName: linkTableName } = breakExternalTableId(field.tableId) + // no table to link to, this is not a valid relationships + if (!allTables[linkTableName]) { + continue + } + const linkTable = allTables[linkTableName] + const definition = { + // if no foreign key specified then use the name of the field in other table + from: field.foreignKey || table.primary[0], + to: field.fieldName, + tableName: linkTableName, + through: undefined, + // need to specify where to put this back into + column: fieldName, + } + if (field.through) { + const { tableName: throughTableName } = breakExternalTableId(field.through) + definition.through = throughTableName + // don't support composite keys for relationships + definition.from = table.primary[0] + definition.to = linkTable.primary[0] + } + relationships.push(definition) + } + return relationships +} diff --git a/packages/server/src/definitions/common.ts b/packages/server/src/definitions/common.ts index 0690f2954e..26c12bd3c9 100644 --- a/packages/server/src/definitions/common.ts +++ b/packages/server/src/definitions/common.ts @@ -9,6 +9,10 @@ export interface TableSchema { type: string fieldName?: string name: string + tableId?: string + relationshipType?: string + through?: string + foreignKey?: string constraints?: { type?: string email?: boolean diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index a04e258415..743d5e159a 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -79,17 +79,22 @@ function addRelationships( return query } for (let relationship of relationships) { - const from = `${fromTable}.${relationship.from}` - const to = `${relationship.tableName}.${relationship.to}` + const from = relationship.from, + to = relationship.to, + toTable = relationship.tableName if (!relationship.through) { // @ts-ignore - query = query.innerJoin(relationship.tableName, from, to) + query = query.innerJoin( + toTable, + `${fromTable}.${from}`, + `${relationship.tableName}.${to}` + ) } else { - const through = relationship + const throughTable = relationship.through query = query // @ts-ignore - .innerJoin(through.tableName, from, through.from) - .innerJoin(relationship.tableName, to, through.to) + .innerJoin(throughTable, `${fromTable}.${from}`, `${throughTable}.${from}`) + .innerJoin(toTable, `${toTable}.${to}`, `${throughTable}.${to}`) } } return query diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index cb15937af2..40ff2de5b6 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -175,19 +175,53 @@ module PostgresModule { type, } - // // TODO: hack for testing - // if (tableName === "persons") { - // tables[tableName].primaryDisplay = "firstname" - // } - // if (columnName.toLowerCase() === "personid" && tableName === "tasks") { - // tables[tableName].schema[columnName] = { - // name: columnName, - // type: "link", - // tableId: buildExternalTableId(datasourceId, "persons"), - // relationshipType: "one-to-many", - // fieldName: "personid", - // } - // } + // TODO: hack for testing + if (tableName === "persons") { + tables[tableName].primaryDisplay = "firstname" + } + if (tableName === "products") { + tables[tableName].primaryDisplay = "productname" + } + if (tableName === "tasks") { + tables[tableName].primaryDisplay = "taskname" + } + if (tableName === "products") { + tables[tableName].schema["tasks"] = { + name: "tasks", + type: "link", + tableId: buildExternalTableId(datasourceId, "tasks"), + relationshipType: "many-to-many", + through: buildExternalTableId(datasourceId, "products_tasks"), + fieldName: "taskid", + } + } + if (tableName === "persons") { + tables[tableName].schema["tasks"] = { + name: "tasks", + type: "link", + tableId: buildExternalTableId(datasourceId, "tasks"), + relationshipType: "many-to-one", + fieldName: "personid", + } + } + if (tableName === "tasks") { + tables[tableName].schema["products"] = { + name: "products", + type: "link", + tableId: buildExternalTableId(datasourceId, "products"), + relationshipType: "many-to-many", + through: buildExternalTableId(datasourceId, "products_tasks"), + fieldName: "productid", + } + tables[tableName].schema["people"] = { + name: "people", + type: "link", + tableId: buildExternalTableId(datasourceId, "persons"), + relationshipType: "one-to-many", + fieldName: "personid", + foreignKey: "personid", + } + } } this.tables = tables }