From 7f0686380c7ee61c211ef7af4ea8861f6cc5d16c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 28 Sep 2020 17:36:59 +0100 Subject: [PATCH] Adding main work to handle creating, deleting and managing the link docs between different linked records. --- packages/server/src/api/controllers/client.js | 2 +- .../server/src/api/controllers/instance.js | 18 ++ packages/server/src/api/controllers/model.js | 3 - packages/server/src/db/linkedRecords.js | 73 ------ .../src/db/linkedRecords/LinkController.js | 210 ++++++++++++++++++ packages/server/src/db/linkedRecords/index.js | 109 +++++++++ 6 files changed, 338 insertions(+), 77 deletions(-) delete mode 100644 packages/server/src/db/linkedRecords.js create mode 100644 packages/server/src/db/linkedRecords/LinkController.js create mode 100644 packages/server/src/db/linkedRecords/index.js diff --git a/packages/server/src/api/controllers/client.js b/packages/server/src/api/controllers/client.js index 400ccde358..31f901001a 100644 --- a/packages/server/src/api/controllers/client.js +++ b/packages/server/src/api/controllers/client.js @@ -25,7 +25,7 @@ const getClientId = ctx => { (ctx.body && ctx.body.clientId) || env.CLIENT_ID if (!clientId) { - ctx.throw(400, "ClientId not suplied") + ctx.throw(400, "ClientId not supplied") } return clientId } diff --git a/packages/server/src/api/controllers/instance.js b/packages/server/src/api/controllers/instance.js index eb7b205240..c74befcc19 100644 --- a/packages/server/src/api/controllers/instance.js +++ b/packages/server/src/api/controllers/instance.js @@ -19,6 +19,8 @@ exports.create = async function(ctx) { applicationId: appId, }, views: { + // view collation information, read before writing any complex views: + // https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification by_username: { map: function(doc) { if (doc.type === "user") { @@ -31,6 +33,22 @@ exports.create = async function(ctx) { emit([doc.type], doc._id) }.toString(), }, + by_link: { + map: function(doc) { + if (doc.type === "link") { + let doc1 = doc.doc1 + let doc2 = doc.doc2 + emit([doc1.modelId, 1, doc1.fieldName, doc1.recordId], { + _id: doc2.recordId, + }) + emit([doc2.modelId, 1, doc2.fieldName, doc2.recordId], { + _id: doc1.recordId, + }) + emit([doc1.modelId, 2, doc1.recordId], { _id: doc2.recordId }) + emit([doc2.modelId, 2, doc2.recordId], { _id: doc1.recordId }) + } + }.toString(), + }, by_automation_trigger: { map: function(doc) { if (doc.type === "automation") { diff --git a/packages/server/src/api/controllers/model.js b/packages/server/src/api/controllers/model.js index 753be501a2..b2776650e9 100644 --- a/packages/server/src/api/controllers/model.js +++ b/packages/server/src/api/controllers/model.js @@ -63,9 +63,6 @@ exports.save = async function(ctx) { name: modelToSave.name, type: "link", modelId: modelToSave._id, - constraints: { - type: "array", - }, } await db.put(linkedModel) } diff --git a/packages/server/src/db/linkedRecords.js b/packages/server/src/db/linkedRecords.js deleted file mode 100644 index 877e290cea..0000000000 --- a/packages/server/src/db/linkedRecords.js +++ /dev/null @@ -1,73 +0,0 @@ -const CouchDB = require("./index") -const emitter = require("../events/index") -const InMemoryQueue = require("../utilities/queue/inMemoryQueue") - -/** - * This functionality makes sure that when records with links are created, updated or deleted they are processed - * correctly - making sure that no stale links are left around and that all links have been made successfully. - */ - -const EventType = { - RECORD_SAVE: "record:save", - RECORD_UPDATE: "record:update", - RECORD_DELETE: "record:delete", - MODEL_SAVE: "model:save", - MODEL_DELETE: "model:delete", -} -const linkedRecordQueue = new InMemoryQueue("linkedRecordQueue") - -function createEmitterCallback(eventName) { - emitter.on(eventName, function(event) { - if (!event || !event.record || !event.record.modelId) { - return - } - linkedRecordQueue.add({ - type: eventName, - event, - }) - }) -} - -for (let typeKey of Object.keys(EventType)) { - createEmitterCallback(EventType[typeKey]) -} - -function doesModelHaveLinkedRecords(model) { - for (let key of Object.keys(model.schema)) { - const { type } = model.schema[key] - if (type === "link") { - return true - } - } - return false -} - -linkedRecordQueue.process(async job => { - let event = job.data - // can't operate without these properties - if (event.instanceId == null || event.modelId == null) { - return - } - const db = new CouchDB(event.instanceId) - let model = event.model == null ? await db.get(event.modelId) : event.model - // model doesn't have links, can stop here - if (!doesModelHaveLinkedRecords(model)) { - return - } - // no linked records to operate on - if (model == null) { - return - } - switch (event.type) { - case EventType.RECORD_SAVE: - break - case EventType.RECORD_UPDATE: - break - case EventType.RECORD_DELETE: - break - case EventType.MODEL_SAVE: - break - case EventType.MODEL_DELETE: - break - } -}) diff --git a/packages/server/src/db/linkedRecords/LinkController.js b/packages/server/src/db/linkedRecords/LinkController.js new file mode 100644 index 0000000000..7125aad990 --- /dev/null +++ b/packages/server/src/db/linkedRecords/LinkController.js @@ -0,0 +1,210 @@ +const CouchDB = require("../index") +const linkedRecords = require("./index") + +/** + * Creates a new link document structure which can be put to the database. It is important to + * note that while this talks about linker/linked the link is bi-directional and for all intent + * and purposes it does not matter from which direction the link was initiated. + * @param {string} modelId1 The ID of the first model (the linker). + * @param {string} modelId2 The ID of the second model (the linked). + * @param {string} fieldName1 The name of the field in the linker table. + * @param {string} fieldName2 The name of the field in the linked table. + * @param {string} recordId1 The ID of the record which is acting as the linker. + * @param {string} recordId2 The ID of the record which is acting as the linked. + * @constructor + */ +function LinkDocument( + modelId1, + fieldName1, + recordId1, + modelId2, + fieldName2, + recordId2 +) { + this.doc1 = { + modelId: modelId1, + fieldName: fieldName1, + recordId: recordId1, + } + this.doc2 = { + modelId: modelId2, + fieldName: fieldName2, + recordId: recordId2, + } +} + +class LinkController { + /** + * Create a new link controller which can be used to handle link updates for an event. + * @param {string} instanceId The instance in which updates will be carried out. + * @param {{modelId: string, model: object|undefined, record: object|undefined}} eventData data about + * what has occurred to drive this update - events are emitted when an operation that matters occurs. + */ + constructor(instanceId, eventData) { + this._instanceId = instanceId + this._db = new CouchDB(instanceId) + this._modelId = eventData.modelId + this._record = eventData.record + this._model = eventData.model + } + + /** + * Retrieves the model, if it was not already found in the eventData. + * @returns {Promise} This will return a model based on the event data, either + * if it was in the event already, or it uses the specified modelId to get it. + */ + async model() { + if (this._model == null) { + this._model = + this._model == null ? await this._db.get(this._modelId) : this._model + } + return this._model + } + + /** + * Checks if the model this was constructed with has any linking columns currently. + * If the model has not been retrieved this will retrieve it based on the eventData. + * @returns {Promise} True if there are any linked fields, otherwise it will return + * false. + */ + async doesModelHaveLinkedFields() { + const model = await this.model() + for (const fieldName of Object.keys(model.schema)) { + const { type } = model.schema[fieldName] + if (type === "link") { + return true + } + } + return false + } + + /** + * Utility function for main getLinkDocuments function - refer to it for functionality. + */ + getLinkDocs(fieldName, recordId) { + return linkedRecords.getLinkDocuments({ + instanceId: this._instanceId, + modelId: this._modelId, + fieldName, + recordId, + }) + } + + // all operations here will assume that the model + // this operation is related to has linked records + /** + * When a record is saved this will carry out the necessary operations to make sure + * the link has been created/updated. + * @returns {Promise} The operation has been completed and the link documents should now + * be accurate. + */ + async recordSaved() { + const model = await this.model() + const record = this._record + let operations = [] + for (let fieldName of Object.keys(model.schema)) { + const field = model.schema[fieldName] + if (field.type === "link") { + // get link docs to compare against + let currentLinkIds = await this.getLinkDocs(fieldName, record._id).map( + doc => doc._id + ) + let toLinkIds = record[fieldName] + for (let linkId of toLinkIds) { + if (currentLinkIds.indexOf(linkId) === -1) { + operations.push( + new LinkDocument( + model._id, + fieldName, + record._id, + field.modelId, + field.fieldName, + linkId + ) + ) + } + const toDeleteIds = currentLinkIds.filter( + id => toLinkIds.indexOf(id) === -1 + ) + operations.concat( + toDeleteIds.map(id => ({ _id: id, _deleted: true })) + ) + } + } + } + await this._db.bulkDocs(operations) + } + + /** + * When a record is deleted this will carry out the necessary operations to make sure + * any links that existed have been removed. + * @returns {Promise} The operation has been completed and the link documents should now + * be accurate. + */ + async recordDeleted() { + const record = this._record + // get link docs to compare against + let toDelete = await this.getLinkDocs(null, record._id).map(doc => { + return { + ...doc, + _deleted: true, + } + }) + await this._db.bulkDocs(toDelete) + } + + /** + * When a model is saved this will carry out the necessary operations to make sure + * any linked models are notified and updated correctly. + * @returns {Promise} The operation has been completed and the link documents should now + * be accurate. + */ + async modelSaved() { + const model = await this.model() + const schema = model.schema + for (const fieldName of Object.keys(schema)) { + const field = schema[fieldName] + if (field.type === "link") { + // create the link field in the other model + const linkedModel = await this._db.get(field.modelId) + linkedModel.schema[field.fieldName] = { + name: model.name, + type: "link", + modelId: model._id, + fieldName: fieldName, + } + await this._db.put(linkedModel) + } + } + } + + /** + * When a model is deleted this will carry out the necessary operations to make sure + * any linked models have the joining column correctly removed as well as removing any + * now stale linking documents. + * @returns {Promise} The operation has been completed and the link documents should now + * be accurate. + */ + async modelDeleted() { + const model = await this.model() + const schema = model.schema + for (const fieldName of Object.keys(schema)) { + let field = schema[fieldName] + if (field.type === "link") { + const linkedModel = await this._db.get(field.modelId) + delete linkedModel.schema[model.name] + await this._db.put(linkedModel) + } + } + // get link docs for this model and configure for deletion + let toDelete = await this.getLinkDocs().map(doc => { + return { + ...doc, + _deleted: true, + } + }) + await this._db.bulkDocs(toDelete) + } +} + +module.exports = LinkController diff --git a/packages/server/src/db/linkedRecords/index.js b/packages/server/src/db/linkedRecords/index.js new file mode 100644 index 0000000000..47c578eff4 --- /dev/null +++ b/packages/server/src/db/linkedRecords/index.js @@ -0,0 +1,109 @@ +const emitter = require("../../events") +const InMemoryQueue = require("../../utilities/queue/inMemoryQueue") +const LinkController = require("./LinkController") +const CouchDB = require("../index") + +/** + * This functionality makes sure that when records with links are created, updated or deleted they are processed + * correctly - making sure that no stale links are left around and that all links have been made successfully. + */ + +const EventType = { + RECORD_SAVE: "record:save", + RECORD_UPDATE: "record:update", + RECORD_DELETE: "record:delete", + MODEL_SAVE: "model:save", + MODEL_DELETE: "model:delete", +} + +const linkedRecordQueue = new InMemoryQueue("linkedRecordQueue") + +function createEmitterCallback(eventName) { + emitter.on(eventName, function(event) { + if (!event || !event.record || !event.record.modelId) { + return + } + linkedRecordQueue.add({ + type: eventName, + event, + }) + }) +} + +for (let typeKey of Object.keys(EventType)) { + createEmitterCallback(EventType[typeKey]) +} + +linkedRecordQueue.process(async job => { + let data = job.data + // can't operate without these properties + if (data.instanceId == null || data.modelId == null) { + return + } + // link controller exists to help manage state, the operation + // of updating links is a particularly stateful task + let linkController = new LinkController(data.instanceId, data) + // model doesn't have links, can stop here + if (!(await linkController.doesModelHaveLinkedFields())) { + return + } + // carry out the logic at a top level so that we can handle + // multiple operations for a single queue entry if desired + switch (data.type) { + case EventType.RECORD_SAVE: + case EventType.RECORD_UPDATE: + await linkController.recordSaved() + break + case EventType.RECORD_DELETE: + await linkController.recordDeleted() + break + case EventType.MODEL_SAVE: + await linkController.modelSaved() + break + case EventType.MODEL_DELETE: + await linkController.modelDeleted() + break + } +}) + +/** + * Gets the linking documents, not the linked documents themselves. + * @param {string} instanceId The instance in which we are searching for linked records. + * @param {string} modelId The model which we are searching for linked records against. + * @param {string|null} fieldName The name of column/field which is being altered, only looking for + * linking documents that are related to it. If this is not specified then the table level will be assumed. + * @param {string|null} recordId The ID of the record which we want to find linking documents for - + * if this is not specified then it will assume model or field level depending on whether the + * field name has been specified. + * @param {boolean|null} includeDoc whether to include docs in the response call, this is considerably slower so only + * use this if actually interested in the docs themselves. + * @returns {Promise} This will return an array of the linking documents that were found + * (if any). + */ +module.exports.getLinkDocuments = async ({ + instanceId, + modelId, + fieldName, + recordId, + includeDoc, +}) => { + const db = new CouchDB(instanceId) + let params + if (fieldName != null && recordId != null) { + params = { key: [modelId, 1, fieldName, recordId] } + } else if (fieldName != null && recordId == null) { + params = { + startKey: [modelId, 1, fieldName], + endKey: [modelId, 1, fieldName, {}], + } + } else if (fieldName == null && recordId != null) { + params = { key: [modelId, 2, recordId] } + } + // only model is known + else { + params = { startKey: [modelId, 1], endKey: [modelId, 1, {}] } + } + params.include_docs = !!includeDoc + const response = await db.query("database/by_link", params) + return response.rows.map(row => row.doc) +}