From 3ffe00fe2f83dea02dbb7cae105a46431b1e54a8 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 10 Feb 2021 15:49:23 +0000 Subject: [PATCH 01/38] Make URL params available to client apps via context --- packages/client/src/components/Router.svelte | 16 +++++++--------- packages/client/src/components/Screen.svelte | 17 +++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/client/src/components/Router.svelte b/packages/client/src/components/Router.svelte index efa0e321aa..ddbe7b77e9 100644 --- a/packages/client/src/components/Router.svelte +++ b/packages/client/src/components/Router.svelte @@ -1,5 +1,5 @@ -{#each configs as config (config.id)} +{#key config.id}
-{/each} +{/key} diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 857d1dd2ad..03af91a2b6 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -9,7 +9,10 @@ const { ViewNames, } = require("../../db/utils") const usersController = require("./user") -const { coerceRowValues, enrichRows } = require("../../utilities") +const { + inputProcessing, + outputProcessing, +} = require("../../utilities/rowProcessor") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -64,7 +67,7 @@ exports.patch = async function(ctx) { row[key] = patchfields[key] } - row = coerceRowValues(row, table) + row = inputProcessing(ctx.user, table, row) const validateResult = await validate({ row, @@ -134,7 +137,7 @@ exports.save = async function(ctx) { const table = await db.get(row.tableId) - row = coerceRowValues(row, table) + row = inputProcessing(ctx.user, table, row) const validateResult = await validate({ row, @@ -204,7 +207,7 @@ exports.fetchView = async function(ctx) { schema: {}, } } - ctx.body = await enrichRows(appId, table, response.rows) + ctx.body = await outputProcessing(appId, table, response.rows) } if (calculation === CALCULATION_TYPES.STATS) { @@ -247,7 +250,7 @@ exports.fetchTableRows = async function(ctx) { ) rows = response.rows.map(row => row.doc) } - ctx.body = await enrichRows(appId, table, rows) + ctx.body = await outputProcessing(appId, table, rows) } exports.find = async function(ctx) { @@ -256,7 +259,7 @@ exports.find = async function(ctx) { try { const table = await db.get(ctx.params.tableId) const row = await findRow(db, appId, ctx.params.tableId, ctx.params.rowId) - ctx.body = await enrichRows(appId, table, row) + ctx.body = await outputProcessing(appId, table, row) } catch (err) { ctx.throw(400, err) } @@ -341,7 +344,7 @@ exports.fetchEnrichedRow = async function(ctx) { keys: linkVals.map(linkVal => linkVal.id), }) // need to include the IDs in these rows for any links they may have - let linkedRows = await enrichRows( + let linkedRows = await outputProcessing( appId, table, response.rows.map(row => row.doc) diff --git a/packages/server/src/api/routes/table.js b/packages/server/src/api/routes/table.js index da5c753b83..0e6b916c24 100644 --- a/packages/server/src/api/routes/table.js +++ b/packages/server/src/api/routes/table.js @@ -7,9 +7,31 @@ const { PermissionLevels, PermissionTypes, } = require("../../utilities/security/permissions") +const joiValidator = require("../../middleware/joi-validator") +const Joi = require("joi") const router = Router() +function generateSaveValidator() { + // prettier-ignore + return joiValidator.body(Joi.object({ + _id: Joi.string(), + _rev: Joi.string(), + type: Joi.string().valid("table"), + primaryDisplay: Joi.string(), + schema: Joi.object().required(), + name: Joi.string().required(), + views: Joi.object(), + autoColumns: Joi.object({ + createdBy: Joi.boolean(), + createdAt: Joi.boolean(), + updatedBy: Joi.boolean(), + updatedAt: Joi.boolean(), + }), + dataImport: Joi.object(), + }).unknown(true)) +} + router .get("/api/tables", authorized(BUILDER), tableController.fetch) .get( @@ -23,6 +45,7 @@ router // allows control over updating a table bodyResource("_id"), authorized(BUILDER), + generateSaveValidator(), tableController.save ) .post( diff --git a/packages/server/src/automations/triggers.js b/packages/server/src/automations/triggers.js index 6634016e3f..e4c91e5610 100644 --- a/packages/server/src/automations/triggers.js +++ b/packages/server/src/automations/triggers.js @@ -2,7 +2,7 @@ const CouchDB = require("../db") const emitter = require("../events/index") const InMemoryQueue = require("../utilities/queue/inMemoryQueue") const { getAutomationParams } = require("../db/utils") -const { coerceValue } = require("../utilities") +const { coerce } = require("../utilities/rowProcessor") let automationQueue = new InMemoryQueue("automationQueue") @@ -240,8 +240,8 @@ module.exports.externalTrigger = async function(automation, params) { // values are likely to be submitted as strings, so we shall convert to correct type const coercedFields = {} const fields = automation.definition.trigger.inputs.fields - for (let key in fields) { - coercedFields[key] = coerceValue(params.fields[key], fields[key]) + for (let key of Object.keys(fields)) { + coercedFields[key] = coerce(params.fields[key], fields[key]) } params.fields = coercedFields } diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index 31cc74b5e6..4cf01dc836 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -3,60 +3,9 @@ const { DocumentTypes, SEPARATOR } = require("../db/utils") const fs = require("fs") const { cloneDeep } = require("lodash/fp") const CouchDB = require("../db") -const { OBJ_STORE_DIRECTORY } = require("../constants") -const linkRows = require("../db/linkedRows") const APP_PREFIX = DocumentTypes.APP + SEPARATOR -/** - * A map of how we convert various properties in rows to each other based on the row type. - */ -const TYPE_TRANSFORM_MAP = { - link: { - "": [], - [null]: [], - [undefined]: undefined, - }, - options: { - "": "", - [null]: "", - [undefined]: undefined, - }, - string: { - "": "", - [null]: "", - [undefined]: undefined, - }, - longform: { - "": "", - [null]: "", - [undefined]: undefined, - }, - number: { - "": null, - [null]: null, - [undefined]: undefined, - parse: n => parseFloat(n), - }, - datetime: { - "": null, - [undefined]: undefined, - [null]: null, - }, - attachment: { - "": [], - [null]: [], - [undefined]: undefined, - }, - boolean: { - "": null, - [null]: null, - [undefined]: undefined, - true: true, - false: false, - }, -} - function confirmAppId(possibleAppId) { return possibleAppId && possibleAppId.startsWith(APP_PREFIX) ? possibleAppId @@ -159,43 +108,6 @@ exports.walkDir = (dirPath, callback) => { } } -/** - * This will coerce a value to the correct types based on the type transform map - * @param {object} row The value to coerce - * @param {object} type The type fo coerce to - * @returns {object} The coerced value - */ -exports.coerceValue = (value, type) => { - // eslint-disable-next-line no-prototype-builtins - if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(value)) { - return TYPE_TRANSFORM_MAP[type][value] - } else if (TYPE_TRANSFORM_MAP[type].parse) { - return TYPE_TRANSFORM_MAP[type].parse(value) - } - - return value -} - -/** - * This will coerce the values in a row to the correct types based on the type transform map and the - * table schema. - * @param {object} row The row which is to be coerced to correct values based on schema, this input - * row will not be updated. - * @param {object} table The table that has been retrieved from DB, this must contain the expected - * schema for the rows. - * @returns {object} The updated row will be returned with all values coerced. - */ -exports.coerceRowValues = (row, table) => { - const clonedRow = cloneDeep(row) - for (let [key, value] of Object.entries(clonedRow)) { - const field = table.schema[key] - if (!field) continue - - clonedRow[key] = exports.coerceValue(value, field.type) - } - return clonedRow -} - exports.getLogoUrl = () => { return "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg" } @@ -213,34 +125,3 @@ exports.getAllApps = async () => { .map(({ value }) => value) } } - -/** - * This function "enriches" the input rows with anything they are supposed to contain, for example - * link records or attachment links. - * @param {string} appId the ID of the application for which rows are being enriched. - * @param {object} table the table from which these rows came from originally, this is used to determine - * the schema of the rows and then enrich. - * @param {object[]} rows the rows which are to be enriched. - * @returns {object[]} the enriched rows will be returned. - */ -exports.enrichRows = async (appId, table, rows) => { - // attach any linked row information - const enriched = await linkRows.attachLinkInfo(appId, rows) - // update the attachments URL depending on hosting - if (env.CLOUD && env.SELF_HOSTED) { - for (let [property, column] of Object.entries(table.schema)) { - if (column.type === "attachment") { - for (let row of enriched) { - if (row[property] == null || row[property].length === 0) { - continue - } - row[property].forEach(attachment => { - attachment.url = `${OBJ_STORE_DIRECTORY}/${appId}/${attachment.url}` - attachment.url = attachment.url.replace("//", "/") - }) - } - } - } - } - return enriched -} diff --git a/packages/server/src/utilities/rowProcessor.js b/packages/server/src/utilities/rowProcessor.js new file mode 100644 index 0000000000..270f636aae --- /dev/null +++ b/packages/server/src/utilities/rowProcessor.js @@ -0,0 +1,119 @@ +const env = require("../environment") +const { OBJ_STORE_DIRECTORY } = require("../constants") +const linkRows = require("../db/linkedRows") +const { cloneDeep } = require("lodash/fp") + +/** + * A map of how we convert various properties in rows to each other based on the row type. + */ +const TYPE_TRANSFORM_MAP = { + link: { + "": [], + [null]: [], + [undefined]: undefined, + }, + options: { + "": "", + [null]: "", + [undefined]: undefined, + }, + string: { + "": "", + [null]: "", + [undefined]: undefined, + }, + longform: { + "": "", + [null]: "", + [undefined]: undefined, + }, + number: { + "": null, + [null]: null, + [undefined]: undefined, + parse: n => parseFloat(n), + }, + datetime: { + "": null, + [undefined]: undefined, + [null]: null, + }, + attachment: { + "": [], + [null]: [], + [undefined]: undefined, + }, + boolean: { + "": null, + [null]: null, + [undefined]: undefined, + true: true, + false: false, + }, +} + +/** + * This will coerce a value to the correct types based on the type transform map + * @param {any} value The value to coerce + * @param {string} type The type fo coerce to + * @returns {any} The coerced value + */ +exports.coerce = (value, type) => { + // eslint-disable-next-line no-prototype-builtins + if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(value)) { + return TYPE_TRANSFORM_MAP[type][value] + } else if (TYPE_TRANSFORM_MAP[type].parse) { + return TYPE_TRANSFORM_MAP[type].parse(value) + } + return value +} + +/** + * Given an input route this function will apply all the necessary pre-processing to it, such as coercion + * of column values or adding auto-column values. + * @param {object} user the user which is performing the input. + * @param {object} row the row which is being created/updated. + * @param {object} table the table which the row is being saved to. + * @returns {object} the row which has been prepared to be written to the DB. + */ +exports.inputProcessing = (user, table, row) => { + const clonedRow = cloneDeep(row) + for (let [key, value] of Object.entries(clonedRow)) { + const field = table.schema[key] + if (!field) continue + + clonedRow[key] = exports.coerce(value, field.type) + } + return clonedRow +} + +/** + * This function enriches the input rows with anything they are supposed to contain, for example + * link records or attachment links. + * @param {string} appId the ID of the application for which rows are being enriched. + * @param {object} table the table from which these rows came from originally, this is used to determine + * the schema of the rows and then enrich. + * @param {object[]} rows the rows which are to be enriched. + * @returns {object[]} the enriched rows will be returned. + */ +exports.outputProcessing = async (appId, table, rows) => { + // attach any linked row information + const outputRows = await linkRows.attachLinkInfo(appId, rows) + // update the attachments URL depending on hosting + if (env.CLOUD && env.SELF_HOSTED) { + for (let [property, column] of Object.entries(table.schema)) { + if (column.type === "attachment") { + for (let row of outputRows) { + if (row[property] == null || row[property].length === 0) { + continue + } + row[property].forEach(attachment => { + attachment.url = `${OBJ_STORE_DIRECTORY}/${appId}/${attachment.url}` + attachment.url = attachment.url.replace("//", "/") + }) + } + } + } + } + return outputRows +} From ca20cbeecac510597b27ea331c6da5753d948e81 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 10 Feb 2021 17:55:19 +0000 Subject: [PATCH 03/38] First lot of work to update the auto fields into schema. --- .../modals/CreateTableModal.svelte | 40 ++++++++++++------- .../builder/src/constants/backend/index.js | 7 ++++ packages/server/src/api/controllers/row.js | 5 ++- packages/server/src/api/controllers/table.js | 3 +- packages/server/src/api/routes/table.js | 6 --- packages/server/src/constants/index.js | 12 ++++++ .../src/db/linkedRows/LinkController.js | 18 +++++---- .../server/src/db/linkedRows/linkUtils.js | 3 +- packages/server/src/utilities/rowProcessor.js | 19 ++++----- 9 files changed, 73 insertions(+), 40 deletions(-) diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index b6bf9ec8d7..b9a2307433 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -14,6 +14,8 @@ import { NEW_ROW_TEMPLATE } from "builderStore/store/screenTemplates/newRowScreen" import { ROW_DETAIL_TEMPLATE } from "builderStore/store/screenTemplates/rowDetailScreen" import { ROW_LIST_TEMPLATE } from "builderStore/store/screenTemplates/rowListScreen" + import { FIELDS } from "constants/backend" + import { cloneDeep } from "lodash/fp" const defaultScreens = [ NEW_ROW_TEMPLATE, @@ -27,11 +29,22 @@ let error = "" let createAutoscreens = true let autoColumns = { - createdBy: false, - createdAt: false, - updatedBy: false, - updatedAt: false, - autoNumber: false, + createdBy: true, + createdAt: true, + updatedBy: true, + updatedAt: true, + autoID: true, + } + + function addAutoColumns(schema) { + for (let [property, enabled] of Object.entries(autoColumns)) { + if (!enabled) { + continue + } + const autoColDef = cloneDeep(FIELDS.AUTO) + autoColDef.subtype = property + schema[property] = autoColDef + } } function checkValid(evt) { @@ -46,8 +59,7 @@ async function saveTable() { let newTable = { name, - schema: dataImport.schema || {}, - autoColumns, + schema: addAutoColumns(dataImport.schema || {}), dataImport, } @@ -100,7 +112,7 @@ bind:value={name} {error} />
- +
+ text="Auto ID" + bind:checked={autoColumns.autoID} />
*) { - margin-top: 10px; + margin-bottom: 10px; } + .toggle-2 :global(> *) { - margin-top: 10px; - margin-left: 10px; + margin-bottom: 10px; + margin-left: 20px; } diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index 80eaf613f8..9ef95f3c61 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -80,6 +80,13 @@ export const FIELDS = { presence: false, }, }, + AUTO: { + name: "Auto Column", + icon: "ri-magic-line", + type: "auto", + // no constraints for auto-columns + // these are fully created serverside + } } export const FILE_TYPES = { diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 0c4163aa22..34f027002c 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -13,6 +13,7 @@ const { inputProcessing, outputProcessing, } = require("../../utilities/rowProcessor") +const { FieldTypes } = require("../../constants") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -261,7 +262,7 @@ exports.search = async function(ctx) { const table = await db.get(ctx.params.tableId) - ctx.body = await enrichRows(appId, table, rows) + ctx.body = await outputProcessing(appId, table, rows) } exports.fetchTableRows = async function(ctx) { @@ -384,7 +385,7 @@ exports.fetchEnrichedRow = async function(ctx) { // insert the link rows in the correct place throughout the main row for (let fieldName of Object.keys(table.schema)) { let field = table.schema[fieldName] - if (field.type === "link") { + if (field.type === FieldTypes.LINK) { row[fieldName] = linkedRows.filter( linkRow => linkRow.tableId === field.tableId ) diff --git a/packages/server/src/api/controllers/table.js b/packages/server/src/api/controllers/table.js index 1fa77b0b2b..d3e7213fdb 100644 --- a/packages/server/src/api/controllers/table.js +++ b/packages/server/src/api/controllers/table.js @@ -8,6 +8,7 @@ const { generateRowID, } = require("../../db/utils") const { isEqual } = require("lodash/fp") +const { FieldTypes } = require("../../constants") async function checkForColumnUpdates(db, oldTable, updatedTable) { let updatedRows @@ -91,7 +92,7 @@ exports.save = async function(ctx) { } // rename row fields when table column is renamed - if (_rename && tableToSave.schema[_rename.updated].type === "link") { + if (_rename && tableToSave.schema[_rename.updated].type === FieldTypes.LINK) { ctx.throw(400, "Cannot rename a linked column.") } else if (_rename && tableToSave.primaryDisplay === _rename.old) { ctx.throw(400, "Cannot rename the display column.") diff --git a/packages/server/src/api/routes/table.js b/packages/server/src/api/routes/table.js index 0e6b916c24..7c7f45afeb 100644 --- a/packages/server/src/api/routes/table.js +++ b/packages/server/src/api/routes/table.js @@ -22,12 +22,6 @@ function generateSaveValidator() { schema: Joi.object().required(), name: Joi.string().required(), views: Joi.object(), - autoColumns: Joi.object({ - createdBy: Joi.boolean(), - createdAt: Joi.boolean(), - updatedBy: Joi.boolean(), - updatedAt: Joi.boolean(), - }), dataImport: Joi.object(), }).unknown(true)) } diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index 2e18de98af..54dedbcf11 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -39,6 +39,18 @@ const USERS_TABLE_SCHEMA = { primaryDisplay: "email", } +exports.FieldTypes = { + STRING: "string", + LONGFORM: "longform", + OPTIONS: "options", + NUMBER: "number", + BOOLEAN: "boolean", + DATETIME: "datetime", + ATTACHMENT: "attachment", + LINK: "link", + AUTO: "auto", +} + exports.AuthTypes = AuthTypes exports.USERS_TABLE_SCHEMA = USERS_TABLE_SCHEMA exports.BUILDER_CONFIG_DB = "builder-config-db" diff --git a/packages/server/src/db/linkedRows/LinkController.js b/packages/server/src/db/linkedRows/LinkController.js index 061a9ac1ae..ca64cb3430 100644 --- a/packages/server/src/db/linkedRows/LinkController.js +++ b/packages/server/src/db/linkedRows/LinkController.js @@ -2,6 +2,7 @@ const CouchDB = require("../index") const { IncludeDocs, getLinkDocuments } = require("./linkUtils") const { generateLinkID } = require("../utils") const Sentry = require("@sentry/node") +const { FieldTypes } = require("../../constants") /** * Creates a new link document structure which can be put to the database. It is important to @@ -26,7 +27,7 @@ function LinkDocument( // build the ID out of unique references to this link document this._id = generateLinkID(tableId1, tableId2, rowId1, rowId2) // required for referencing in view - this.type = "link" + this.type = FieldTypes.LINK this.doc1 = { tableId: tableId1, fieldName: fieldName1, @@ -75,7 +76,7 @@ class LinkController { } for (let fieldName of Object.keys(table.schema)) { const { type } = table.schema[fieldName] - if (type === "link") { + if (type === FieldTypes.LINK) { return true } } @@ -123,7 +124,7 @@ class LinkController { // get the links this row wants to make const rowField = row[fieldName] const field = table.schema[fieldName] - if (field.type === "link" && rowField != null) { + if (field.type === FieldTypes.LINK && rowField != null) { // check which links actual pertain to the update in this row const thisFieldLinkDocs = linkDocs.filter( linkDoc => @@ -234,7 +235,7 @@ class LinkController { const schema = table.schema for (let fieldName of Object.keys(schema)) { const field = schema[fieldName] - if (field.type === "link") { + if (field.type === FieldTypes.LINK) { // handle this in a separate try catch, want // the put to bubble up as an error, if can't update // table for some reason @@ -247,7 +248,7 @@ class LinkController { // create the link field in the other table linkedTable.schema[field.fieldName] = { name: field.fieldName, - type: "link", + type: FieldTypes.LINK, // these are the props of the table that initiated the link tableId: table._id, fieldName: fieldName, @@ -274,7 +275,10 @@ class LinkController { for (let fieldName of Object.keys(oldTable.schema)) { const field = oldTable.schema[fieldName] // this field has been removed from the table schema - if (field.type === "link" && newTable.schema[fieldName] == null) { + if ( + field.type === FieldTypes.LINK && + newTable.schema[fieldName] == null + ) { await this.removeFieldFromTable(fieldName) } } @@ -295,7 +299,7 @@ class LinkController { for (let fieldName of Object.keys(schema)) { const field = schema[fieldName] try { - if (field.type === "link") { + if (field.type === FieldTypes.LINK) { const linkedTable = await this._db.get(field.tableId) delete linkedTable.schema[field.fieldName] await this._db.put(linkedTable) diff --git a/packages/server/src/db/linkedRows/linkUtils.js b/packages/server/src/db/linkedRows/linkUtils.js index cb669cf5c7..ee22a87410 100644 --- a/packages/server/src/db/linkedRows/linkUtils.js +++ b/packages/server/src/db/linkedRows/linkUtils.js @@ -1,6 +1,7 @@ const CouchDB = require("../index") const Sentry = require("@sentry/node") const { ViewNames, getQueryIndex } = require("../utils") +const { FieldTypes } = require("../../constants") /** * Only needed so that boolean parameters are being used for includeDocs @@ -23,7 +24,7 @@ exports.createLinkView = async appId => { const designDoc = await db.get("_design/database") const view = { map: function(doc) { - if (doc.type === "link") { + if (doc.type === FieldTypes.LINK) { let doc1 = doc.doc1 let doc2 = doc.doc2 emit([doc1.tableId, doc1.rowId], { diff --git a/packages/server/src/utilities/rowProcessor.js b/packages/server/src/utilities/rowProcessor.js index 270f636aae..7ce080cf9f 100644 --- a/packages/server/src/utilities/rowProcessor.js +++ b/packages/server/src/utilities/rowProcessor.js @@ -2,48 +2,49 @@ const env = require("../environment") const { OBJ_STORE_DIRECTORY } = require("../constants") const linkRows = require("../db/linkedRows") const { cloneDeep } = require("lodash/fp") +const { FieldTypes } = require("../constants") /** * A map of how we convert various properties in rows to each other based on the row type. */ const TYPE_TRANSFORM_MAP = { - link: { + [FieldTypes.LINK]: { "": [], [null]: [], [undefined]: undefined, }, - options: { + [FieldTypes.OPTIONS]: { "": "", [null]: "", [undefined]: undefined, }, - string: { + [FieldTypes.STRING]: { "": "", [null]: "", [undefined]: undefined, }, - longform: { + [FieldTypes.LONGFORM]: { "": "", [null]: "", [undefined]: undefined, }, - number: { + [FieldTypes.NUMBER]: { "": null, [null]: null, [undefined]: undefined, parse: n => parseFloat(n), }, - datetime: { + [FieldTypes.DATETIME]: { "": null, [undefined]: undefined, [null]: null, }, - attachment: { + [FieldTypes.ATTACHMENT]: { "": [], [null]: [], [undefined]: undefined, }, - boolean: { + [FieldTypes.BOOLEAN]: { "": null, [null]: null, [undefined]: undefined, @@ -102,7 +103,7 @@ exports.outputProcessing = async (appId, table, rows) => { // update the attachments URL depending on hosting if (env.CLOUD && env.SELF_HOSTED) { for (let [property, column] of Object.entries(table.schema)) { - if (column.type === "attachment") { + if (column.type === FieldTypes.ATTACHMENT) { for (let row of outputRows) { if (row[property] == null || row[property].length === 0) { continue From 41b1920b272de455197a2b74ece0308d67e72f83 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 12 Feb 2021 11:17:13 +0000 Subject: [PATCH 04/38] Avoid relationship picker making rows calls for empty string ID --- .../src/LinkedRowSelector.svelte | 59 ------------------- .../src/forms/RelationshipField.svelte | 4 +- 2 files changed, 2 insertions(+), 61 deletions(-) delete mode 100644 packages/standard-components/src/LinkedRowSelector.svelte diff --git a/packages/standard-components/src/LinkedRowSelector.svelte b/packages/standard-components/src/LinkedRowSelector.svelte deleted file mode 100644 index a2956d6b93..0000000000 --- a/packages/standard-components/src/LinkedRowSelector.svelte +++ /dev/null @@ -1,59 +0,0 @@ - - -{#if linkedTable != null} - {#if linkedTable.primaryDisplay == null} - {#if showLabel} - - {/if} - - {:else} - - {#each allRows as row} - - {/each} - - {/if} -{/if} diff --git a/packages/standard-components/src/forms/RelationshipField.svelte b/packages/standard-components/src/forms/RelationshipField.svelte index f09595606f..8109283f6e 100644 --- a/packages/standard-components/src/forms/RelationshipField.svelte +++ b/packages/standard-components/src/forms/RelationshipField.svelte @@ -24,7 +24,7 @@ $: fetchTable(linkedTableId) const fetchTable = async id => { - if (id != null) { + if (id) { const result = await API.fetchTableDefinition(id) if (!result.error) { tableDefinition = result @@ -33,7 +33,7 @@ } const fetchRows = async id => { - if (id != null) { + if (id) { const rows = await API.fetchTableData(id) options = rows && !rows.error ? rows : [] } From 4f1a0ac6451b293333f7b6ab5de178e5a51a08ad Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Feb 2021 20:34:54 +0000 Subject: [PATCH 05/38] Fixing an issue with RBAC, there was a mutable issue where a server builtin resource was getting updated, fixed this by not exposing the mutable structure, instead exposing a function which provides a new object everytime. --- .../server/src/api/controllers/permission.js | 9 ++++----- packages/server/src/api/controllers/role.js | 5 +++-- packages/server/src/middleware/authenticated.js | 5 +++-- .../src/utilities/security/permissions.js | 11 ++++++++--- packages/server/src/utilities/security/roles.js | 17 +++++++++++------ .../server/src/utilities/security/utilities.js | 4 ++-- 6 files changed, 31 insertions(+), 20 deletions(-) diff --git a/packages/server/src/api/controllers/permission.js b/packages/server/src/api/controllers/permission.js index c505332c61..56c9d3f8c6 100644 --- a/packages/server/src/api/controllers/permission.js +++ b/packages/server/src/api/controllers/permission.js @@ -1,5 +1,5 @@ const { - BUILTIN_PERMISSIONS, + getBuiltinPermissions, PermissionLevels, isPermissionLevelHigherThanRead, higherPermission, @@ -8,11 +8,10 @@ const { isBuiltin, getDBRoleID, getExternalRoleID, - BUILTIN_ROLES, + getBuiltinRoles, } = require("../../utilities/security/roles") const { getRoleParams } = require("../../db/utils") const CouchDB = require("../../db") -const { cloneDeep } = require("lodash/fp") const { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, @@ -65,7 +64,7 @@ async function updatePermissionOnRole( // the permission is for a built in, make sure it exists if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) { - const builtin = cloneDeep(BUILTIN_ROLES[roleId]) + const builtin = getBuiltinRoles()[roleId] builtin._id = getDBRoleID(builtin._id) dbRoles.push(builtin) } @@ -110,7 +109,7 @@ async function updatePermissionOnRole( } exports.fetchBuiltin = function(ctx) { - ctx.body = Object.values(BUILTIN_PERMISSIONS) + ctx.body = Object.values(getBuiltinPermissions()) } exports.fetchLevels = function(ctx) { diff --git a/packages/server/src/api/controllers/role.js b/packages/server/src/api/controllers/role.js index 440dbfde35..2c29d1030e 100644 --- a/packages/server/src/api/controllers/role.js +++ b/packages/server/src/api/controllers/role.js @@ -1,6 +1,6 @@ const CouchDB = require("../../db") const { - BUILTIN_ROLES, + getBuiltinRoles, BUILTIN_ROLE_IDS, Role, getRole, @@ -58,10 +58,11 @@ exports.fetch = async function(ctx) { }) ) let roles = body.rows.map(row => row.doc) + const builtinRoles = getBuiltinRoles() // need to combine builtin with any DB record of them (for sake of permissions) for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) { - const builtinRole = BUILTIN_ROLES[builtinRoleId] + const builtinRole = builtinRoles[builtinRoleId] const dbBuiltin = roles.filter( dbRole => getExternalRoleID(dbRole._id) === builtinRoleId )[0] diff --git a/packages/server/src/middleware/authenticated.js b/packages/server/src/middleware/authenticated.js index e4e678abad..659baa8f6c 100644 --- a/packages/server/src/middleware/authenticated.js +++ b/packages/server/src/middleware/authenticated.js @@ -1,6 +1,6 @@ const jwt = require("jsonwebtoken") const STATUS_CODES = require("../utilities/statusCodes") -const { getRole, BUILTIN_ROLES } = require("../utilities/security/roles") +const { getRole, getBuiltinRoles } = require("../utilities/security/roles") const { AuthTypes } = require("../constants") const { getAppId, @@ -20,6 +20,7 @@ module.exports = async (ctx, next) => { // we hold it in state as a let appId = getAppId(ctx) const cookieAppId = ctx.cookies.get(getCookieName("currentapp")) + const builtinRoles = getBuiltinRoles() if (appId && cookieAppId !== appId) { setCookie(ctx, appId, "currentapp") } else if (cookieAppId) { @@ -40,7 +41,7 @@ module.exports = async (ctx, next) => { ctx.appId = appId ctx.user = { appId, - role: BUILTIN_ROLES.PUBLIC, + role: builtinRoles.PUBLIC, } await next() return diff --git a/packages/server/src/utilities/security/permissions.js b/packages/server/src/utilities/security/permissions.js index 342654f9ba..083de730b5 100644 --- a/packages/server/src/utilities/security/permissions.js +++ b/packages/server/src/utilities/security/permissions.js @@ -1,4 +1,5 @@ const { flatten } = require("lodash") +const { cloneDeep } = require("lodash/fp") const PermissionLevels = { READ: "read", @@ -70,7 +71,7 @@ exports.BUILTIN_PERMISSION_IDS = { POWER: "power", } -exports.BUILTIN_PERMISSIONS = { +const BUILTIN_PERMISSIONS = { PUBLIC: { _id: exports.BUILTIN_PERMISSION_IDS.PUBLIC, name: "Public", @@ -121,8 +122,12 @@ exports.BUILTIN_PERMISSIONS = { }, } +exports.getBuiltinPermissions = () => { + return cloneDeep(BUILTIN_PERMISSIONS) +} + exports.getBuiltinPermissionByID = id => { - const perms = Object.values(exports.BUILTIN_PERMISSIONS) + const perms = Object.values(BUILTIN_PERMISSIONS) return perms.find(perm => perm._id === id) } @@ -155,7 +160,7 @@ exports.doesHaveResourcePermission = ( } exports.doesHaveBasePermission = (permType, permLevel, permissionIds) => { - const builtins = Object.values(exports.BUILTIN_PERMISSIONS) + const builtins = Object.values(BUILTIN_PERMISSIONS) let permissions = flatten( builtins .filter(builtin => permissionIds.indexOf(builtin._id) !== -1) diff --git a/packages/server/src/utilities/security/roles.js b/packages/server/src/utilities/security/roles.js index 660f190d6f..38297f59b2 100644 --- a/packages/server/src/utilities/security/roles.js +++ b/packages/server/src/utilities/security/roles.js @@ -26,7 +26,7 @@ Role.prototype.addInheritance = function(inherits) { return this } -exports.BUILTIN_ROLES = { +const BUILTIN_ROLES = { ADMIN: new Role(BUILTIN_IDS.ADMIN, "Admin") .addPermission(BUILTIN_PERMISSION_IDS.ADMIN) .addInheritance(BUILTIN_IDS.POWER), @@ -44,11 +44,15 @@ exports.BUILTIN_ROLES = { ), } -exports.BUILTIN_ROLE_ID_ARRAY = Object.values(exports.BUILTIN_ROLES).map( +exports.getBuiltinRoles = () => { + return cloneDeep(BUILTIN_ROLES) +} + +exports.BUILTIN_ROLE_ID_ARRAY = Object.values(BUILTIN_ROLES).map( role => role._id ) -exports.BUILTIN_ROLE_NAME_ARRAY = Object.values(exports.BUILTIN_ROLES).map( +exports.BUILTIN_ROLE_NAME_ARRAY = Object.values(BUILTIN_ROLES).map( role => role.name ) @@ -60,17 +64,18 @@ function isBuiltin(role) { * Works through the inheritance ranks to see how far up the builtin stack this ID is. */ function builtinRoleToNumber(id) { + const builtins = exports.getBuiltinRoles() const MAX = Object.values(BUILTIN_IDS).length + 1 if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) { return MAX } - let role = exports.BUILTIN_ROLES[id], + let role = builtins, count = 0 do { if (!role) { break } - role = exports.BUILTIN_ROLES[role.inherits] + role = builtins[role.inherits] count++ } while (role !== null) return count @@ -107,7 +112,7 @@ exports.getRole = async (appId, roleId) => { // but can be extended by a doc stored about them (e.g. permissions) if (isBuiltin(roleId)) { role = cloneDeep( - Object.values(exports.BUILTIN_ROLES).find(role => role._id === roleId) + Object.values(BUILTIN_ROLES).find(role => role._id === roleId) ) } try { diff --git a/packages/server/src/utilities/security/utilities.js b/packages/server/src/utilities/security/utilities.js index 9d191b9572..f22b11dbd3 100644 --- a/packages/server/src/utilities/security/utilities.js +++ b/packages/server/src/utilities/security/utilities.js @@ -6,7 +6,7 @@ const { } = require("../../utilities/security/permissions") const { lowerBuiltinRoleID, - BUILTIN_ROLES, + getBuiltinRoles, } = require("../../utilities/security/roles") const { DocumentTypes } = require("../../db/utils") @@ -44,7 +44,7 @@ exports.getPermissionType = resourceId => { exports.getBasePermissions = resourceId => { const type = exports.getPermissionType(resourceId) const permissions = {} - for (let [roleId, role] of Object.entries(BUILTIN_ROLES)) { + for (let [roleId, role] of Object.entries(getBuiltinRoles())) { if (!role.permissionId) { continue } From 4b7459828764d00e72cc0ac0d40ebbc577bbee9c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Feb 2021 20:38:00 +0000 Subject: [PATCH 06/38] Linting. --- .../backend/DataTable/popovers/ManageAccessPopover.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/popovers/ManageAccessPopover.svelte b/packages/builder/src/components/backend/DataTable/popovers/ManageAccessPopover.svelte index e8f6f6df0b..ee8162c8d4 100644 --- a/packages/builder/src/components/backend/DataTable/popovers/ManageAccessPopover.svelte +++ b/packages/builder/src/components/backend/DataTable/popovers/ManageAccessPopover.svelte @@ -30,7 +30,9 @@
Who Can Access This Data?
- +
From 8bf10544c2a6cd2ed3099648d24fb1ce112d3ad4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Feb 2021 20:41:30 +0000 Subject: [PATCH 07/38] Fixing test case. --- packages/server/src/utilities/security/roles.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/security/roles.js b/packages/server/src/utilities/security/roles.js index 38297f59b2..79fd720078 100644 --- a/packages/server/src/utilities/security/roles.js +++ b/packages/server/src/utilities/security/roles.js @@ -69,7 +69,7 @@ function builtinRoleToNumber(id) { if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) { return MAX } - let role = builtins, + let role = builtins[id], count = 0 do { if (!role) { From 4b1855974c61e42aed5d93b56f3757d2c927df7a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Feb 2021 17:47:14 +0000 Subject: [PATCH 08/38] Work in progress, getting the server backend mostly ready for this work. --- .../modals/CreateTableModal.svelte | 30 +++--- .../builder/src/constants/backend/index.js | 55 +++++++++-- packages/server/src/api/controllers/row.js | 37 ++++---- packages/server/src/constants/index.js | 8 ++ .../src/db/linkedRows/LinkController.js | 9 +- packages/server/src/db/linkedRows/index.js | 4 +- packages/server/src/db/utils.js | 16 +++- packages/server/src/utilities/linkedRows.js | 0 packages/server/src/utilities/rowProcessor.js | 95 ++++++++++++++++--- 9 files changed, 194 insertions(+), 60 deletions(-) delete mode 100644 packages/server/src/utilities/linkedRows.js diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index b9a2307433..5ae060254b 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -14,8 +14,7 @@ import { NEW_ROW_TEMPLATE } from "builderStore/store/screenTemplates/newRowScreen" import { ROW_DETAIL_TEMPLATE } from "builderStore/store/screenTemplates/rowDetailScreen" import { ROW_LIST_TEMPLATE } from "builderStore/store/screenTemplates/rowListScreen" - import { FIELDS } from "constants/backend" - import { cloneDeep } from "lodash/fp" + import { AUTO_COLUMN_SUB_TYPES, buildAutoColumn } from "constants/backend" const defaultScreens = [ NEW_ROW_TEMPLATE, @@ -23,33 +22,34 @@ ROW_LIST_TEMPLATE, ] + $: tableNames = $backendUiStore.tables.map(table => table.name) + let modal let name let dataImport let error = "" let createAutoscreens = true let autoColumns = { - createdBy: true, - createdAt: true, - updatedBy: true, - updatedAt: true, - autoID: true, + [AUTO_COLUMN_SUB_TYPES.AUTO_ID]: {enabled: true, name: "Auto ID"}, + [AUTO_COLUMN_SUB_TYPES.CREATED_BY]: {enabled: true, name: "Created By"}, + [AUTO_COLUMN_SUB_TYPES.CREATED_AT]: {enabled: true, name: "Created At"}, + [AUTO_COLUMN_SUB_TYPES.UPDATED_BY]: {enabled: true, name: "Updated By"}, + [AUTO_COLUMN_SUB_TYPES.UPDATED_AT]: {enabled: true, name: "Updated At"}, } - function addAutoColumns(schema) { - for (let [property, enabled] of Object.entries(autoColumns)) { - if (!enabled) { + function addAutoColumns(tableName, schema) { + for (let [subtype, col] of Object.entries(autoColumns)) { + if (!col.enabled) { continue } - const autoColDef = cloneDeep(FIELDS.AUTO) - autoColDef.subtype = property - schema[property] = autoColDef + schema[col.name] = buildAutoColumn(tableName, col.name, subtype) } + return schema } function checkValid(evt) { const tableName = evt.target.value - if ($backendUiStore.models?.some(model => model.name === tableName)) { + if (tableNames.includes(tableName)) { error = `Table with name ${tableName} already exists. Please choose another name.` return } @@ -59,7 +59,7 @@ async function saveTable() { let newTable = { name, - schema: addAutoColumns(dataImport.schema || {}), + schema: addAutoColumns(name, dataImport.schema || {}), dataImport, } diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index a3076b18d5..62fe0cb5ab 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -80,13 +80,14 @@ export const FIELDS = { presence: false, }, }, - AUTO: { - name: "Auto Column", - icon: "ri-magic-line", - type: "auto", - // no constraints for auto-columns - // these are fully created serverside - } +} + +export const AUTO_COLUMN_SUB_TYPES = { + CREATED_BY: "createdBy", + CREATED_AT: "createdAt", + UPDATED_BY: "updatedBy", + UPDATED_AT: "updatedAt", + AUTO_ID: "autoID", } export const FILE_TYPES = { @@ -107,3 +108,43 @@ export const Roles = { PUBLIC: "PUBLIC", BUILDER: "BUILDER", } + +export const USER_TABLE_ID = "ta_users" + +export function isAutoColumnUserRelationship(subtype) { + return subtype === AUTO_COLUMN_SUB_TYPES.CREATED_BY || + subtype === AUTO_COLUMN_SUB_TYPES.UPDATED_BY +} + +export function buildAutoColumn(tableName, name, subtype) { + let type + switch (subtype) { + case AUTO_COLUMN_SUB_TYPES.UPDATED_BY: + case AUTO_COLUMN_SUB_TYPES.CREATED_BY: + type = FIELDS.LINK.type + break + case AUTO_COLUMN_SUB_TYPES.AUTO_ID: + type = FIELDS.NUMBER.type + break + default: + type = FIELDS.STRING.type + break + } + if (Object.values(AUTO_COLUMN_SUB_TYPES).indexOf(subtype) === -1) { + throw "Cannot build auto column with supplied subtype" + } + const base = { + name, + type, + subtype, + icon: "ri-magic-line", + autocolumn: true, + // no constraints, this should never have valid inputs + constraints: {}, + } + if (isAutoColumnUserRelationship(subtype)) { + base.tableId = USER_TABLE_ID + base.fieldName = `${tableName}-${name}` + } + return base +} diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 34f027002c..078e4b722f 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -58,18 +58,17 @@ async function findRow(db, appId, tableId, rowId) { exports.patch = async function(ctx) { const appId = ctx.user.appId const db = new CouchDB(appId) - let row = await db.get(ctx.params.rowId) - const table = await db.get(row.tableId) + let dbRow = await db.get(ctx.params.rowId) + let dbTable = await db.get(dbRow.tableId) const patchfields = ctx.request.body - // need to build up full patch fields before coerce for (let key of Object.keys(patchfields)) { - if (!table.schema[key]) continue - row[key] = patchfields[key] + if (!dbTable.schema[key]) continue + dbRow[key] = patchfields[key] } - row = inputProcessing(ctx.user, table, row) - + // this returns the table and row incase they have been updated + let { table, row } = await inputProcessing(ctx.user, dbTable, dbRow) const validateResult = await validate({ row, table, @@ -114,32 +113,34 @@ exports.patch = async function(ctx) { exports.save = async function(ctx) { const appId = ctx.user.appId const db = new CouchDB(appId) - let row = ctx.request.body - row.tableId = ctx.params.tableId + let inputs = ctx.request.body + inputs.tableId = ctx.params.tableId // TODO: find usage of this and break out into own endpoint - if (ctx.request.body.type === "delete") { + if (inputs.type === "delete") { await bulkDelete(ctx) - ctx.body = ctx.request.body.rows + ctx.body = inputs.rows return } // if the row obj had an _id then it will have been retrieved const existingRow = ctx.preExisting if (existingRow) { - ctx.params.rowId = row._id + ctx.params.rowId = inputs._id await exports.patch(ctx) return } - if (!row._rev && !row._id) { - row._id = generateRowID(row.tableId) + if (!inputs._rev && !inputs._id) { + inputs._id = generateRowID(inputs.tableId) } - const table = await db.get(row.tableId) - - row = inputProcessing(ctx.user, table, row) - + // this returns the table and row incase they have been updated + let { table, row } = await inputProcessing( + ctx.user, + await db.get(inputs.tableId), + inputs + ) const validateResult = await validate({ row, table, diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index d8038e2a9e..314e220a97 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -51,6 +51,14 @@ exports.FieldTypes = { AUTO: "auto", } +exports.AutoFieldSubTypes = { + CREATED_BY: "createdBy", + CREATED_AT: "createdAt", + UPDATED_BY: "updatedBy", + UPDATED_AT: "updatedAt", + AUTO_ID: "autoID", +} + exports.AuthTypes = AuthTypes exports.USERS_TABLE_SCHEMA = USERS_TABLE_SCHEMA exports.BUILDER_CONFIG_DB = "builder-config-db" diff --git a/packages/server/src/db/linkedRows/LinkController.js b/packages/server/src/db/linkedRows/LinkController.js index d69566d67c..54a0bfdd58 100644 --- a/packages/server/src/db/linkedRows/LinkController.js +++ b/packages/server/src/db/linkedRows/LinkController.js @@ -25,7 +25,14 @@ function LinkDocument( rowId2 ) { // build the ID out of unique references to this link document - this._id = generateLinkID(tableId1, tableId2, rowId1, rowId2) + this._id = generateLinkID( + tableId1, + tableId2, + rowId1, + rowId2, + fieldName1, + fieldName2 + ) // required for referencing in view this.type = FieldTypes.LINK this.doc1 = { diff --git a/packages/server/src/db/linkedRows/index.js b/packages/server/src/db/linkedRows/index.js index 02af98eef9..fb536c5c44 100644 --- a/packages/server/src/db/linkedRows/index.js +++ b/packages/server/src/db/linkedRows/index.js @@ -5,7 +5,7 @@ const { createLinkView, getUniqueByProp, } = require("./linkUtils") -const _ = require("lodash") +const { flatten } = require("lodash") /** * This functionality makes sure that when rows with links are created, updated or deleted they are processed @@ -101,7 +101,7 @@ exports.attachLinkInfo = async (appId, rows) => { } let tableIds = [...new Set(rows.map(el => el.tableId))] // start by getting all the link values for performance reasons - let responses = _.flatten( + let responses = flatten( await Promise.all( tableIds.map(tableId => getLinkDocuments({ diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index 5e7e74d711..6ca55b6336 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -138,10 +138,22 @@ exports.generateAutomationID = () => { * @param {string} tableId2 The ID of the linked table. * @param {string} rowId1 The ID of the linker row. * @param {string} rowId2 The ID of the linked row. + * @param {string} fieldName1 The name of the field in the linker row. + * @param {string} fieldName2 the name of the field in the linked row. * @returns {string} The new link doc ID which the automation doc can be stored under. */ -exports.generateLinkID = (tableId1, tableId2, rowId1, rowId2) => { - return `${DocumentTypes.LINK}${SEPARATOR}${tableId1}${SEPARATOR}${tableId2}${SEPARATOR}${rowId1}${SEPARATOR}${rowId2}` +exports.generateLinkID = ( + tableId1, + tableId2, + rowId1, + rowId2, + fieldName1, + fieldName2 +) => { + const tables = `${SEPARATOR}${tableId1}${SEPARATOR}${tableId2}` + const rows = `${SEPARATOR}${rowId1}${SEPARATOR}${rowId2}` + const fields = `${SEPARATOR}${fieldName1}${SEPARATOR}${fieldName2}` + return `${DocumentTypes.LINK}${tables}${rows}${fields}` } /** diff --git a/packages/server/src/utilities/linkedRows.js b/packages/server/src/utilities/linkedRows.js deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/server/src/utilities/rowProcessor.js b/packages/server/src/utilities/rowProcessor.js index fe18eefdc6..c2885d8440 100644 --- a/packages/server/src/utilities/rowProcessor.js +++ b/packages/server/src/utilities/rowProcessor.js @@ -2,13 +2,18 @@ const env = require("../environment") const { OBJ_STORE_DIRECTORY } = require("../constants") const linkRows = require("../db/linkedRows") const { cloneDeep } = require("lodash/fp") -const { FieldTypes } = require("../constants") +const { FieldTypes, AutoFieldSubTypes } = require("../constants") +const CouchDB = require("../db") +const { ViewNames } = require("../db/utils") + +const BASE_AUTO_ID = 1 +const USER_TABLE_ID = ViewNames.USERS /** * A map of how we convert various properties in rows to each other based on the row type. */ const TYPE_TRANSFORM_MAP = { - link: { + [FieldTypes.LINK]: { "": [], [null]: [], [undefined]: undefined, @@ -19,44 +24,102 @@ const TYPE_TRANSFORM_MAP = { return link }, }, - options: { + [FieldTypes.OPTIONS]: { "": "", [null]: "", [undefined]: undefined, }, - string: { + [FieldTypes.STRING]: { "": "", [null]: "", [undefined]: undefined, }, - longform: { + [FieldTypes.LONGFORM]: { "": "", [null]: "", [undefined]: undefined, }, - number: { + [FieldTypes.NUMBER]: { "": null, [null]: null, [undefined]: undefined, parse: n => parseFloat(n), }, - datetime: { + [FieldTypes.DATETIME]: { "": null, [undefined]: undefined, [null]: null, }, - attachment: { + [FieldTypes.ATTACHMENT]: { "": [], [null]: [], [undefined]: undefined, }, - boolean: { + [FieldTypes.BOOLEAN]: { "": null, [null]: null, [undefined]: undefined, true: true, false: false, }, + [FieldTypes.AUTO]: { + parse: () => undefined, + }, +} + +function getAutoRelationshipName(table, columnName) { + return `${table.name}-${columnName}` +} + +/** + * This will update any auto columns that are found on the row/table with the correct information based on + * time now and the current logged in user making the request. + * @param {Object} user The user to be used for an appId as well as the createdBy and createdAt fields. + * @param {Object} table The table which is to be used for the schema, as well as handling auto IDs incrementing. + * @param {Object} row The row which is to be updated with information for the auto columns. + * @returns {Promise<{row: Object, table: Object}>} The updated row and table, the table may need to be updated + * for automatic ID purposes. + */ +async function processAutoColumn(user, table, row) { + let now = new Date().toISOString() + // if a row doesn't have a revision then it doesn't exist yet + const creating = !row._rev + let tableUpdated = false + for (let [key, schema] of Object.entries(table.schema)) { + if (!schema.autocolumn) { + continue + } + switch (schema.subtype) { + case AutoFieldSubTypes.CREATED_BY: + if (creating) { + row[key] = [user.userId] + } + break + case AutoFieldSubTypes.CREATED_AT: + if (creating) { + row[key] = now + } + break + case AutoFieldSubTypes.UPDATED_BY: + row[key] = [user.userId] + break + case AutoFieldSubTypes.UPDATED_AT: + row[key] = now + break + case AutoFieldSubTypes.AUTO_ID: + schema.lastID = !schema.lastID ? BASE_AUTO_ID : schema.lastID + 1 + row[key] = schema.lastID + tableUpdated = true + break + } + } + if (tableUpdated) { + const db = new CouchDB(user.appId) + const response = await db.put(table) + // update the revision + table._rev = response._rev + } + return { table, row } } /** @@ -65,7 +128,7 @@ const TYPE_TRANSFORM_MAP = { * @param {object} type The type fo coerce to * @returns {object} The coerced value */ -exports.coerceValue = (row, type) => { +exports.coerce = (row, type) => { // eslint-disable-next-line no-prototype-builtins if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) { return TYPE_TRANSFORM_MAP[type][row] @@ -84,15 +147,17 @@ exports.coerceValue = (row, type) => { * @param {object} table the table which the row is being saved to. * @returns {object} the row which has been prepared to be written to the DB. */ -exports.inputProcessing = (user, table, row) => { - const clonedRow = cloneDeep(row) +exports.inputProcessing = async (user, table, row) => { + let clonedRow = cloneDeep(row) for (let [key, value] of Object.entries(clonedRow)) { const field = table.schema[key] - if (!field) continue - + if (!field) { + continue + } clonedRow[key] = exports.coerce(value, field.type) } - return clonedRow + // handle auto columns - this returns an object like {table, row} + return processAutoColumn(user, table, clonedRow) } /** From 1a2b9bbef4d44f6358de9b3c3ddcae36efdc717b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Feb 2021 18:41:26 +0000 Subject: [PATCH 09/38] Removing unused stuff. --- packages/server/src/utilities/rowProcessor.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor.js b/packages/server/src/utilities/rowProcessor.js index c2885d8440..ceb31daa3b 100644 --- a/packages/server/src/utilities/rowProcessor.js +++ b/packages/server/src/utilities/rowProcessor.js @@ -4,10 +4,8 @@ const linkRows = require("../db/linkedRows") const { cloneDeep } = require("lodash/fp") const { FieldTypes, AutoFieldSubTypes } = require("../constants") const CouchDB = require("../db") -const { ViewNames } = require("../db/utils") const BASE_AUTO_ID = 1 -const USER_TABLE_ID = ViewNames.USERS /** * A map of how we convert various properties in rows to each other based on the row type. @@ -67,10 +65,6 @@ const TYPE_TRANSFORM_MAP = { }, } -function getAutoRelationshipName(table, columnName) { - return `${table.name}-${columnName}` -} - /** * This will update any auto columns that are found on the row/table with the correct information based on * time now and the current logged in user making the request. From 245cd0a791ece2ef31865994ca82f393b55892f2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Feb 2021 18:53:20 +0000 Subject: [PATCH 10/38] Fixing issue with relationships. --- packages/server/src/db/linkedRows/linkUtils.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/db/linkedRows/linkUtils.js b/packages/server/src/db/linkedRows/linkUtils.js index ee22a87410..c07c17eda7 100644 --- a/packages/server/src/db/linkedRows/linkUtils.js +++ b/packages/server/src/db/linkedRows/linkUtils.js @@ -24,7 +24,8 @@ exports.createLinkView = async appId => { const designDoc = await db.get("_design/database") const view = { map: function(doc) { - if (doc.type === FieldTypes.LINK) { + // everything in this must remain constant as its going to Pouch, no external variables + if (doc.type === "link") { let doc1 = doc.doc1 let doc2 = doc.doc2 emit([doc1.tableId, doc1.rowId], { From 23cac6a9acc3dabff0b8619cc3cc919847c8a255 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Feb 2021 19:01:15 +0000 Subject: [PATCH 11/38] Fixing issue with linked rows not handling uniqueness correctly when links between tables are using fieldnames for uniqueness. --- packages/server/src/db/linkedRows/index.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/src/db/linkedRows/index.js b/packages/server/src/db/linkedRows/index.js index fb536c5c44..5ea758a976 100644 --- a/packages/server/src/db/linkedRows/index.js +++ b/packages/server/src/db/linkedRows/index.js @@ -118,8 +118,12 @@ exports.attachLinkInfo = async (appId, rows) => { // have to get unique as the previous table query can // return duplicates, could be querying for both tables in a relation const linkVals = getUniqueByProp( - responses.filter(el => el.thisId === row._id), - "id" + responses + // find anything that matches the row's ID we are searching for + .filter(el => el.thisId === row._id) + // create a unique ID which we can use for getting only unique ones + .map(el => ({ ...el, unique: el.id + el.fieldName })), + "unique" ) for (let linkVal of linkVals) { // work out which link pertains to this row From 846772bfebb33f2b76340cc2b93ad38c65d9317d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Feb 2021 19:59:30 +0000 Subject: [PATCH 12/38] Adding the ability to create/control auto-columns from the create/edit column modal. --- .../screenTemplates/utils/commonComponents.js | 4 ++ .../DataTable/modals/CreateEditColumn.svelte | 38 +++++++++++++++---- .../DataTable/modals/CreateEditRow.svelte | 8 ++-- .../modals/CreateTableModal.svelte | 10 +---- .../components/integration/QueryViewer.svelte | 1 - .../builder/src/constants/backend/index.js | 28 +++++++++++--- 6 files changed, 65 insertions(+), 24 deletions(-) diff --git a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js index 4c127fbe0b..ee7a4da0fd 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js +++ b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js @@ -179,6 +179,10 @@ export function makeDatasourceFormComponents(datasource) { let fields = Object.keys(schema || {}) fields.forEach(field => { const fieldSchema = schema[field] + // skip autocolumns + if (fieldSchema.autocolumn) { + return + } const fieldType = typeof fieldSchema === "object" ? fieldSchema.type : fieldSchema const componentType = fieldTypeToComponentMap[fieldType] diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index bc18b7559c..0045d2261c 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -10,12 +10,13 @@ import { cloneDeep } from "lodash/fp" import { backendUiStore } from "builderStore" import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" - import { FIELDS } from "constants/backend" + import { FIELDS, getAutoColumnInformation, buildAutoColumn, AUTO_COLUMN_SUB_TYPES } from "constants/backend" import { notifier } from "builderStore/store/notifications" import ValuesList from "components/common/ValuesList.svelte" import DatePicker from "components/common/DatePicker.svelte" import ConfirmDialog from "components/common/ConfirmDialog.svelte" + const AUTO_COL = "auto" let fieldDefinitions = cloneDeep(FIELDS) export let onClosed @@ -43,7 +44,17 @@ $backendUiStore.selectedTable?._id === TableNames.USERS && UNEDITABLE_USER_FIELDS.includes(field.name) + // used to select what different options can be displayed for column type + $: canBeSearched = field.type !== 'link' && + field.subtype !== AUTO_COLUMN_SUB_TYPES.CREATED_BY && + field.subtype !== AUTO_COLUMN_SUB_TYPES.UPDATED_BY + $: canBeDisplay = field.type !== 'link' && field.type !== AUTO_COL + $: canBeRequired = field.type !== 'link' && !uneditable && field.type !== AUTO_COL + async function saveColumn() { + if (field.type === AUTO_COL) { + field = buildAutoColumn($backendUiStore.draftTable.name, field.name, field.subtype) + } backendUiStore.update(state => { backendUiStore.actions.tables.saveField({ originalName, @@ -67,11 +78,14 @@ } function handleFieldConstraints(event) { - const { type, constraints } = fieldDefinitions[ + const definition = fieldDefinitions[ event.target.value.toUpperCase() ] - field.type = type - field.constraints = constraints + if (!definition) { + return + } + field.type = definition.type + field.constraints = definition.constraints } function onChangeRequired(e) { @@ -124,9 +138,10 @@ {#each Object.values(fieldDefinitions) as field} {/each} + - {#if field.type !== 'link' && !uneditable} + {#if canBeRequired} {/if} - {#if field.type !== 'link'} + {#if canBeDisplay} + {/if} + {#if canBeSearched} + {:else if field.type === AUTO_COL} + {/if}