From 7dc9abf54b324fc8e07ff3cdbc17625569d99d5b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 10 Aug 2022 16:19:08 +0100 Subject: [PATCH] Validating datasources fully, initial work towards validating components and including the build in the CLI. --- packages/cli/package.json | 6 +- packages/cli/src/exec.js | 25 ++++++ packages/cli/src/plugins/constants.js | 6 ++ packages/cli/src/plugins/index.js | 54 ++++++++++-- packages/cli/src/plugins/validate.js | 87 +++++++++++++++++++ packages/cli/yarn.lock | 45 ++++++++++ packages/server/src/definitions/datasource.ts | 84 ++++-------------- .../types/src/documents/app/datasource.ts | 69 +++++++++++++++ 8 files changed, 297 insertions(+), 79 deletions(-) create mode 100644 packages/cli/src/exec.js create mode 100644 packages/cli/src/plugins/constants.js create mode 100644 packages/cli/src/plugins/validate.js diff --git a/packages/cli/package.json b/packages/cli/package.json index 5a048fb17d..413703af9b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -26,8 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "1.1.32-alpha.6", - "@budibase/string-templates": "^1.2.28", + "@budibase/backend-core": "1.2.28-alpha.0", + "@budibase/string-templates": "1.2.28-alpha.0", + "@budibase/types": "1.2.28-alpha.0", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", @@ -36,6 +37,7 @@ "dotenv": "16.0.1", "download": "^8.0.0", "inquirer": "8.0.0", + "joi": "^17.6.0", "lookpath": "1.1.0", "node-fetch": "2", "pkg": "5.7.0", diff --git a/packages/cli/src/exec.js b/packages/cli/src/exec.js new file mode 100644 index 0000000000..07520336c4 --- /dev/null +++ b/packages/cli/src/exec.js @@ -0,0 +1,25 @@ +const util = require("util") +const exec = util.promisify(require("child_process").exec) + +exports.exec = async command => { + const { stdout } = await exec(command) + return stdout +} + +exports.utilityInstalled = async utilName => { + try { + await exports.exec(`${utilName} --version`) + return true + } catch (err) { + return false + } +} + +exports.runPkgCommand = async command => { + const yarn = await exports.utilityInstalled("yarn") + const npm = await exports.utilityInstalled("npm") + if (!yarn && !npm) { + throw new Error("Must have yarn or npm installed to run build.") + } + await exports.exec(yarn ? `yarn ${command}` : `npm run ${command}`) +} diff --git a/packages/cli/src/plugins/constants.js b/packages/cli/src/plugins/constants.js new file mode 100644 index 0000000000..37d0748979 --- /dev/null +++ b/packages/cli/src/plugins/constants.js @@ -0,0 +1,6 @@ +exports.PluginTypes = { + COMPONENT: "component", + DATASOURCE: "datasource", +} + +exports.PLUGIN_TYPES_ARR = Object.values(exports.PluginTypes) diff --git a/packages/cli/src/plugins/index.js b/packages/cli/src/plugins/index.js index c0d4fa2cfe..6ec99ac7fa 100644 --- a/packages/cli/src/plugins/index.js +++ b/packages/cli/src/plugins/index.js @@ -3,21 +3,28 @@ const { CommandWords } = require("../constants") const { getSkeleton, fleshOutSkeleton } = require("./skeleton") const questions = require("../questions") const fs = require("fs") - -const PLUGIN_TYPES = ["component", "datasource"] +const { PLUGIN_TYPES_ARR } = require("./constants") +const { validate } = require("./validate") +const { runPkgCommand } = require("../exec") +const { join } = require("path") +const { success, error, info } = require("../utils") async function init(opts) { const type = opts["init"] || opts - if (!type || !PLUGIN_TYPES.includes(type)) { - console.error( - "Please provide a type to init, either 'component' or 'datasource'." + if (!type || !PLUGIN_TYPES_ARR.includes(type)) { + console.log( + error( + "Please provide a type to init, either 'component' or 'datasource'." + ) ) return } - console.log("Lets get some details about your new plugin:") + console.log(info("Lets get some details about your new plugin:")) const name = await questions.string("Name", `budibase-${type}`) if (fs.existsSync(name)) { - console.error("Directory by plugin name already exists, pick a new name.") + console.log( + error("Directory by plugin name already exists, pick a new name.") + ) return } const desc = await questions.string( @@ -28,10 +35,39 @@ async function init(opts) { // get the skeleton await getSkeleton(type, name) await fleshOutSkeleton(name, desc, version) - console.log(`Plugin created in directory "${name}"`) + console.log(info(`Plugin created in directory "${name}"`)) } -async function build() {} +async function build() { + console.log(info("Verifying plugin...")) + const schema = fs.readFileSync("schema.json", "utf8") + const pkg = fs.readFileSync("package.json", "utf8") + let name, version + try { + const schemaJson = JSON.parse(schema) + const pkgJson = JSON.parse(pkg) + if (!pkgJson.name || !pkgJson.version || !pkgJson.description) { + throw new Error( + "package.json is missing one of 'name', 'version' or 'description'." + ) + } + name = pkgJson.name + version = pkgJson.version + validate(schemaJson) + } catch (err) { + if (err && err.message && err.message.includes("not valid JSON")) { + console.log(error(`schema.json is not valid JSON: ${err.message}`)) + } else { + console.log(error(`Invalid schema/package.json: ${err.message}`)) + } + return + } + console.log(success("Verified!")) + console.log(info("Building plugin...")) + await runPkgCommand("build") + const output = join("dist", `${name}-${version}.tar.gz`) + console.log(success(`Build complete - output in: ${output}`)) +} const command = new Command(`${CommandWords.PLUGIN}`) .addHelp( diff --git a/packages/cli/src/plugins/validate.js b/packages/cli/src/plugins/validate.js new file mode 100644 index 0000000000..b460670c7f --- /dev/null +++ b/packages/cli/src/plugins/validate.js @@ -0,0 +1,87 @@ +const { PluginTypes } = require("./constants") +const { DatasourceFieldTypes, QueryTypes } = require("@budibase/types") +const joi = require("joi") + +const DATASOURCE_TYPES = [ + "Relational", + "Non-relational", + "Spreadsheet", + "Object store", + "Graph", + "API", +] + +function runJoi(validator, schema) { + const { error } = validator.validate(schema) + if (error) { + throw error + } +} + +function validateComponent(schema) { + const validator = joi.object({ + type: joi.string().allow("component").required(), + metadata: joi.object().unknown(true).required(), + schema: joi + .object({ + name: joi.string().required(), + settings: joi.array().items(joi.object().unknown(true)).required(), + }) + .unknown(true), + }) + runJoi(validator, schema) +} + +function validateDatasource(schema) { + const fieldValidator = joi.object({ + type: joi + .string() + .allow(...Object.values(DatasourceFieldTypes)) + .required(), + required: joi.boolean().required(), + default: joi.any(), + display: joi.string(), + }) + + const queryValidator = joi + .object({ + type: joi.string().allow(...Object.values(QueryTypes)), + fields: joi.object().pattern(joi.string(), fieldValidator), + }) + .required() + + const validator = joi.object({ + type: joi.string().allow("datasource").required(), + metadata: joi.object().unknown(true).required(), + schema: joi.object({ + docs: joi.string(), + friendlyName: joi.string().required(), + type: joi.string().allow(...DATASOURCE_TYPES), + description: joi.string().required(), + datasource: joi.object().pattern(joi.string(), fieldValidator).required(), + query: joi + .object({ + create: queryValidator, + read: queryValidator, + update: queryValidator, + delete: queryValidator, + }) + .unknown(true) + .required(), + }), + }) + runJoi(validator, schema) +} + +exports.validate = schema => { + switch (schema.type) { + case PluginTypes.COMPONENT: + validateComponent(schema) + break + case PluginTypes.DATASOURCE: + validateDatasource(schema) + break + default: + throw new Error(`Unknown plugin type - check schema.json: ${schema.type}`) + } +} diff --git a/packages/cli/yarn.lock b/packages/cli/yarn.lock index 95813f60cf..c17b48e8bb 100644 --- a/packages/cli/yarn.lock +++ b/packages/cli/yarn.lock @@ -118,6 +118,11 @@ resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.32-alpha.6.tgz#95d8d73c7ed6ebc22ff26a44365127a478e19409" integrity sha512-AKKxrzVqGtcSzZZ2fP6i2Vgv6ICN9NEEE1dmzRk9AImZS+XKQ9VgVpdE+4gHgFK7L0gBYAsiaoEpCbbrI/+NoQ== +"@budibase/types@^1.2.31": + version "1.2.31" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.2.31.tgz#4715bca331ecd5eac23f95bfdee2eb147ef57814" + integrity sha512-/R03MleZRMtf6JW/nCKBqd/bBIkbFnwr8EV1Y3t6EySh8fnhM2PdhlWlpf/BrE0zMoiuBn4JMFl2vJ2Mzo/aoA== + "@eslint/eslintrc@^0.4.3": version "0.4.3" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.3.tgz#9e42981ef035beb3dd49add17acb96e8ff6f394c" @@ -133,6 +138,18 @@ minimatch "^3.0.4" strip-json-comments "^3.1.1" +"@hapi/hoek@^9.0.0": + version "9.3.0" + resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" + integrity sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ== + +"@hapi/topo@^5.0.0": + version "5.1.0" + resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.1.0.tgz#dc448e332c6c6e37a4dc02fd84ba8d44b9afb012" + integrity sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg== + dependencies: + "@hapi/hoek" "^9.0.0" + "@humanwhocodes/config-array@^0.5.0": version "0.5.0" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.5.0.tgz#1407967d4c6eecd7388f83acf1eaf4d0c6e58ef9" @@ -183,6 +200,23 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" +"@sideway/address@^4.1.3": + version "4.1.4" + resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0" + integrity sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw== + dependencies: + "@hapi/hoek" "^9.0.0" + +"@sideway/formula@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.0.tgz#fe158aee32e6bd5de85044be615bc08478a0a13c" + integrity sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg== + +"@sideway/pinpoint@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df" + integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== + "@sindresorhus/is@^0.7.0": version "0.7.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd" @@ -2455,6 +2489,17 @@ jmespath@0.15.0: resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" integrity sha512-+kHj8HXArPfpPEKGLZ+kB5ONRTCiGQXo8RQYL0hH8t6pWXUBBK5KkkQmTNOwKK4LEsd0yTsgtjJVm4UBSZea4w== +joi@^17.6.0: + version "17.6.0" + resolved "https://registry.yarnpkg.com/joi/-/joi-17.6.0.tgz#0bb54f2f006c09a96e75ce687957bd04290054b2" + integrity sha512-OX5dG6DTbcr/kbMFj0KGYxuew69HPcAE3K/sZpEV2nP6e/j/C0HV+HNiBPCASxdx5T7DMoa0s8UeHWMnb6n2zw== + dependencies: + "@hapi/hoek" "^9.0.0" + "@hapi/topo" "^5.0.0" + "@sideway/address" "^4.1.3" + "@sideway/formula" "^3.0.0" + "@sideway/pinpoint" "^2.0.0" + join-component@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5" diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 9752fc947a..604dae50c3 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -1,73 +1,21 @@ import { Row, Table, Base } from "./common" +import { + Operation, + QueryTypes, + SortDirection, + SourceNames, +} from "@budibase/types" -export enum Operation { - CREATE = "CREATE", - READ = "READ", - UPDATE = "UPDATE", - DELETE = "DELETE", - BULK_CREATE = "BULK_CREATE", - CREATE_TABLE = "CREATE_TABLE", - UPDATE_TABLE = "UPDATE_TABLE", - DELETE_TABLE = "DELETE_TABLE", -} - -export enum SortDirection { - ASCENDING = "ASCENDING", - DESCENDING = "DESCENDING", -} - -export enum QueryTypes { - SQL = "sql", - JSON = "json", - FIELDS = "fields", -} - -export enum DatasourceFieldTypes { - STRING = "string", - LONGFORM = "longForm", - BOOLEAN = "boolean", - NUMBER = "number", - PASSWORD = "password", - LIST = "list", - OBJECT = "object", - JSON = "json", - FILE = "file", -} - -export enum SourceNames { - POSTGRES = "POSTGRES", - DYNAMODB = "DYNAMODB", - MONGODB = "MONGODB", - ELASTICSEARCH = "ELASTICSEARCH", - COUCHDB = "COUCHDB", - SQL_SERVER = "SQL_SERVER", - S3 = "S3", - AIRTABLE = "AIRTABLE", - MYSQL = "MYSQL", - ARANGODB = "ARANGODB", - REST = "REST", - ORACLE = "ORACLE", - GOOGLE_SHEETS = "GOOGLE_SHEETS", - FIRESTORE = "FIRESTORE", - REDIS = "REDIS", - SNOWFLAKE = "SNOWFLAKE", -} - -export enum IncludeRelationships { - INCLUDE = 1, - EXCLUDE = 0, -} - -export enum FilterTypes { - STRING = "string", - FUZZY = "fuzzy", - RANGE = "range", - EQUAL = "equal", - NOT_EQUAL = "notEqual", - EMPTY = "empty", - NOT_EMPTY = "notEmpty", - ONE_OF = "oneOf", -} +// these were previously exported here - moved to types for re-use +export { + Operation, + SortDirection, + QueryTypes, + DatasourceFieldTypes, + SourceNames, + IncludeRelationships, + FilterTypes, +} from "@budibase/types" export interface QueryDefinition { type: QueryTypes diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index 3a8704a0a9..6a5dd054da 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -1,5 +1,74 @@ import { Document } from "../document" +export enum Operation { + CREATE = "CREATE", + READ = "READ", + UPDATE = "UPDATE", + DELETE = "DELETE", + BULK_CREATE = "BULK_CREATE", + CREATE_TABLE = "CREATE_TABLE", + UPDATE_TABLE = "UPDATE_TABLE", + DELETE_TABLE = "DELETE_TABLE", +} + +export enum SortDirection { + ASCENDING = "ASCENDING", + DESCENDING = "DESCENDING", +} + +export enum QueryTypes { + SQL = "sql", + JSON = "json", + FIELDS = "fields", +} + +export enum DatasourceFieldTypes { + STRING = "string", + LONGFORM = "longForm", + BOOLEAN = "boolean", + NUMBER = "number", + PASSWORD = "password", + LIST = "list", + OBJECT = "object", + JSON = "json", + FILE = "file", +} + +export enum SourceNames { + POSTGRES = "POSTGRES", + DYNAMODB = "DYNAMODB", + MONGODB = "MONGODB", + ELASTICSEARCH = "ELASTICSEARCH", + COUCHDB = "COUCHDB", + SQL_SERVER = "SQL_SERVER", + S3 = "S3", + AIRTABLE = "AIRTABLE", + MYSQL = "MYSQL", + ARANGODB = "ARANGODB", + REST = "REST", + ORACLE = "ORACLE", + GOOGLE_SHEETS = "GOOGLE_SHEETS", + FIRESTORE = "FIRESTORE", + REDIS = "REDIS", + SNOWFLAKE = "SNOWFLAKE", +} + +export enum IncludeRelationships { + INCLUDE = 1, + EXCLUDE = 0, +} + +export enum FilterTypes { + STRING = "string", + FUZZY = "fuzzy", + RANGE = "range", + EQUAL = "equal", + NOT_EQUAL = "notEqual", + EMPTY = "empty", + NOT_EMPTY = "notEmpty", + ONE_OF = "oneOf", +} + export interface Datasource extends Document { source: string }