Validating datasources fully, initial work towards validating components and including the build in the CLI.

packages/cli/package.json

@@ -26,8 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "1.1.32-alpha.6",
-    "@budibase/string-templates": "^1.2.28",
+    "@budibase/backend-core": "1.2.28-alpha.0",
+    "@budibase/string-templates": "1.2.28-alpha.0",
+    "@budibase/types": "1.2.28-alpha.0",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",
@@ -36,6 +37,7 @@
     "dotenv": "16.0.1",
     "download": "^8.0.0",
     "inquirer": "8.0.0",
+    "joi": "^17.6.0",
     "lookpath": "1.1.0",
     "node-fetch": "2",
     "pkg": "5.7.0",

packages/cli/src/exec.js

@@ -0,0 +1,25 @@
+const util = require("util")
+const exec = util.promisify(require("child_process").exec)
+
+exports.exec = async command => {
+  const { stdout } = await exec(command)
+  return stdout
+}
+
+exports.utilityInstalled = async utilName => {
+  try {
+    await exports.exec(`${utilName} --version`)
+    return true
+  } catch (err) {
+    return false
+  }
+}
+
+exports.runPkgCommand = async command => {
+  const yarn = await exports.utilityInstalled("yarn")
+  const npm = await exports.utilityInstalled("npm")
+  if (!yarn && !npm) {
+    throw new Error("Must have yarn or npm installed to run build.")
+  }
+  await exports.exec(yarn ? `yarn ${command}` : `npm run ${command}`)
+}

packages/cli/src/plugins/constants.js

@@ -0,0 +1,6 @@
+exports.PluginTypes = {
+  COMPONENT: "component",
+  DATASOURCE: "datasource",
+}
+
+exports.PLUGIN_TYPES_ARR = Object.values(exports.PluginTypes)

packages/cli/src/plugins/index.js

@@ -3,21 +3,28 @@ const { CommandWords } = require("../constants")
 const { getSkeleton, fleshOutSkeleton } = require("./skeleton")
 const questions = require("../questions")
 const fs = require("fs")
-
-const PLUGIN_TYPES = ["component", "datasource"]
+const { PLUGIN_TYPES_ARR } = require("./constants")
+const { validate } = require("./validate")
+const { runPkgCommand } = require("../exec")
+const { join } = require("path")
+const { success, error, info } = require("../utils")
 
 async function init(opts) {
   const type = opts["init"] || opts
-  if (!type || !PLUGIN_TYPES.includes(type)) {
-    console.error(
-      "Please provide a type to init, either 'component' or 'datasource'."
+  if (!type || !PLUGIN_TYPES_ARR.includes(type)) {
+    console.log(
+      error(
+        "Please provide a type to init, either 'component' or 'datasource'."
+      )
     )
     return
   }
-  console.log("Lets get some details about your new plugin:")
+  console.log(info("Lets get some details about your new plugin:"))
   const name = await questions.string("Name", `budibase-${type}`)
   if (fs.existsSync(name)) {
-    console.error("Directory by plugin name already exists, pick a new name.")
+    console.log(
+      error("Directory by plugin name already exists, pick a new name.")
+    )
     return
   }
   const desc = await questions.string(
@@ -28,10 +35,39 @@ async function init(opts) {
   // get the skeleton
   await getSkeleton(type, name)
   await fleshOutSkeleton(name, desc, version)
-  console.log(`Plugin created in directory "${name}"`)
+  console.log(info(`Plugin created in directory "${name}"`))
 }
 
-async function build() {}
+async function build() {
+  console.log(info("Verifying plugin..."))
+  const schema = fs.readFileSync("schema.json", "utf8")
+  const pkg = fs.readFileSync("package.json", "utf8")
+  let name, version
+  try {
+    const schemaJson = JSON.parse(schema)
+    const pkgJson = JSON.parse(pkg)
+    if (!pkgJson.name || !pkgJson.version || !pkgJson.description) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
+    name = pkgJson.name
+    version = pkgJson.version
+    validate(schemaJson)
+  } catch (err) {
+    if (err && err.message && err.message.includes("not valid JSON")) {
+      console.log(error(`schema.json is not valid JSON: ${err.message}`))
+    } else {
+      console.log(error(`Invalid schema/package.json: ${err.message}`))
+    }
+    return
+  }
+  console.log(success("Verified!"))
+  console.log(info("Building plugin..."))
+  await runPkgCommand("build")
+  const output = join("dist", `${name}-${version}.tar.gz`)
+  console.log(success(`Build complete - output in: ${output}`))
+}
 
 const command = new Command(`${CommandWords.PLUGIN}`)
   .addHelp(

packages/cli/src/plugins/validate.js

@@ -0,0 +1,87 @@
+const { PluginTypes } = require("./constants")
+const { DatasourceFieldTypes, QueryTypes } = require("@budibase/types")
+const joi = require("joi")
+
+const DATASOURCE_TYPES = [
+  "Relational",
+  "Non-relational",
+  "Spreadsheet",
+  "Object store",
+  "Graph",
+  "API",
+]
+
+function runJoi(validator, schema) {
+  const { error } = validator.validate(schema)
+  if (error) {
+    throw error
+  }
+}
+
+function validateComponent(schema) {
+  const validator = joi.object({
+    type: joi.string().allow("component").required(),
+    metadata: joi.object().unknown(true).required(),
+    schema: joi
+      .object({
+        name: joi.string().required(),
+        settings: joi.array().items(joi.object().unknown(true)).required(),
+      })
+      .unknown(true),
+  })
+  runJoi(validator, schema)
+}
+
+function validateDatasource(schema) {
+  const fieldValidator = joi.object({
+    type: joi
+      .string()
+      .allow(...Object.values(DatasourceFieldTypes))
+      .required(),
+    required: joi.boolean().required(),
+    default: joi.any(),
+    display: joi.string(),
+  })
+
+  const queryValidator = joi
+    .object({
+      type: joi.string().allow(...Object.values(QueryTypes)),
+      fields: joi.object().pattern(joi.string(), fieldValidator),
+    })
+    .required()
+
+  const validator = joi.object({
+    type: joi.string().allow("datasource").required(),
+    metadata: joi.object().unknown(true).required(),
+    schema: joi.object({
+      docs: joi.string(),
+      friendlyName: joi.string().required(),
+      type: joi.string().allow(...DATASOURCE_TYPES),
+      description: joi.string().required(),
+      datasource: joi.object().pattern(joi.string(), fieldValidator).required(),
+      query: joi
+        .object({
+          create: queryValidator,
+          read: queryValidator,
+          update: queryValidator,
+          delete: queryValidator,
+        })
+        .unknown(true)
+        .required(),
+    }),
+  })
+  runJoi(validator, schema)
+}
+
+exports.validate = schema => {
+  switch (schema.type) {
+    case PluginTypes.COMPONENT:
+      validateComponent(schema)
+      break
+    case PluginTypes.DATASOURCE:
+      validateDatasource(schema)
+      break
+    default:
+      throw new Error(`Unknown plugin type - check schema.json: ${schema.type}`)
+  }
+}

packages/cli/yarn.lock

@@ -118,6 +118,11 @@
   resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.32-alpha.6.tgz#95d8d73c7ed6ebc22ff26a44365127a478e19409"
   integrity sha512-AKKxrzVqGtcSzZZ2fP6i2Vgv6ICN9NEEE1dmzRk9AImZS+XKQ9VgVpdE+4gHgFK7L0gBYAsiaoEpCbbrI/+NoQ==
 
+"@budibase/types@^1.2.31":
+  version "1.2.31"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.2.31.tgz#4715bca331ecd5eac23f95bfdee2eb147ef57814"
+  integrity sha512-/R03MleZRMtf6JW/nCKBqd/bBIkbFnwr8EV1Y3t6EySh8fnhM2PdhlWlpf/BrE0zMoiuBn4JMFl2vJ2Mzo/aoA==
+
 "@eslint/eslintrc@^0.4.3":
   version "0.4.3"
   resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.3.tgz#9e42981ef035beb3dd49add17acb96e8ff6f394c"
@@ -133,6 +138,18 @@
     minimatch "^3.0.4"
     strip-json-comments "^3.1.1"
 
+"@hapi/hoek@^9.0.0":
+  version "9.3.0"
+  resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb"
+  integrity sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==
+
+"@hapi/topo@^5.0.0":
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.1.0.tgz#dc448e332c6c6e37a4dc02fd84ba8d44b9afb012"
+  integrity sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==
+  dependencies:
+    "@hapi/hoek" "^9.0.0"
+
 "@humanwhocodes/config-array@^0.5.0":
   version "0.5.0"
   resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.5.0.tgz#1407967d4c6eecd7388f83acf1eaf4d0c6e58ef9"
@@ -183,6 +200,23 @@
     "@nodelib/fs.scandir" "2.1.5"
     fastq "^1.6.0"
 
+"@sideway/address@^4.1.3":
+  version "4.1.4"
+  resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0"
+  integrity sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw==
+  dependencies:
+    "@hapi/hoek" "^9.0.0"
+
+"@sideway/formula@^3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.0.tgz#fe158aee32e6bd5de85044be615bc08478a0a13c"
+  integrity sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg==
+
+"@sideway/pinpoint@^2.0.0":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df"
+  integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==
+
 "@sindresorhus/is@^0.7.0":
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd"
@@ -2455,6 +2489,17 @@ jmespath@0.15.0:
   resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
   integrity sha512-+kHj8HXArPfpPEKGLZ+kB5ONRTCiGQXo8RQYL0hH8t6pWXUBBK5KkkQmTNOwKK4LEsd0yTsgtjJVm4UBSZea4w==
 
+joi@^17.6.0:
+  version "17.6.0"
+  resolved "https://registry.yarnpkg.com/joi/-/joi-17.6.0.tgz#0bb54f2f006c09a96e75ce687957bd04290054b2"
+  integrity sha512-OX5dG6DTbcr/kbMFj0KGYxuew69HPcAE3K/sZpEV2nP6e/j/C0HV+HNiBPCASxdx5T7DMoa0s8UeHWMnb6n2zw==
+  dependencies:
+    "@hapi/hoek" "^9.0.0"
+    "@hapi/topo" "^5.0.0"
+    "@sideway/address" "^4.1.3"
+    "@sideway/formula" "^3.0.0"
+    "@sideway/pinpoint" "^2.0.0"
+
 join-component@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5"

packages/server/src/definitions/datasource.ts

@@ -1,73 +1,21 @@
 import { Row, Table, Base } from "./common"
+import {
+  Operation,
+  QueryTypes,
+  SortDirection,
+  SourceNames,
+} from "@budibase/types"
 
-export enum Operation {
-  CREATE = "CREATE",
-  READ = "READ",
-  UPDATE = "UPDATE",
-  DELETE = "DELETE",
-  BULK_CREATE = "BULK_CREATE",
-  CREATE_TABLE = "CREATE_TABLE",
-  UPDATE_TABLE = "UPDATE_TABLE",
-  DELETE_TABLE = "DELETE_TABLE",
-}
-
-export enum SortDirection {
-  ASCENDING = "ASCENDING",
-  DESCENDING = "DESCENDING",
-}
-
-export enum QueryTypes {
-  SQL = "sql",
-  JSON = "json",
-  FIELDS = "fields",
-}
-
-export enum DatasourceFieldTypes {
-  STRING = "string",
-  LONGFORM = "longForm",
-  BOOLEAN = "boolean",
-  NUMBER = "number",
-  PASSWORD = "password",
-  LIST = "list",
-  OBJECT = "object",
-  JSON = "json",
-  FILE = "file",
-}
-
-export enum SourceNames {
-  POSTGRES = "POSTGRES",
-  DYNAMODB = "DYNAMODB",
-  MONGODB = "MONGODB",
-  ELASTICSEARCH = "ELASTICSEARCH",
-  COUCHDB = "COUCHDB",
-  SQL_SERVER = "SQL_SERVER",
-  S3 = "S3",
-  AIRTABLE = "AIRTABLE",
-  MYSQL = "MYSQL",
-  ARANGODB = "ARANGODB",
-  REST = "REST",
-  ORACLE = "ORACLE",
-  GOOGLE_SHEETS = "GOOGLE_SHEETS",
-  FIRESTORE = "FIRESTORE",
-  REDIS = "REDIS",
-  SNOWFLAKE = "SNOWFLAKE",
-}
-
-export enum IncludeRelationships {
-  INCLUDE = 1,
-  EXCLUDE = 0,
-}
-
-export enum FilterTypes {
-  STRING = "string",
-  FUZZY = "fuzzy",
-  RANGE = "range",
-  EQUAL = "equal",
-  NOT_EQUAL = "notEqual",
-  EMPTY = "empty",
-  NOT_EMPTY = "notEmpty",
-  ONE_OF = "oneOf",
-}
+// these were previously exported here - moved to types for re-use
+export {
+  Operation,
+  SortDirection,
+  QueryTypes,
+  DatasourceFieldTypes,
+  SourceNames,
+  IncludeRelationships,
+  FilterTypes,
+} from "@budibase/types"
 
 export interface QueryDefinition {
   type: QueryTypes

packages/types/src/documents/app/datasource.ts

@@ -1,5 +1,74 @@
 import { Document } from "../document"
 
+export enum Operation {
+  CREATE = "CREATE",
+  READ = "READ",
+  UPDATE = "UPDATE",
+  DELETE = "DELETE",
+  BULK_CREATE = "BULK_CREATE",
+  CREATE_TABLE = "CREATE_TABLE",
+  UPDATE_TABLE = "UPDATE_TABLE",
+  DELETE_TABLE = "DELETE_TABLE",
+}
+
+export enum SortDirection {
+  ASCENDING = "ASCENDING",
+  DESCENDING = "DESCENDING",
+}
+
+export enum QueryTypes {
+  SQL = "sql",
+  JSON = "json",
+  FIELDS = "fields",
+}
+
+export enum DatasourceFieldTypes {
+  STRING = "string",
+  LONGFORM = "longForm",
+  BOOLEAN = "boolean",
+  NUMBER = "number",
+  PASSWORD = "password",
+  LIST = "list",
+  OBJECT = "object",
+  JSON = "json",
+  FILE = "file",
+}
+
+export enum SourceNames {
+  POSTGRES = "POSTGRES",
+  DYNAMODB = "DYNAMODB",
+  MONGODB = "MONGODB",
+  ELASTICSEARCH = "ELASTICSEARCH",
+  COUCHDB = "COUCHDB",
+  SQL_SERVER = "SQL_SERVER",
+  S3 = "S3",
+  AIRTABLE = "AIRTABLE",
+  MYSQL = "MYSQL",
+  ARANGODB = "ARANGODB",
+  REST = "REST",
+  ORACLE = "ORACLE",
+  GOOGLE_SHEETS = "GOOGLE_SHEETS",
+  FIRESTORE = "FIRESTORE",
+  REDIS = "REDIS",
+  SNOWFLAKE = "SNOWFLAKE",
+}
+
+export enum IncludeRelationships {
+  INCLUDE = 1,
+  EXCLUDE = 0,
+}
+
+export enum FilterTypes {
+  STRING = "string",
+  FUZZY = "fuzzy",
+  RANGE = "range",
+  EQUAL = "equal",
+  NOT_EQUAL = "notEqual",
+  EMPTY = "empty",
+  NOT_EMPTY = "notEmpty",
+  ONE_OF = "oneOf",
+}
+
 export interface Datasource extends Document {
   source: string
 }