From b9b8d59005988baa27e78b49d2bc8664cbe90965 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 30 Aug 2022 10:49:19 +0100 Subject: [PATCH 01/37] add plugins ui --- packages/bbui/src/Modal/ModalContent.svelte | 5 +- .../src/pages/builder/portal/_layout.svelte | 2 + .../plugins/_components/AddPluginModal.svelte | 122 ++++++++++++++++++ .../plugins/_components/PluginRow.svelte | 81 ++++++++++++ .../portal/manage/plugins/index.svelte | 94 ++++++++++++++ packages/builder/src/stores/portal/index.js | 1 + packages/builder/src/stores/portal/plugins.js | 47 +++++++ packages/frontend-core/src/api/plugins.js | 17 ++- packages/server/src/api/controllers/plugin.ts | 14 +- packages/server/src/api/routes/plugin.ts | 4 +- 10 files changed, 377 insertions(+), 10 deletions(-) create mode 100644 packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte create mode 100644 packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte create mode 100644 packages/builder/src/pages/builder/portal/manage/plugins/index.svelte create mode 100644 packages/builder/src/stores/portal/plugins.js diff --git a/packages/bbui/src/Modal/ModalContent.svelte b/packages/bbui/src/Modal/ModalContent.svelte index 6d609d6f1b..4c70f2390a 100644 --- a/packages/bbui/src/Modal/ModalContent.svelte +++ b/packages/bbui/src/Modal/ModalContent.svelte @@ -24,7 +24,7 @@ export let secondaryAction = undefined export let secondaryButtonWarning = false export let dataCy = null - + export let buttonCta = true const { hide, cancel } = getContext(Context.Modal) let loading = false $: confirmDisabled = disabled || loading @@ -93,7 +93,6 @@ class="spectrum-ButtonGroup spectrum-Dialog-buttonGroup spectrum-Dialog-buttonGroup--noFooter" > - {#if showSecondaryButton && secondaryButtonText && secondaryAction}
+
+
+
+ @@ -61,7 +98,7 @@
- +
{#each authOptions[sourceValue] as option} {#if option === "Upload"} @@ -82,7 +119,7 @@ {:else}
- +
{/if} {/each} diff --git a/packages/builder/src/stores/portal/plugins.js b/packages/builder/src/stores/portal/plugins.js index 6a1d7cc2cb..531ac1c396 100644 --- a/packages/builder/src/stores/portal/plugins.js +++ b/packages/builder/src/stores/portal/plugins.js @@ -17,6 +17,41 @@ export function createPluginsStore() { }) } + async function createPlugin(type, source, name, url, auth) { + let pluginData = { + type, + source, + name, + url, + } + + switch (source) { + case "github": + pluginData.githubToken = auth + break + case "url": + pluginData.header = auth + break + case "npm": + pluginData.npmToken = auth + break + } + + let resp = await API.createPlugin(pluginData) + console.log(resp) + // TODO_RIC + // let newPlugin = resp.plugins[0] + // update(state => { + // const currentIdx = state.findIndex(plugin => plugin._id === newPlugin._id) + // if (currentIdx >= 0) { + // state.splice(currentIdx, 1, newPlugin) + // } else { + // state.push(newPlugin) + // } + // return state + // }) + } + async function uploadPlugin(file, source) { let data = new FormData() data.append("file", file) @@ -35,6 +70,7 @@ export function createPluginsStore() { return { subscribe, load, + createPlugin, deletePlugin, uploadPlugin, } diff --git a/packages/frontend-core/src/api/plugins.js b/packages/frontend-core/src/api/plugins.js index 108e1b50a7..484cf763bb 100644 --- a/packages/frontend-core/src/api/plugins.js +++ b/packages/frontend-core/src/api/plugins.js @@ -11,6 +11,16 @@ export const buildPluginEndpoints = API => ({ }) }, + /** + * Creates a plugin from URL, Github or NPM + */ + createPlugin: async data => { + return await API.post({ + url: `/api/plugin`, + body: data, + }) + }, + /** * Gets a list of all plugins */ diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index e8a2dd1746..b6d3d31354 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -1,5 +1,9 @@ import { ObjectStoreBuckets } from "../../constants" -import { extractPluginTarball } from "../../utilities/fileSystem" +import { + extractPluginTarball, + npmPlugin, + getPluginMetadata, +} from "../../utilities/fileSystem" import { getGlobalDB } from "@budibase/backend-core/tenancy" import { generatePluginID, getPluginParams } from "../../db/utils" import { uploadDirectory } from "@budibase/backend-core/objectStore" @@ -39,10 +43,48 @@ export async function upload(ctx: any) { } } catch (err: any) { const errMsg = err?.message ? err?.message : err + ctx.throw(400, `Failed to import plugin: ${errMsg}`) } } +export async function create(ctx: any) { + const { type, source, name, url, header, githubToken, npmToken } = + ctx.request.body + let metadata + let directory + + switch (source) { + case "npm": + // const { metadata: metadataNpm, directory: directoryNpm } = await npmPlugin(url, name) + // metadata = metadataNpm + // directory = directoryNpm + + console.log(22222, await getPluginMetadata(await npmPlugin(url, name))) + break + case "github": + console.log("github") + break + case "url": + console.log("url") + break + } + + // try { + // const doc = storePlugin(metadata, directory, source) + // + // ctx.body = { + // message: "Plugin uploaded successfully", + // plugins: doc, + // } + // } catch (err: any) { + // const errMsg = err?.message ? err?.message : err + // + // ctx.throw(400, `Failed to import plugin: ${errMsg}`) + // } + ctx.status = 200 +} + export async function fetch(ctx: any) { ctx.body = await getPlugins() } @@ -54,9 +96,12 @@ export async function destroy(ctx: any) { ctx.status = 200 } -export async function processPlugin(plugin: FileType, source?: string) { +export async function storePlugin( + metadata: any, + directory: any, + source?: string +) { const db = getGlobalDB() - const { metadata, directory } = await extractPluginTarball(plugin) const version = metadata.package.version, name = metadata.package.name, description = metadata.package.description @@ -99,3 +144,8 @@ export async function processPlugin(plugin: FileType, source?: string) { _rev: response.rev, } } + +export async function processPlugin(plugin: FileType, source?: string) { + const { metadata, directory } = await extractPluginTarball(plugin) + return await storePlugin(metadata, directory, source) +} diff --git a/packages/server/src/api/routes/plugin.ts b/packages/server/src/api/routes/plugin.ts index 1ca49cf300..2508115a29 100644 --- a/packages/server/src/api/routes/plugin.ts +++ b/packages/server/src/api/routes/plugin.ts @@ -7,6 +7,7 @@ const router = new Router() router .post("/api/plugin/upload/:source", authorized(BUILDER), controller.upload) + .post("/api/plugin", authorized(BUILDER), controller.create) .get("/api/plugin", authorized(BUILDER), controller.fetch) .delete( "/api/plugin/:pluginId/:pluginRev", diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 821e905fbc..9c70edf8a6 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -1,6 +1,9 @@ const { budibaseTempDir } = require("../budibaseDir") const fs = require("fs") const { join } = require("path") +// const { promisify } = require("util") +// const exec = promisify(require("child_process").exec) +// const streamPipeline = promisify(require("stream")) const uuid = require("uuid/v4") const { doWithDB, @@ -29,6 +32,7 @@ const MemoryStream = require("memorystream") const { getAppId } = require("@budibase/backend-core/context") const tar = require("tar") const fetch = require("node-fetch") +// const fileType = require("file-type") const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules") @@ -326,11 +330,11 @@ exports.cleanup = appIds => { } } -exports.extractPluginTarball = async file => { - if (!file.name.endsWith(".tar.gz")) { +const extractPluginTarball = async (file, ext = ".tar.gz") => { + if (!file.name.endsWith(ext)) { throw new Error("Plugin must be compressed into a gzipped tarball.") } - const path = join(budibaseTempDir(), file.name.split(".tar.gz")[0]) + const path = join(budibaseTempDir(), file.name.split(ext)[0]) // remove old tmp directories automatically - don't combine if (fs.existsSync(path)) { fs.rmSync(path, { recursive: true, force: true }) @@ -340,6 +344,63 @@ exports.extractPluginTarball = async file => { file: file.path, C: path, }) + + return await getPluginMetadata(path) +} +exports.extractPluginTarball = extractPluginTarball + +exports.npmPlugin = async (url, name = "") => { + let npmTarball = url + let filename = name + let path = join(budibaseTempDir(), name) + + if (!npmTarball.includes(".tgz")) { + const npmPackageURl = url.replace( + "https://www.npmjs.com/package/", + "https://registry.npmjs.org/" + ) + const response = await fetch(npmPackageURl) + if (response.status === 200) { + let npmDetails = await response.json() + filename = npmDetails.name + path = join(budibaseTempDir(), filename) + const npmVersion = npmDetails["dist-tags"].latest + npmTarball = npmDetails.versions[npmVersion].dist.tarball + } else { + throw "Cannot get package details" + } + } + + try { + if (fs.existsSync(path)) { + fs.rmSync(path, { recursive: true, force: true }) + } + fs.mkdirSync(path) + + const response = await fetch(npmTarball) + if (!response.ok) + throw new Error(`Loading NPM plugin failed ${response.statusText}`) + + // const dest = fs.createWriteStream(`${path}/${filename}.tgz`) + await response.body.pipe( + await tar.x({ + strip: 1, + C: path, + }) + ) + + // const readStream = fs.createReadStream(`${path}/${filename}.tgz`) + // readStream.pipe( + + // ) + } catch (e) { + throw `Cannot store package locally: ${e.message}` + } + + return path +} + +const getPluginMetadata = async path => { let metadata = {} try { const pkg = fs.readFileSync(join(path, "package.json"), "utf8") @@ -349,8 +410,10 @@ exports.extractPluginTarball = async file => { } catch (err) { throw new Error("Unable to process schema.json/package.json in plugin.") } + return { metadata, directory: path } } +exports.getPluginMetadata = getPluginMetadata exports.getDatasourcePlugin = async (name, url, hash) => { if (!fs.existsSync(DATASOURCE_PATH)) { From 0e2a86460ce37ea331a3f3e519767a3e0714e788 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Wed, 31 Aug 2022 16:09:47 +0100 Subject: [PATCH 04/37] uploading npm and url plugins --- .../plugins/_components/AddPluginModal.svelte | 16 ++---- packages/builder/src/stores/portal/plugins.js | 11 ++--- packages/server/src/api/controllers/plugin.ts | 45 ++++++++--------- .../server/src/utilities/fileSystem/index.js | 49 +++++++++++-------- 4 files changed, 60 insertions(+), 61 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte index e45655d20d..1fce6746ad 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte @@ -11,10 +11,10 @@ import { plugins } from "stores/portal" let authOptions = { - NPM: ["NPM Token", "URL"], + NPM: ["URL"], Github: ["Github Token", "URL"], - URL: ["Header", "URL"], - File: ["Path", "Header"], + URL: ["Headers", "URL"], + File: ["Path", "Headers"], Upload: ["Upload"], } let file @@ -50,17 +50,11 @@ source, nameValue, url, - dynamicValues["Header"] + dynamicValues["Headers"] ) break case "npm": - await plugins.createPlugin( - typeValue, - source, - nameValue, - url, - dynamicValues["NPM Token"] - ) + await plugins.createPlugin(typeValue, source, nameValue, url) break } } diff --git a/packages/builder/src/stores/portal/plugins.js b/packages/builder/src/stores/portal/plugins.js index 531ac1c396..4e3d952fe3 100644 --- a/packages/builder/src/stores/portal/plugins.js +++ b/packages/builder/src/stores/portal/plugins.js @@ -17,7 +17,7 @@ export function createPluginsStore() { }) } - async function createPlugin(type, source, name, url, auth) { + async function createPlugin(type, source, name, url, auth = null) { let pluginData = { type, source, @@ -26,19 +26,16 @@ export function createPluginsStore() { } switch (source) { - case "github": - pluginData.githubToken = auth - break case "url": - pluginData.header = auth + pluginData.headers = auth break case "npm": pluginData.npmToken = auth break } - let resp = await API.createPlugin(pluginData) - console.log(resp) + let res = await API.createPlugin(pluginData) + console.log("RESP", res) // TODO_RIC // let newPlugin = resp.plugins[0] // update(state => { diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index b6d3d31354..dc70ea8b62 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -1,8 +1,8 @@ import { ObjectStoreBuckets } from "../../constants" import { extractPluginTarball, - npmPlugin, - getPluginMetadata, + createNpmPlugin, + createUrlPlugin, } from "../../utilities/fileSystem" import { getGlobalDB } from "@budibase/backend-core/tenancy" import { generatePluginID, getPluginParams } from "../../db/utils" @@ -49,39 +49,40 @@ export async function upload(ctx: any) { } export async function create(ctx: any) { - const { type, source, name, url, header, githubToken, npmToken } = - ctx.request.body + const { type, source, name, url, headers, githubToken } = ctx.request.body let metadata let directory switch (source) { case "npm": - // const { metadata: metadataNpm, directory: directoryNpm } = await npmPlugin(url, name) - // metadata = metadataNpm - // directory = directoryNpm - - console.log(22222, await getPluginMetadata(await npmPlugin(url, name))) + const { metadata: metadataNpm, directory: directoryNpm } = + await createNpmPlugin(url, name) + metadata = metadataNpm + directory = directoryNpm break case "github": console.log("github") break case "url": - console.log("url") + const { metadata: metadataUrl, directory: directoryUrl } = + await createUrlPlugin(url, name, headers) + metadata = metadataUrl + directory = directoryUrl break } - // try { - // const doc = storePlugin(metadata, directory, source) - // - // ctx.body = { - // message: "Plugin uploaded successfully", - // plugins: doc, - // } - // } catch (err: any) { - // const errMsg = err?.message ? err?.message : err - // - // ctx.throw(400, `Failed to import plugin: ${errMsg}`) - // } + try { + const doc = storePlugin(metadata, directory, source) + + ctx.body = { + message: "Plugin uploaded successfully", + plugins: doc, + } + } catch (err: any) { + const errMsg = err?.message ? err?.message : err + + ctx.throw(400, `Failed to import plugin: ${errMsg}`) + } ctx.status = 200 } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 9c70edf8a6..710e3c0294 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -1,9 +1,8 @@ const { budibaseTempDir } = require("../budibaseDir") const fs = require("fs") const { join } = require("path") -// const { promisify } = require("util") -// const exec = promisify(require("child_process").exec) -// const streamPipeline = promisify(require("stream")) +const { promisify } = require("util") +const streamPipeline = promisify(require("stream").pipeline) const uuid = require("uuid/v4") const { doWithDB, @@ -32,7 +31,6 @@ const MemoryStream = require("memorystream") const { getAppId } = require("@budibase/backend-core/context") const tar = require("tar") const fetch = require("node-fetch") -// const fileType = require("file-type") const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules") @@ -349,10 +347,9 @@ const extractPluginTarball = async (file, ext = ".tar.gz") => { } exports.extractPluginTarball = extractPluginTarball -exports.npmPlugin = async (url, name = "") => { +exports.createNpmPlugin = async (url, name = "") => { let npmTarball = url - let filename = name - let path = join(budibaseTempDir(), name) + let pluginName = name if (!npmTarball.includes(".tgz")) { const npmPackageURl = url.replace( @@ -362,8 +359,7 @@ exports.npmPlugin = async (url, name = "") => { const response = await fetch(npmPackageURl) if (response.status === 200) { let npmDetails = await response.json() - filename = npmDetails.name - path = join(budibaseTempDir(), filename) + pluginName = npmDetails.name const npmVersion = npmDetails["dist-tags"].latest npmTarball = npmDetails.versions[npmVersion].dist.tarball } else { @@ -371,36 +367,47 @@ exports.npmPlugin = async (url, name = "") => { } } + return await downloadUnzipPlugin(pluginName, npmTarball) +} + +exports.createUrlPlugin = async (url, name = "", headers = {}) => { + if (!url.includes(".tgz") && !url.includes(".tar.gz")) { + throw new Error("Plugin must be compressed into a gzipped tarball.") + } + + return await downloadUnzipPlugin(name, url, headers) +} + +const downloadUnzipPlugin = async (name, url, headers = {}) => { + console.log(name, url, headers) + const path = join(budibaseTempDir(), name) try { + // Remove first if exists if (fs.existsSync(path)) { fs.rmSync(path, { recursive: true, force: true }) } fs.mkdirSync(path) - const response = await fetch(npmTarball) + const response = await fetch(url, { headers }) if (!response.ok) throw new Error(`Loading NPM plugin failed ${response.statusText}`) - // const dest = fs.createWriteStream(`${path}/${filename}.tgz`) - await response.body.pipe( - await tar.x({ + await streamPipeline( + response.body, + tar.x({ strip: 1, C: path, }) ) - - // const readStream = fs.createReadStream(`${path}/${filename}.tgz`) - // readStream.pipe( - - // ) + return await getPluginMetadata(path) } catch (e) { - throw `Cannot store package locally: ${e.message}` + throw `Cannot store plugin locally: ${e.message}` } - - return path } +exports.downloadUnzipPlugin = downloadUnzipPlugin const getPluginMetadata = async path => { + console.log(path) let metadata = {} try { const pkg = fs.readFileSync(join(path, "package.json"), "utf8") From 6daa49927ed4e58bcdefe94bba6b64bd9c0a3335 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Wed, 31 Aug 2022 17:53:00 +0100 Subject: [PATCH 05/37] plugins npm and url working --- packages/builder/src/stores/portal/plugins.js | 22 +++++++++---------- packages/server/src/api/controllers/plugin.ts | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/stores/portal/plugins.js b/packages/builder/src/stores/portal/plugins.js index 4e3d952fe3..a498bf663a 100644 --- a/packages/builder/src/stores/portal/plugins.js +++ b/packages/builder/src/stores/portal/plugins.js @@ -36,17 +36,17 @@ export function createPluginsStore() { let res = await API.createPlugin(pluginData) console.log("RESP", res) - // TODO_RIC - // let newPlugin = resp.plugins[0] - // update(state => { - // const currentIdx = state.findIndex(plugin => plugin._id === newPlugin._id) - // if (currentIdx >= 0) { - // state.splice(currentIdx, 1, newPlugin) - // } else { - // state.push(newPlugin) - // } - // return state - // }) + + let newPlugin = res.plugins[0] + update(state => { + const currentIdx = state.findIndex(plugin => plugin._id === newPlugin._id) + if (currentIdx >= 0) { + state.splice(currentIdx, 1, newPlugin) + } else { + state.push(newPlugin) + } + return state + }) } async function uploadPlugin(file, source) { diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index dc70ea8b62..26caf38806 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -72,11 +72,11 @@ export async function create(ctx: any) { } try { - const doc = storePlugin(metadata, directory, source) + const doc = await storePlugin(metadata, directory, source) ctx.body = { message: "Plugin uploaded successfully", - plugins: doc, + plugins: [doc], } } catch (err: any) { const errMsg = err?.message ? err?.message : err From 54eed13541f4075547b8fcc7081c8616d8938b4b Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Thu, 1 Sep 2022 20:04:45 +0100 Subject: [PATCH 06/37] create plugin github public and private --- packages/builder/src/stores/portal/plugins.js | 4 +- packages/server/src/api/controllers/plugin.ts | 6 +- .../server/src/utilities/fileSystem/index.js | 65 ++++++++++++++++++- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/stores/portal/plugins.js b/packages/builder/src/stores/portal/plugins.js index a498bf663a..821934307d 100644 --- a/packages/builder/src/stores/portal/plugins.js +++ b/packages/builder/src/stores/portal/plugins.js @@ -32,10 +32,12 @@ export function createPluginsStore() { case "npm": pluginData.npmToken = auth break + case "github": + pluginData.githubToken = auth + break } let res = await API.createPlugin(pluginData) - console.log("RESP", res) let newPlugin = res.plugins[0] update(state => { diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index 26caf38806..1159b08dce 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -3,6 +3,7 @@ import { extractPluginTarball, createNpmPlugin, createUrlPlugin, + createGithubPlugin, } from "../../utilities/fileSystem" import { getGlobalDB } from "@budibase/backend-core/tenancy" import { generatePluginID, getPluginParams } from "../../db/utils" @@ -61,7 +62,10 @@ export async function create(ctx: any) { directory = directoryNpm break case "github": - console.log("github") + const { metadata: metadataGithub, directory: directoryGithub } = + await createGithubPlugin(ctx, url, name, githubToken) + metadata = metadataGithub + directory = directoryGithub break case "url": const { metadata: metadataUrl, directory: directoryUrl } = diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 710e3c0294..d8c201a380 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -31,6 +31,7 @@ const MemoryStream = require("memorystream") const { getAppId } = require("@budibase/backend-core/context") const tar = require("tar") const fetch = require("node-fetch") +const { NodeVM } = require("vm2") const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules") @@ -378,8 +379,69 @@ exports.createUrlPlugin = async (url, name = "", headers = {}) => { return await downloadUnzipPlugin(name, url, headers) } +exports.createGithubPlugin = async (ctx, url, name = "", token = "") => { + let githubRepositoryUrl + let githubUrl + + if (url.includes(".git")) { + githubRepositoryUrl = token + ? url.replace("https://", `https://${token}@`) + : url + githubUrl = url.replace(".git", "") + } else { + githubRepositoryUrl = token + ? `${url}.git`.replace("https://", `https://${token}@`) + : `${url}.git` + githubUrl = url + } + + const githubApiUrl = githubUrl.replace( + "https://github.com/", + "https://api.github.com/repos/" + ) + const headers = token ? { Authorization: `Bearer ${token}` } : {} + try { + const pluginRaw = await fetch(githubApiUrl, { headers }) + if (pluginRaw.status !== 200) { + throw `Repository not found` + } + + let pluginDetails = await pluginRaw.json() + const pluginName = pluginDetails.name || name + + const path = join(budibaseTempDir(), pluginName) + // Remove first if exists + if (fs.existsSync(path)) { + fs.rmSync(path, { recursive: true, force: true }) + } + fs.mkdirSync(path) + + const script = ` +module.exports = async () => { + const child_process = require('child_process') + child_process.execSync(\`git clone ${githubRepositoryUrl} ${join( + budibaseTempDir(), + pluginName + )}\`); +} +` + const scriptRunner = new NodeVM({ + require: { + external: true, + builtin: ["child_process"], + root: "./", + }, + }).run(script) + + await scriptRunner() + + return await getPluginMetadata(path) + } catch (e) { + throw e.message + } +} + const downloadUnzipPlugin = async (name, url, headers = {}) => { - console.log(name, url, headers) const path = join(budibaseTempDir(), name) try { // Remove first if exists @@ -407,7 +469,6 @@ const downloadUnzipPlugin = async (name, url, headers = {}) => { exports.downloadUnzipPlugin = downloadUnzipPlugin const getPluginMetadata = async path => { - console.log(path) let metadata = {} try { const pkg = fs.readFileSync(join(path, "package.json"), "utf8") From 5a08834def2c5ff8ec6804c91d2868896789e238 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Fri, 2 Sep 2022 19:16:13 +0100 Subject: [PATCH 07/37] delete plugin through modal confirmation --- .../_components/DeletePluginModal.svelte | 31 +++++++++++++++++++ .../plugins/_components/PluginRow.svelte | 16 +++++++--- .../portal/manage/plugins/index.svelte | 16 +++++++++- 3 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 packages/builder/src/pages/builder/portal/manage/plugins/_components/DeletePluginModal.svelte diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/DeletePluginModal.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/DeletePluginModal.svelte new file mode 100644 index 0000000000..3907c1a0a2 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/DeletePluginModal.svelte @@ -0,0 +1,31 @@ + + + + + Are you sure you want to delete {plugin?.name} + + diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte index 3a92a699e6..9c166b8b1b 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte @@ -1,6 +1,8 @@
@@ -41,9 +51,7 @@ - plugins.deletePlugin(plugin._id, plugin._rev)} - icon="Delete">Delete remove(plugin)} icon="Delete">Delete editGroup(plugin)} icon="Edit">Edit diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte index 61227786b0..489333b095 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte @@ -13,9 +13,12 @@ import { plugins } from "stores/portal" import PluginRow from "./_components/PluginRow.svelte" import AddPluginModal from "./_components/AddPluginModal.svelte" + import DeletePluginModal from "./_components/DeletePluginModal.svelte" let modal + let deleteModal let searchTerm = "" + let removePlugin let filterOptions = [ { label: "All Plugins", value: "all" }, @@ -31,6 +34,14 @@ .filter(plugin => plugin?.name?.toLowerCase().includes(searchTerm.toLowerCase()) ) + + const deletePlugin = evt => { + const { detail } = evt + + deleteModal.show() + removePlugin = detail + } + onMount(async () => { await plugins.load() }) @@ -66,7 +77,7 @@ {#if $plugins} {#each filteredPlugins as plugin} - + {/each} {/if} @@ -75,6 +86,9 @@ + + + From 32b336cc2ddf2749b419a29d1883e95a68419feb Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Mon, 5 Sep 2022 10:13:55 +0100 Subject: [PATCH 10/37] cleaning minio folder when deleting plugin --- packages/server/src/api/controllers/plugin.ts | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index 337df0fc9e..8ec898d67d 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -4,11 +4,14 @@ import { createNpmPlugin, createUrlPlugin, createGithubPlugin, - loadJSFile + loadJSFile, } from "../../utilities/fileSystem" import { getGlobalDB } from "@budibase/backend-core/tenancy" import { generatePluginID, getPluginParams } from "../../db/utils" -import { uploadDirectory } from "@budibase/backend-core/objectStore" +import { + uploadDirectory, + deleteFolder, +} from "@budibase/backend-core/objectStore" import { PluginType, FileType } from "@budibase/types" import env from "../../environment" @@ -98,7 +101,20 @@ export async function fetch(ctx: any) { export async function destroy(ctx: any) { const db = getGlobalDB() - await db.remove(ctx.params.pluginId, ctx.params.pluginRev) + const { pluginId, pluginRev } = ctx.params + + try { + const plugin = await db.get(pluginId) + const bucketPath = `${plugin.name}/` + await deleteFolder(ObjectStoreBuckets.PLUGINS, bucketPath) + + await db.remove(pluginId, pluginRev) + } catch (err: any) { + const errMsg = err?.message ? err?.message : err + + ctx.throw(400, `Failed to delete plugin: ${errMsg}`) + } + ctx.message = `Plugin ${ctx.params.pluginId} deleted.` ctx.status = 200 } From a03d007821ea3224f21edfb692ee15c6355059eb Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Mon, 5 Sep 2022 10:28:09 +0100 Subject: [PATCH 11/37] random name in case it is needed --- packages/server/src/api/controllers/plugin.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index 8ec898d67d..254a008781 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -55,9 +55,11 @@ export async function upload(ctx: any) { } export async function create(ctx: any) { - const { type, source, name, url, headers, githubToken } = ctx.request.body + const { type, source, url, headers, githubToken } = ctx.request.body let metadata let directory + // Generating random name as a backup and needed for url + let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000) switch (source) { case "npm": From cb4b918e5a3cf79733f306c9db305d81a101ad47 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 5 Sep 2022 11:27:43 +0100 Subject: [PATCH 12/37] remove name selection from UI --- packages/bbui/src/Modal/ModalContent.svelte | 2 +- .../plugins/_components/AddPluginModal.svelte | 1 + .../manage/plugins/_components/PluginRow.svelte | 14 ++++---------- packages/builder/src/stores/portal/plugins.js | 3 +-- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/bbui/src/Modal/ModalContent.svelte b/packages/bbui/src/Modal/ModalContent.svelte index 4c70f2390a..d946268da6 100644 --- a/packages/bbui/src/Modal/ModalContent.svelte +++ b/packages/bbui/src/Modal/ModalContent.svelte @@ -88,7 +88,7 @@
- {#if showCancelButton || showConfirmButton} + {#if showCancelButton || showConfirmButton || $$slots.footer}
diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte index 7149136dec..2ccc9283e9 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte @@ -92,6 +92,7 @@ { if (!e.detail || e.detail.length === 0) { diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte index 0eabad111a..30fddb2f76 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte @@ -8,9 +8,9 @@ Label, Input, } from "@budibase/bbui" - import { plugins } from "stores/portal" export let plugin + export let deletePlugin let detailsModal @@ -74,17 +74,11 @@
- deletePlugin(plugin)} warning + >Delete - +
diff --git a/packages/builder/src/stores/portal/plugins.js b/packages/builder/src/stores/portal/plugins.js index 821934307d..9d558561b6 100644 --- a/packages/builder/src/stores/portal/plugins.js +++ b/packages/builder/src/stores/portal/plugins.js @@ -17,11 +17,10 @@ export function createPluginsStore() { }) } - async function createPlugin(type, source, name, url, auth = null) { + async function createPlugin(type, source, url, auth = null) { let pluginData = { type, source, - name, url, } From 5ad21ad3332bc21ad75cab0045c8d1df198e027f Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 5 Sep 2022 14:39:19 +0100 Subject: [PATCH 13/37] improve responsiveness of plugins ui --- .../plugins/_components/PluginRow.svelte | 27 +++++++++++++------ packages/server/src/api/controllers/plugin.ts | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte index 30fddb2f76..848611c248 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte @@ -12,6 +12,11 @@ export let plugin export let deletePlugin + let icon = + plugin.schema.type === "component" + ? plugin.schema.schema.icon || "Book" + : plugin.schema.schema.icon || "Beaker" + let detailsModal @@ -19,7 +24,7 @@
- +
-
+
+ + +
+ +
-
+
-
+
-
+
-
+
@@ -88,7 +99,7 @@ diff --git a/packages/builder/src/stores/portal/plugins.js b/packages/builder/src/stores/portal/plugins.js index a3d62ca6e3..9703b30cb7 100644 --- a/packages/builder/src/stores/portal/plugins.js +++ b/packages/builder/src/stores/portal/plugins.js @@ -50,11 +50,11 @@ export function createPluginsStore() { }) } - async function uploadPlugin(file, source) { + async function uploadPlugin(file, source, updatePlugin) { let data = new FormData() data.append("file", file) data.append("source", source) - + data.append("update", updatePlugin) let resp = await API.uploadPlugin(data) let newPlugin = resp.plugins[0] update(state => { diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index ce35d2fad5..a84a077b33 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -39,7 +39,11 @@ export async function upload(ctx: any) { let docs = [] // can do single or multiple plugins for (let plugin of plugins) { - const doc = await processPlugin(plugin, ctx.request.body.source) + const doc = await processPlugin( + plugin, + ctx.request.body.source, + ctx.request.body.update + ) docs.push(doc) } ctx.body = { @@ -128,7 +132,8 @@ export async function destroy(ctx: any) { export async function storePlugin( metadata: any, directory: any, - source?: string + source?: string, + update?: boolean ) { const db = getGlobalDB() const version = metadata.package.version, @@ -167,6 +172,9 @@ export async function storePlugin( const existing = await db.get(pluginId) rev = existing._rev } catch (err) { + if (update) { + throw new Error("Unable to update. Plugin does not exist") + } rev = undefined } let doc = { @@ -185,6 +193,7 @@ export async function storePlugin( source, } } + const response = await db.put(doc) return { ...doc, @@ -192,11 +201,15 @@ export async function storePlugin( } } -export async function processPlugin(plugin: FileType, source?: string) { +export async function processPlugin( + plugin: FileType, + source?: string, + update?: boolean +) { if (!env.SELF_HOSTED) { throw new Error("Plugins not supported outside of self-host.") } const { metadata, directory } = await uploadedFilePlugin(plugin) - return await storePlugin(metadata, directory, source) + return await storePlugin(metadata, directory, source, update) } From 0ebe0b827575e97694b56ecf258c8dc1c8255367 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 7 Sep 2022 16:08:29 +0100 Subject: [PATCH 20/37] remove verify and fix deletion bug --- packages/bbui/src/Modal/ModalContent.svelte | 3 +- .../plugins/_components/AddPluginModal.svelte | 49 ++-------------- .../_components/DeletePluginModal.svelte | 5 +- .../plugins/_components/PluginRow.svelte | 58 +------------------ packages/builder/src/stores/portal/plugins.js | 3 +- packages/server/src/api/controllers/plugin.ts | 20 ++----- 6 files changed, 15 insertions(+), 123 deletions(-) diff --git a/packages/bbui/src/Modal/ModalContent.svelte b/packages/bbui/src/Modal/ModalContent.svelte index d946268da6..25fac63ec8 100644 --- a/packages/bbui/src/Modal/ModalContent.svelte +++ b/packages/bbui/src/Modal/ModalContent.svelte @@ -24,7 +24,6 @@ export let secondaryAction = undefined export let secondaryButtonWarning = false export let dataCy = null - export let buttonCta = true const { hide, cancel } = getContext(Context.Modal) let loading = false $: confirmDisabled = disabled || loading @@ -113,7 +112,7 @@
{/if} {/each} - -
-
- -
-
- {#if verificationSuccessful} - Verification Successful - {:else} - Verify your source - {/if} -
-
diff --git a/packages/builder/src/stores/portal/plugins.js b/packages/builder/src/stores/portal/plugins.js index 9703b30cb7..28f9bfc42d 100644 --- a/packages/builder/src/stores/portal/plugins.js +++ b/packages/builder/src/stores/portal/plugins.js @@ -50,11 +50,10 @@ export function createPluginsStore() { }) } - async function uploadPlugin(file, source, updatePlugin) { + async function uploadPlugin(file, source) { let data = new FormData() data.append("file", file) data.append("source", source) - data.append("update", updatePlugin) let resp = await API.uploadPlugin(data) let newPlugin = resp.plugins[0] update(state => { diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index a84a077b33..cb4d592812 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -39,11 +39,7 @@ export async function upload(ctx: any) { let docs = [] // can do single or multiple plugins for (let plugin of plugins) { - const doc = await processPlugin( - plugin, - ctx.request.body.source, - ctx.request.body.update - ) + const doc = await processPlugin(plugin, ctx.request.body.source) docs.push(doc) } ctx.body = { @@ -132,8 +128,7 @@ export async function destroy(ctx: any) { export async function storePlugin( metadata: any, directory: any, - source?: string, - update?: boolean + source?: string ) { const db = getGlobalDB() const version = metadata.package.version, @@ -172,9 +167,6 @@ export async function storePlugin( const existing = await db.get(pluginId) rev = existing._rev } catch (err) { - if (update) { - throw new Error("Unable to update. Plugin does not exist") - } rev = undefined } let doc = { @@ -201,15 +193,11 @@ export async function storePlugin( } } -export async function processPlugin( - plugin: FileType, - source?: string, - update?: boolean -) { +export async function processPlugin(plugin: FileType, source?: string) { if (!env.SELF_HOSTED) { throw new Error("Plugins not supported outside of self-host.") } const { metadata, directory } = await uploadedFilePlugin(plugin) - return await storePlugin(metadata, directory, source, update) + return await storePlugin(metadata, directory, source) } From ac0443a876cf0c36e3bde3f83edaf8412a49c0a7 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 7 Sep 2022 17:49:29 +0100 Subject: [PATCH 21/37] fix build --- .../portal/manage/plugins/_components/AddPluginModal.svelte | 6 +++--- packages/server/src/api/controllers/plugin/utils.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte index f45eea9dc9..3520c18991 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte @@ -3,14 +3,14 @@ import { plugins } from "stores/portal" let authOptions = { + URL: ["Headers", "URL"], NPM: ["URL"], Github: ["Github Token", "URL"], - URL: ["Headers", "URL"], "File Upload": ["File Upload"], } let file - let sourceValue = "NPM" - let typeValue = "Datasource" + let sourceValue = "URL" + let typeValue = "Component" let dynamicValues = {} let validation diff --git a/packages/server/src/api/controllers/plugin/utils.js b/packages/server/src/api/controllers/plugin/utils.js index 86f8754a7a..df76ebc121 100644 --- a/packages/server/src/api/controllers/plugin/utils.js +++ b/packages/server/src/api/controllers/plugin/utils.js @@ -1,4 +1,3 @@ -import fetch from "node-fetch" import { createTempFolder, getPluginMetadata, @@ -8,6 +7,7 @@ import { deleteFolderFileSystem, } from "../../../utilities/fileSystem" import { join } from "path" +const fetch = require("node-fetch") export const uploadedFilePlugin = async file => { if (!file.name.endsWith(".tar.gz")) { From 3ef877f886a6a1d9b557559a1032cf2f4ddacb19 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 7 Sep 2022 18:11:17 +0100 Subject: [PATCH 22/37] fix build again --- packages/server/src/api/controllers/plugin/utils.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/plugin/utils.js b/packages/server/src/api/controllers/plugin/utils.js index df76ebc121..e15a72789e 100644 --- a/packages/server/src/api/controllers/plugin/utils.js +++ b/packages/server/src/api/controllers/plugin/utils.js @@ -1,12 +1,12 @@ -import { +const { createTempFolder, getPluginMetadata, findFileRec, downloadTarballDirect, extractTarball, deleteFolderFileSystem, -} from "../../../utilities/fileSystem" -import { join } from "path" +} = require("../../../utilities/fileSystem") +const join = require("path") const fetch = require("node-fetch") export const uploadedFilePlugin = async file => { From 28e8ce329baa007c5bfc7dbfaa151d5ad2645143 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Fri, 9 Sep 2022 11:10:48 +0100 Subject: [PATCH 23/37] fixed building issue --- packages/server/src/api/controllers/plugin/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/plugin/utils.js b/packages/server/src/api/controllers/plugin/utils.js index e15a72789e..f2a9fd6981 100644 --- a/packages/server/src/api/controllers/plugin/utils.js +++ b/packages/server/src/api/controllers/plugin/utils.js @@ -6,7 +6,7 @@ const { extractTarball, deleteFolderFileSystem, } = require("../../../utilities/fileSystem") -const join = require("path") +const { join } = require("path") const fetch = require("node-fetch") export const uploadedFilePlugin = async file => { From e5806b832fbc80dddd9a50ad0976fcc6038f15e4 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Fri, 9 Sep 2022 11:12:14 +0100 Subject: [PATCH 24/37] moved validate to backend-core to be used in server and cli --- packages/backend-core/plugins.js | 3 + packages/backend-core/src/plugin/index.ts | 1 + packages/backend-core/src/plugin/utils.js | 94 +++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 packages/backend-core/plugins.js create mode 100644 packages/backend-core/src/plugin/index.ts create mode 100644 packages/backend-core/src/plugin/utils.js diff --git a/packages/backend-core/plugins.js b/packages/backend-core/plugins.js new file mode 100644 index 0000000000..018e214dcb --- /dev/null +++ b/packages/backend-core/plugins.js @@ -0,0 +1,3 @@ +module.exports = { + ...require("./src/plugin"), +} diff --git a/packages/backend-core/src/plugin/index.ts b/packages/backend-core/src/plugin/index.ts new file mode 100644 index 0000000000..3eeaeaa90c --- /dev/null +++ b/packages/backend-core/src/plugin/index.ts @@ -0,0 +1 @@ +export * from "./utils" diff --git a/packages/backend-core/src/plugin/utils.js b/packages/backend-core/src/plugin/utils.js new file mode 100644 index 0000000000..020fb4484d --- /dev/null +++ b/packages/backend-core/src/plugin/utils.js @@ -0,0 +1,94 @@ +const { + DatasourceFieldType, + QueryType, + PluginType, +} = require("@budibase/types") +const joi = require("joi") + +const DATASOURCE_TYPES = [ + "Relational", + "Non-relational", + "Spreadsheet", + "Object store", + "Graph", + "API", +] + +function runJoi(validator, schema) { + const { error } = validator.validate(schema) + if (error) { + throw error + } +} + +function validateComponent(schema) { + const validator = joi.object({ + type: joi.string().allow("component").required(), + metadata: joi.object().unknown(true).required(), + hash: joi.string().optional(), + version: joi.string().optional(), + schema: joi + .object({ + name: joi.string().required(), + settings: joi.array().items(joi.object().unknown(true)).required(), + }) + .unknown(true), + }) + runJoi(validator, schema) +} + +function validateDatasource(schema) { + const fieldValidator = joi.object({ + type: joi + .string() + .allow(...Object.values(DatasourceFieldType)) + .required(), + required: joi.boolean().required(), + default: joi.any(), + display: joi.string(), + }) + + const queryValidator = joi + .object({ + type: joi.string().allow(...Object.values(QueryType)), + fields: joi.object().pattern(joi.string(), fieldValidator), + }) + .required() + + const validator = joi.object({ + type: joi.string().allow("datasource").required(), + metadata: joi.object().unknown(true).required(), + hash: joi.string().optional(), + version: joi.string().optional(), + schema: joi.object({ + docs: joi.string(), + friendlyName: joi.string().required(), + type: joi.string().allow(...DATASOURCE_TYPES), + description: joi.string().required(), + datasource: joi.object().pattern(joi.string(), fieldValidator).required(), + query: joi + .object({ + create: queryValidator, + read: queryValidator, + update: queryValidator, + delete: queryValidator, + }) + .unknown(true) + .required(), + }), + }) + runJoi(validator, schema) +} + +exports.validate = schema => { + switch (schema?.type) { + case PluginType.COMPONENT: + validateComponent(schema) + break + case PluginType.DATASOURCE: + validateDatasource(schema) + break + default: + throw new Error(`Unknown plugin type - check schema.json: ${schema.type}`) + } +} From 82615491312ed6b168aaf357df89eb8d1ce79214 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Fri, 9 Sep 2022 11:13:18 +0100 Subject: [PATCH 25/37] added plugin validation --- packages/server/src/api/controllers/plugin.ts | 6 +++++- packages/string-templates/yarn.lock | 16 ---------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index cb4d592812..3941798ac4 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -7,6 +7,7 @@ import { uploadedFilePlugin, } from "./plugin/utils" import { getGlobalDB } from "@budibase/backend-core/tenancy" +import { validate } from "@budibase/backend-core/plugins" import { generatePluginID, getPluginParams } from "../../db/utils" import { uploadDirectory, @@ -80,13 +81,16 @@ export async function create(ctx: any) { directory = directoryGithub break case "url": + const headersObj = JSON.parse(headers || null) || {} const { metadata: metadataUrl, directory: directoryUrl } = - await uploadedUrlPlugin(url, name, headers) + await uploadedUrlPlugin(url, name, headersObj) metadata = metadataUrl directory = directoryUrl break } + validate(metadata?.schema) + const doc = await storePlugin(metadata, directory, source) ctx.body = { diff --git a/packages/string-templates/yarn.lock b/packages/string-templates/yarn.lock index dc7e8f1852..8e71e59912 100644 --- a/packages/string-templates/yarn.lock +++ b/packages/string-templates/yarn.lock @@ -568,15 +568,6 @@ magic-string "^0.25.7" resolve "^1.17.0" -"@rollup/plugin-inject@^4.0.0": - version "4.0.4" - resolved "https://registry.yarnpkg.com/@rollup/plugin-inject/-/plugin-inject-4.0.4.tgz#fbeee66e9a700782c4f65c8b0edbafe58678fbc2" - integrity sha512-4pbcU4J/nS+zuHk+c+OL3WtmEQhqxlZ9uqfjQMQDOHOPld7PsCd8k5LWs8h5wjwJN7MgnAn768F2sDxEP4eNFQ== - dependencies: - "@rollup/pluginutils" "^3.1.0" - estree-walker "^2.0.1" - magic-string "^0.25.7" - "@rollup/plugin-json@^4.1.0": version "4.1.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-json/-/plugin-json-4.1.0.tgz#54e09867ae6963c593844d8bd7a9c718294496f3" @@ -3796,13 +3787,6 @@ rollup-plugin-node-resolve@^5.2.0: resolve "^1.11.1" rollup-pluginutils "^2.8.1" -rollup-plugin-polyfill-node@^0.10.2: - version "0.10.2" - resolved "https://registry.yarnpkg.com/rollup-plugin-polyfill-node/-/rollup-plugin-polyfill-node-0.10.2.tgz#b2128646851fcb9475ddfd5bc22ca1a8c568738d" - integrity sha512-5GMywXiLiuQP6ZzED/LO/Q0HyDi2W6b8VN+Zd3oB0opIjyRs494Me2ZMaqKWDNbGiW4jvvzl6L2n4zRgxS9cSQ== - dependencies: - "@rollup/plugin-inject" "^4.0.0" - rollup-plugin-terser@^7.0.2: version "7.0.2" resolved "https://registry.yarnpkg.com/rollup-plugin-terser/-/rollup-plugin-terser-7.0.2.tgz#e8fbba4869981b2dc35ae7e8a502d5c6c04d324d" From 829d1bf70d598f85fa3a0860fc6395f3905ab264 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Fri, 9 Sep 2022 11:25:17 +0100 Subject: [PATCH 26/37] cleaned plugin validate from cli --- packages/cli/src/plugins/constants.js | 6 -- packages/cli/src/plugins/index.js | 6 +- packages/cli/src/plugins/validate.js | 91 -------------------- packages/types/src/documents/plugin/index.ts | 2 + 4 files changed, 5 insertions(+), 100 deletions(-) delete mode 100644 packages/cli/src/plugins/constants.js delete mode 100644 packages/cli/src/plugins/validate.js diff --git a/packages/cli/src/plugins/constants.js b/packages/cli/src/plugins/constants.js deleted file mode 100644 index 37d0748979..0000000000 --- a/packages/cli/src/plugins/constants.js +++ /dev/null @@ -1,6 +0,0 @@ -exports.PluginTypes = { - COMPONENT: "component", - DATASOURCE: "datasource", -} - -exports.PLUGIN_TYPES_ARR = Object.values(exports.PluginTypes) diff --git a/packages/cli/src/plugins/index.js b/packages/cli/src/plugins/index.js index 643406bc4d..714187df56 100644 --- a/packages/cli/src/plugins/index.js +++ b/packages/cli/src/plugins/index.js @@ -3,8 +3,8 @@ const { CommandWords } = require("../constants") const { getSkeleton, fleshOutSkeleton } = require("./skeleton") const questions = require("../questions") const fs = require("fs") -const { PLUGIN_TYPES_ARR } = require("./constants") -const { validate } = require("./validate") +const { PLUGIN_TYPE_ARR } = require("@budibase/types") +const { validate } = require("@budibase/backend-core/plugins") const { runPkgCommand } = require("../exec") const { join } = require("path") const { success, error, info } = require("../utils") @@ -24,7 +24,7 @@ function checkInPlugin() { async function init(opts) { const type = opts["init"] || opts - if (!type || !PLUGIN_TYPES_ARR.includes(type)) { + if (!type || !PLUGIN_TYPE_ARR.includes(type)) { console.log( error( "Please provide a type to init, either 'component' or 'datasource'." diff --git a/packages/cli/src/plugins/validate.js b/packages/cli/src/plugins/validate.js deleted file mode 100644 index a6b4555cbd..0000000000 --- a/packages/cli/src/plugins/validate.js +++ /dev/null @@ -1,91 +0,0 @@ -const { PluginTypes } = require("./constants") -const { DatasourceFieldType, QueryType } = require("@budibase/types") -const joi = require("joi") - -const DATASOURCE_TYPES = [ - "Relational", - "Non-relational", - "Spreadsheet", - "Object store", - "Graph", - "API", -] - -function runJoi(validator, schema) { - const { error } = validator.validate(schema) - if (error) { - throw error - } -} - -function validateComponent(schema) { - const validator = joi.object({ - type: joi.string().allow("component").required(), - metadata: joi.object().unknown(true).required(), - hash: joi.string().optional(), - version: joi.string().optional(), - schema: joi - .object({ - name: joi.string().required(), - settings: joi.array().items(joi.object().unknown(true)).required(), - }) - .unknown(true), - }) - runJoi(validator, schema) -} - -function validateDatasource(schema) { - const fieldValidator = joi.object({ - type: joi - .string() - .allow(...Object.values(DatasourceFieldType)) - .required(), - required: joi.boolean().required(), - default: joi.any(), - display: joi.string(), - }) - - const queryValidator = joi - .object({ - type: joi.string().allow(...Object.values(QueryType)), - fields: joi.object().pattern(joi.string(), fieldValidator), - }) - .required() - - const validator = joi.object({ - type: joi.string().allow("datasource").required(), - metadata: joi.object().unknown(true).required(), - hash: joi.string().optional(), - version: joi.string().optional(), - schema: joi.object({ - docs: joi.string(), - friendlyName: joi.string().required(), - type: joi.string().allow(...DATASOURCE_TYPES), - description: joi.string().required(), - datasource: joi.object().pattern(joi.string(), fieldValidator).required(), - query: joi - .object({ - create: queryValidator, - read: queryValidator, - update: queryValidator, - delete: queryValidator, - }) - .unknown(true) - .required(), - }), - }) - runJoi(validator, schema) -} - -exports.validate = schema => { - switch (schema.type) { - case PluginTypes.COMPONENT: - validateComponent(schema) - break - case PluginTypes.DATASOURCE: - validateDatasource(schema) - break - default: - throw new Error(`Unknown plugin type - check schema.json: ${schema.type}`) - } -} diff --git a/packages/types/src/documents/plugin/index.ts b/packages/types/src/documents/plugin/index.ts index 8133e91523..2d7e480882 100644 --- a/packages/types/src/documents/plugin/index.ts +++ b/packages/types/src/documents/plugin/index.ts @@ -7,3 +7,5 @@ export interface FileType { path: string name: string } + +export const PLUGIN_TYPE_ARR = Object.values(exports.PluginType) \ No newline at end of file From 5f8a6c065bcc67d23aa64ea90b0e5ef863d6a65c Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Fri, 9 Sep 2022 11:26:20 +0100 Subject: [PATCH 27/37] added array to types --- packages/types/src/documents/plugin/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/documents/plugin/index.ts b/packages/types/src/documents/plugin/index.ts index 2d7e480882..f696d7a2aa 100644 --- a/packages/types/src/documents/plugin/index.ts +++ b/packages/types/src/documents/plugin/index.ts @@ -8,4 +8,4 @@ export interface FileType { name: string } -export const PLUGIN_TYPE_ARR = Object.values(exports.PluginType) \ No newline at end of file +export const PLUGIN_TYPE_ARR = Object.values(exports.PluginType) From 171970108e2aa5dae4530f16970dd94e11030deb Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Fri, 9 Sep 2022 12:01:04 +0100 Subject: [PATCH 28/37] fix build error exporting and declaring plugins --- packages/backend-core/src/index.ts | 2 ++ packages/backend-core/src/plugin/index.ts | 8 +++++++- packages/server/src/module.d.ts | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 74e79e7b95..d9dbe58264 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -18,6 +18,7 @@ import * as dbConstants from "./db/constants" import logging from "./logging" import pino from "./pino" import * as middleware from "./middleware" +import plugins from "./plugin" // mimic the outer package exports import * as db from "./pkg/db" @@ -56,6 +57,7 @@ const core = { errors, logging, roles, + plugins, ...pino, ...errorClasses, middleware, diff --git a/packages/backend-core/src/plugin/index.ts b/packages/backend-core/src/plugin/index.ts index 3eeaeaa90c..a6d1853007 100644 --- a/packages/backend-core/src/plugin/index.ts +++ b/packages/backend-core/src/plugin/index.ts @@ -1 +1,7 @@ -export * from "./utils" +import * as utils from "./utils" + +const pkg = { + ...utils, +} + +export = pkg diff --git a/packages/server/src/module.d.ts b/packages/server/src/module.d.ts index eb1d1feb48..ada45a3a66 100644 --- a/packages/server/src/module.d.ts +++ b/packages/server/src/module.d.ts @@ -11,3 +11,4 @@ declare module "@budibase/backend-core/encryption" declare module "@budibase/backend-core/utils" declare module "@budibase/backend-core/redis" declare module "@budibase/backend-core/objectStore" +declare module "@budibase/backend-core/plugins" From 945665448d3f85dc0ba956e8d75fba750471e18b Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Fri, 9 Sep 2022 12:15:38 +0100 Subject: [PATCH 29/37] fixing node exports --- packages/server/src/api/controllers/plugin/utils.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/plugin/utils.js b/packages/server/src/api/controllers/plugin/utils.js index f2a9fd6981..37d8ad6c3d 100644 --- a/packages/server/src/api/controllers/plugin/utils.js +++ b/packages/server/src/api/controllers/plugin/utils.js @@ -9,7 +9,7 @@ const { const { join } = require("path") const fetch = require("node-fetch") -export const uploadedFilePlugin = async file => { +exports.uploadedFilePlugin = async file => { if (!file.name.endsWith(".tar.gz")) { throw new Error("Plugin must be compressed into a gzipped tarball.") } @@ -19,7 +19,7 @@ export const uploadedFilePlugin = async file => { return await getPluginMetadata(path) } -export const uploadedNpmPlugin = async (url, name, headers = {}) => { +exports.uploadedNpmPlugin = async (url, name, headers = {}) => { let npmTarballUrl = url let pluginName = name @@ -66,7 +66,7 @@ export const uploadedNpmPlugin = async (url, name, headers = {}) => { return await getPluginMetadata(path) } -export const uploadedUrlPlugin = async (url, name = "", headers = {}) => { +exports.uploadedUrlPlugin = async (url, name = "", headers = {}) => { if (!url.includes(".tar.gz")) { throw new Error("Plugin must be compressed into a gzipped tarball.") } @@ -76,7 +76,7 @@ export const uploadedUrlPlugin = async (url, name = "", headers = {}) => { return await getPluginMetadata(path) } -export const uploadedGithubPlugin = async (ctx, url, name = "", token = "") => { +exports.uploadedGithubPlugin = async (ctx, url, name = "", token = "") => { let githubUrl = url if (!githubUrl.includes("https://github.com/")) { @@ -140,7 +140,7 @@ export const uploadedGithubPlugin = async (ctx, url, name = "", token = "") => { } } -export const downloadUnzipTarball = async (url, name, headers = {}) => { +const downloadUnzipTarball = async (url, name, headers = {}) => { try { const path = createTempFolder(name) From c89eb1994447b5268e362b318f258173b4e14ab5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 12 Sep 2022 16:19:28 +0100 Subject: [PATCH 30/37] Undo reverse logic when filtering by component type --- .../portal/manage/plugins/index.svelte | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte index 7e19a0cbbe..46740d13fa 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte @@ -16,21 +16,23 @@ let modal let searchTerm = "" - + let filter = "all" let filterOptions = [ { label: "All Plugins", value: "all" }, - { label: "Components", value: "datasource" }, - { label: "Datasources", value: "component" }, + { label: "Components", value: "component" }, + { label: "Datasources", value: "datasource" }, ] - let filter = "all" - $: filteredPlugins = - filter === "all" && searchTerm.length === 0 - ? $plugins - : $plugins - .filter(plugin => plugin.schema.type !== filter) - .filter(plugin => - plugin?.name?.toLowerCase().includes(searchTerm.toLowerCase()) - ) + + $: filteredPlugins = $plugins + .filter(plugin => { + return filter === "all" || plugin.schema.type === filter + }) + .filter(plugin => { + return ( + !searchTerm || + plugin?.name?.toLowerCase().includes(searchTerm.toLowerCase()) + ) + }) onMount(async () => { await plugins.load() From 0415a67ef045785032c498ce559b53d4e9553dc5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 12 Sep 2022 16:19:58 +0100 Subject: [PATCH 31/37] Fix capitalisation --- .../src/pages/builder/portal/manage/plugins/index.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte index 46740d13fa..a31cf71f03 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte @@ -18,7 +18,7 @@ let searchTerm = "" let filter = "all" let filterOptions = [ - { label: "All Plugins", value: "all" }, + { label: "All plugins", value: "all" }, { label: "Components", value: "component" }, { label: "Datasources", value: "datasource" }, ] From 722a087af5add3036b976bfd2edce2767ddaeb5e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 12 Sep 2022 16:21:47 +0100 Subject: [PATCH 32/37] Reverting organisation page changes. --- .../builder/portal/settings/organisation.svelte | 14 +++----------- .../api/controllers/{plugin.ts => plugin/index.ts} | 10 +++++----- packages/server/src/utilities/fileSystem/index.js | 10 ++++------ 3 files changed, 12 insertions(+), 22 deletions(-) rename packages/server/src/api/controllers/{plugin.ts => plugin/index.ts} (95%) diff --git a/packages/builder/src/pages/builder/portal/settings/organisation.svelte b/packages/builder/src/pages/builder/portal/settings/organisation.svelte index d52bd66d47..8c091ce952 100644 --- a/packages/builder/src/pages/builder/portal/settings/organisation.svelte +++ b/packages/builder/src/pages/builder/portal/settings/organisation.svelte @@ -15,7 +15,6 @@ import { API } from "api" import { writable } from "svelte/store" import { redirect } from "@roxi/routify" - import { onMount } from "svelte" // Only admins allowed here $: { @@ -34,12 +33,11 @@ }) let loading = false - async function uploadLogo() { + async function uploadLogo(file) { try { let data = new FormData() - data.append("file", $values.logo) - await API.uploadPlugin(data) - notifications.success("Plugin uploaded successfully") + data.append("file", file) + await API.uploadLogo(data) } catch (error) { notifications.error("Error uploading logo") } @@ -73,11 +71,6 @@ } loading = false } - - onMount(async () => { - const plugins = await API.getPlugins() - console.log(plugins) - }) {#if $auth.isAdmin} @@ -113,7 +106,6 @@ } }} /> -
diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin/index.ts similarity index 95% rename from packages/server/src/api/controllers/plugin.ts rename to packages/server/src/api/controllers/plugin/index.ts index 3941798ac4..de4aa2a50a 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin/index.ts @@ -1,20 +1,20 @@ -import { ObjectStoreBuckets } from "../../constants" -import { loadJSFile } from "../../utilities/fileSystem" +import { ObjectStoreBuckets } from "../../../constants" +import { loadJSFile } from "../../../utilities/fileSystem" import { uploadedNpmPlugin, uploadedUrlPlugin, uploadedGithubPlugin, uploadedFilePlugin, -} from "./plugin/utils" +} from "./utils" import { getGlobalDB } from "@budibase/backend-core/tenancy" import { validate } from "@budibase/backend-core/plugins" -import { generatePluginID, getPluginParams } from "../../db/utils" +import { generatePluginID, getPluginParams } from "../../../db/utils" import { uploadDirectory, deleteFolder, } from "@budibase/backend-core/objectStore" import { PluginType, FileType } from "@budibase/types" -import env from "../../environment" +import env from "../../../environment" export async function getPlugins(type?: PluginType) { const db = getGlobalDB() diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index ef9088dd3e..40cc456356 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -426,36 +426,34 @@ exports.getDatasourcePlugin = async (name, url, hash) => { /** * Find for a file recursively from start path applying filter, return first match */ -const findFileRec = (startPath, filter) => { +exports.findFileRec = (startPath, filter) => { if (!fs.existsSync(startPath)) { return } - var files = fs.readdirSync(startPath) + const files = fs.readdirSync(startPath) for (let i = 0, len = files.length; i < len; i++) { const filename = join(startPath, files[i]) const stat = fs.lstatSync(filename) if (stat.isDirectory()) { - return findFileRec(filename, filter) + return exports.findFileRec(filename, filter) } else if (filename.endsWith(filter)) { return filename } } } -exports.findFileRec = findFileRec /** * Remove a folder which is not empty from the file system */ -const deleteFolderFileSystem = path => { +exports.deleteFolderFileSystem = path => { if (!fs.existsSync(path)) { return } fs.rmSync(path, { recursive: true, force: true }) } -exports.deleteFolderFileSystem = deleteFolderFileSystem /** * Full function definition for below can be found in the utilities. From b31cc043edb44179c25de89da4525942e90eda9c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 12 Sep 2022 16:35:42 +0100 Subject: [PATCH 33/37] Update plugins UI styles --- .../plugins/_components/PluginRow.svelte | 21 ++++++---- .../portal/manage/plugins/index.svelte | 40 +++++++++---------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte index dd35ce752d..09a67c3583 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte @@ -21,7 +21,7 @@ : plugin.schema.schema.icon || "Beaker" -
+
detailsModal.show()}>
@@ -42,9 +42,8 @@
{plugin.schema.type.charAt(0).toUpperCase() + plugin.schema.type.slice(1)}
-
- +
@@ -98,13 +97,19 @@ From 27af24ec264b8fea506394f180f6c11d8d7aee86 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 12 Sep 2022 16:39:41 +0100 Subject: [PATCH 34/37] Revert organisation page --- .../portal/settings/organisation.svelte | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/organisation.svelte b/packages/builder/src/pages/builder/portal/settings/organisation.svelte index d52bd66d47..6d1f26ce36 100644 --- a/packages/builder/src/pages/builder/portal/settings/organisation.svelte +++ b/packages/builder/src/pages/builder/portal/settings/organisation.svelte @@ -15,7 +15,6 @@ import { API } from "api" import { writable } from "svelte/store" import { redirect } from "@roxi/routify" - import { onMount } from "svelte" // Only admins allowed here $: { @@ -34,12 +33,11 @@ }) let loading = false - async function uploadLogo() { + async function uploadLogo(file) { try { let data = new FormData() - data.append("file", $values.logo) - await API.uploadPlugin(data) - notifications.success("Plugin uploaded successfully") + data.append("file", file) + await API.uploadLogo(data) } catch (error) { notifications.error("Error uploading logo") } @@ -73,11 +71,6 @@ } loading = false } - - onMount(async () => { - const plugins = await API.getPlugins() - console.log(plugins) - }) {#if $auth.isAdmin} @@ -95,14 +88,14 @@ Information Here you can update your logo and organization name. -
-
+
+
-