From 0e2a86460ce37ea331a3f3e519767a3e0714e788 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Wed, 31 Aug 2022 16:09:47 +0100 Subject: [PATCH] uploading npm and url plugins --- .../plugins/_components/AddPluginModal.svelte | 16 ++---- packages/builder/src/stores/portal/plugins.js | 11 ++--- packages/server/src/api/controllers/plugin.ts | 45 ++++++++--------- .../server/src/utilities/fileSystem/index.js | 49 +++++++++++-------- 4 files changed, 60 insertions(+), 61 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte index e45655d20d..1fce6746ad 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte @@ -11,10 +11,10 @@ import { plugins } from "stores/portal" let authOptions = { - NPM: ["NPM Token", "URL"], + NPM: ["URL"], Github: ["Github Token", "URL"], - URL: ["Header", "URL"], - File: ["Path", "Header"], + URL: ["Headers", "URL"], + File: ["Path", "Headers"], Upload: ["Upload"], } let file @@ -50,17 +50,11 @@ source, nameValue, url, - dynamicValues["Header"] + dynamicValues["Headers"] ) break case "npm": - await plugins.createPlugin( - typeValue, - source, - nameValue, - url, - dynamicValues["NPM Token"] - ) + await plugins.createPlugin(typeValue, source, nameValue, url) break } } diff --git a/packages/builder/src/stores/portal/plugins.js b/packages/builder/src/stores/portal/plugins.js index 531ac1c396..4e3d952fe3 100644 --- a/packages/builder/src/stores/portal/plugins.js +++ b/packages/builder/src/stores/portal/plugins.js @@ -17,7 +17,7 @@ export function createPluginsStore() { }) } - async function createPlugin(type, source, name, url, auth) { + async function createPlugin(type, source, name, url, auth = null) { let pluginData = { type, source, @@ -26,19 +26,16 @@ export function createPluginsStore() { } switch (source) { - case "github": - pluginData.githubToken = auth - break case "url": - pluginData.header = auth + pluginData.headers = auth break case "npm": pluginData.npmToken = auth break } - let resp = await API.createPlugin(pluginData) - console.log(resp) + let res = await API.createPlugin(pluginData) + console.log("RESP", res) // TODO_RIC // let newPlugin = resp.plugins[0] // update(state => { diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index b6d3d31354..dc70ea8b62 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -1,8 +1,8 @@ import { ObjectStoreBuckets } from "../../constants" import { extractPluginTarball, - npmPlugin, - getPluginMetadata, + createNpmPlugin, + createUrlPlugin, } from "../../utilities/fileSystem" import { getGlobalDB } from "@budibase/backend-core/tenancy" import { generatePluginID, getPluginParams } from "../../db/utils" @@ -49,39 +49,40 @@ export async function upload(ctx: any) { } export async function create(ctx: any) { - const { type, source, name, url, header, githubToken, npmToken } = - ctx.request.body + const { type, source, name, url, headers, githubToken } = ctx.request.body let metadata let directory switch (source) { case "npm": - // const { metadata: metadataNpm, directory: directoryNpm } = await npmPlugin(url, name) - // metadata = metadataNpm - // directory = directoryNpm - - console.log(22222, await getPluginMetadata(await npmPlugin(url, name))) + const { metadata: metadataNpm, directory: directoryNpm } = + await createNpmPlugin(url, name) + metadata = metadataNpm + directory = directoryNpm break case "github": console.log("github") break case "url": - console.log("url") + const { metadata: metadataUrl, directory: directoryUrl } = + await createUrlPlugin(url, name, headers) + metadata = metadataUrl + directory = directoryUrl break } - // try { - // const doc = storePlugin(metadata, directory, source) - // - // ctx.body = { - // message: "Plugin uploaded successfully", - // plugins: doc, - // } - // } catch (err: any) { - // const errMsg = err?.message ? err?.message : err - // - // ctx.throw(400, `Failed to import plugin: ${errMsg}`) - // } + try { + const doc = storePlugin(metadata, directory, source) + + ctx.body = { + message: "Plugin uploaded successfully", + plugins: doc, + } + } catch (err: any) { + const errMsg = err?.message ? err?.message : err + + ctx.throw(400, `Failed to import plugin: ${errMsg}`) + } ctx.status = 200 } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 9c70edf8a6..710e3c0294 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -1,9 +1,8 @@ const { budibaseTempDir } = require("../budibaseDir") const fs = require("fs") const { join } = require("path") -// const { promisify } = require("util") -// const exec = promisify(require("child_process").exec) -// const streamPipeline = promisify(require("stream")) +const { promisify } = require("util") +const streamPipeline = promisify(require("stream").pipeline) const uuid = require("uuid/v4") const { doWithDB, @@ -32,7 +31,6 @@ const MemoryStream = require("memorystream") const { getAppId } = require("@budibase/backend-core/context") const tar = require("tar") const fetch = require("node-fetch") -// const fileType = require("file-type") const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules") @@ -349,10 +347,9 @@ const extractPluginTarball = async (file, ext = ".tar.gz") => { } exports.extractPluginTarball = extractPluginTarball -exports.npmPlugin = async (url, name = "") => { +exports.createNpmPlugin = async (url, name = "") => { let npmTarball = url - let filename = name - let path = join(budibaseTempDir(), name) + let pluginName = name if (!npmTarball.includes(".tgz")) { const npmPackageURl = url.replace( @@ -362,8 +359,7 @@ exports.npmPlugin = async (url, name = "") => { const response = await fetch(npmPackageURl) if (response.status === 200) { let npmDetails = await response.json() - filename = npmDetails.name - path = join(budibaseTempDir(), filename) + pluginName = npmDetails.name const npmVersion = npmDetails["dist-tags"].latest npmTarball = npmDetails.versions[npmVersion].dist.tarball } else { @@ -371,36 +367,47 @@ exports.npmPlugin = async (url, name = "") => { } } + return await downloadUnzipPlugin(pluginName, npmTarball) +} + +exports.createUrlPlugin = async (url, name = "", headers = {}) => { + if (!url.includes(".tgz") && !url.includes(".tar.gz")) { + throw new Error("Plugin must be compressed into a gzipped tarball.") + } + + return await downloadUnzipPlugin(name, url, headers) +} + +const downloadUnzipPlugin = async (name, url, headers = {}) => { + console.log(name, url, headers) + const path = join(budibaseTempDir(), name) try { + // Remove first if exists if (fs.existsSync(path)) { fs.rmSync(path, { recursive: true, force: true }) } fs.mkdirSync(path) - const response = await fetch(npmTarball) + const response = await fetch(url, { headers }) if (!response.ok) throw new Error(`Loading NPM plugin failed ${response.statusText}`) - // const dest = fs.createWriteStream(`${path}/${filename}.tgz`) - await response.body.pipe( - await tar.x({ + await streamPipeline( + response.body, + tar.x({ strip: 1, C: path, }) ) - - // const readStream = fs.createReadStream(`${path}/${filename}.tgz`) - // readStream.pipe( - - // ) + return await getPluginMetadata(path) } catch (e) { - throw `Cannot store package locally: ${e.message}` + throw `Cannot store plugin locally: ${e.message}` } - - return path } +exports.downloadUnzipPlugin = downloadUnzipPlugin const getPluginMetadata = async path => { + console.log(path) let metadata = {} try { const pkg = fs.readFileSync(join(path, "package.json"), "utf8")