uploading npm and url plugins

packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte

@@ -11,10 +11,10 @@
   import { plugins } from "stores/portal"
 
   let authOptions = {
-    NPM: ["NPM Token", "URL"],
+    NPM: ["URL"],
     Github: ["Github Token", "URL"],
-    URL: ["Header", "URL"],
-    File: ["Path", "Header"],
+    URL: ["Headers", "URL"],
+    File: ["Path", "Headers"],
     Upload: ["Upload"],
   }
   let file
@@ -50,17 +50,11 @@
           source,
           nameValue,
           url,
-          dynamicValues["Header"]
+          dynamicValues["Headers"]
         )
         break
       case "npm":
-        await plugins.createPlugin(
-          typeValue,
-          source,
-          nameValue,
-          url,
-          dynamicValues["NPM Token"]
-        )
+        await plugins.createPlugin(typeValue, source, nameValue, url)
         break
     }
   }

packages/builder/src/stores/portal/plugins.js

@@ -17,7 +17,7 @@ export function createPluginsStore() {
     })
   }
 
-  async function createPlugin(type, source, name, url, auth) {
+  async function createPlugin(type, source, name, url, auth = null) {
     let pluginData = {
       type,
       source,
@@ -26,19 +26,16 @@ export function createPluginsStore() {
     }
 
     switch (source) {
-      case "github":
-        pluginData.githubToken = auth
-        break
       case "url":
-        pluginData.header = auth
+        pluginData.headers = auth
         break
       case "npm":
         pluginData.npmToken = auth
         break
     }
 
-    let resp = await API.createPlugin(pluginData)
-    console.log(resp)
+    let res = await API.createPlugin(pluginData)
+    console.log("RESP", res)
     // TODO_RIC
     // let newPlugin = resp.plugins[0]
     // update(state => {

packages/server/src/api/controllers/plugin.ts

@@ -1,8 +1,8 @@
 import { ObjectStoreBuckets } from "../../constants"
 import {
   extractPluginTarball,
-  npmPlugin,
-  getPluginMetadata,
+  createNpmPlugin,
+  createUrlPlugin,
 } from "../../utilities/fileSystem"
 import { getGlobalDB } from "@budibase/backend-core/tenancy"
 import { generatePluginID, getPluginParams } from "../../db/utils"
@@ -49,39 +49,40 @@ export async function upload(ctx: any) {
 }
 
 export async function create(ctx: any) {
-  const { type, source, name, url, header, githubToken, npmToken } =
-    ctx.request.body
+  const { type, source, name, url, headers, githubToken } = ctx.request.body
   let metadata
   let directory
 
   switch (source) {
     case "npm":
-      // const { metadata: metadataNpm, directory: directoryNpm } = await npmPlugin(url, name)
-      // metadata = metadataNpm
-      // directory = directoryNpm
-
-      console.log(22222, await getPluginMetadata(await npmPlugin(url, name)))
+      const { metadata: metadataNpm, directory: directoryNpm } =
+        await createNpmPlugin(url, name)
+      metadata = metadataNpm
+      directory = directoryNpm
       break
     case "github":
       console.log("github")
       break
     case "url":
-      console.log("url")
+      const { metadata: metadataUrl, directory: directoryUrl } =
+        await createUrlPlugin(url, name, headers)
+      metadata = metadataUrl
+      directory = directoryUrl
       break
   }
 
-  // try {
-  //   const doc = storePlugin(metadata, directory, source)
-  //
-  //   ctx.body = {
-  //     message: "Plugin uploaded successfully",
-  //     plugins: doc,
-  //   }
-  // } catch (err: any) {
-  //   const errMsg = err?.message ? err?.message : err
-  //
-  //   ctx.throw(400, `Failed to import plugin: ${errMsg}`)
-  // }
+  try {
+    const doc = storePlugin(metadata, directory, source)
+
+    ctx.body = {
+      message: "Plugin uploaded successfully",
+      plugins: doc,
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
   ctx.status = 200
 }
 

packages/server/src/utilities/fileSystem/index.js

@@ -1,9 +1,8 @@
 const { budibaseTempDir } = require("../budibaseDir")
 const fs = require("fs")
 const { join } = require("path")
-// const { promisify } = require("util")
-// const exec = promisify(require("child_process").exec)
-// const streamPipeline = promisify(require("stream"))
+const { promisify } = require("util")
+const streamPipeline = promisify(require("stream").pipeline)
 const uuid = require("uuid/v4")
 const {
   doWithDB,
@@ -32,7 +31,6 @@ const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
 const tar = require("tar")
 const fetch = require("node-fetch")
-// const fileType = require("file-type")
 
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
@@ -349,10 +347,9 @@ const extractPluginTarball = async (file, ext = ".tar.gz") => {
 }
 exports.extractPluginTarball = extractPluginTarball
 
-exports.npmPlugin = async (url, name = "") => {
+exports.createNpmPlugin = async (url, name = "") => {
   let npmTarball = url
-  let filename = name
-  let path = join(budibaseTempDir(), name)
+  let pluginName = name
 
   if (!npmTarball.includes(".tgz")) {
     const npmPackageURl = url.replace(
@@ -362,8 +359,7 @@ exports.npmPlugin = async (url, name = "") => {
     const response = await fetch(npmPackageURl)
     if (response.status === 200) {
       let npmDetails = await response.json()
-      filename = npmDetails.name
-      path = join(budibaseTempDir(), filename)
+      pluginName = npmDetails.name
       const npmVersion = npmDetails["dist-tags"].latest
       npmTarball = npmDetails.versions[npmVersion].dist.tarball
     } else {
@@ -371,36 +367,47 @@ exports.npmPlugin = async (url, name = "") => {
     }
   }
 
+  return await downloadUnzipPlugin(pluginName, npmTarball)
+}
+
+exports.createUrlPlugin = async (url, name = "", headers = {}) => {
+  if (!url.includes(".tgz") && !url.includes(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+
+  return await downloadUnzipPlugin(name, url, headers)
+}
+
+const downloadUnzipPlugin = async (name, url, headers = {}) => {
+  console.log(name, url, headers)
+  const path = join(budibaseTempDir(), name)
   try {
+    // Remove first if exists
     if (fs.existsSync(path)) {
       fs.rmSync(path, { recursive: true, force: true })
     }
     fs.mkdirSync(path)
 
-    const response = await fetch(npmTarball)
+    const response = await fetch(url, { headers })
     if (!response.ok)
       throw new Error(`Loading NPM plugin failed ${response.statusText}`)
 
-    // const dest = fs.createWriteStream(`${path}/${filename}.tgz`)
-    await response.body.pipe(
-      await tar.x({
+    await streamPipeline(
+      response.body,
+      tar.x({
         strip: 1,
         C: path,
       })
     )
-
-    // const readStream = fs.createReadStream(`${path}/${filename}.tgz`)
-    // readStream.pipe(
-
-    // )
+    return await getPluginMetadata(path)
   } catch (e) {
-    throw `Cannot store package locally: ${e.message}`
+    throw `Cannot store plugin locally: ${e.message}`
   }
-
-  return path
 }
+exports.downloadUnzipPlugin = downloadUnzipPlugin
 
 const getPluginMetadata = async path => {
+  console.log(path)
   let metadata = {}
   try {
     const pkg = fs.readFileSync(join(path, "package.json"), "utf8")