Merge branch 'plugins-dev-experience' of github.com:Budibase/budibase into plugins-dev-experience

packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte

@@ -0,0 +1,73 @@
+<script>
+  import { createEventDispatcher } from "svelte"
+  import { Heading, Detail } from "@budibase/bbui"
+  import ICONS from "../icons"
+
+  export let integration
+  export let integrationType
+  export let schema
+
+  let dispatcher = createEventDispatcher()
+
+  function getIcon(integrationType, schema) {
+    if (schema.custom) {
+      return ICONS.CUSTOM
+    } else {
+      return ICONS[integrationType]
+    }
+  }
+</script>
+
+<div
+  class:selected={integration.type === integrationType}
+  on:click={() => dispatcher("selected", integrationType)}
+  class="item hoverable"
+>
+  <div class="item-body" class:with-type={!!schema.type}>
+    <svelte:component
+      this={getIcon(integrationType, schema)}
+      height="20"
+      width="20"
+    />
+    <div class="text">
+      <Heading size="XXS">{schema.friendlyName}</Heading>
+      {#if schema.type}
+        <Detail size="S">{schema.type || ""}</Detail>
+      {/if}
+    </div>
+  </div>
+</div>
+
+<style>
+  .item {
+    cursor: pointer;
+    display: grid;
+    grid-gap: var(--spectrum-alias-grid-margin-xsmall);
+    padding: var(--spectrum-alias-item-padding-s)
+      var(--spectrum-alias-item-padding-m);
+    background: var(--spectrum-alias-background-color-secondary);
+    transition: background 0.13s ease-out;
+    border: solid var(--spectrum-alias-border-color);
+    border-radius: 5px;
+    box-sizing: border-box;
+    border-width: 2px;
+  }
+  .item:hover,
+  .item.selected {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+
+  .item-body {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-m);
+  }
+  .item-body.with-type {
+    align-items: flex-start;
+  }
+  .item-body.with-type :global(svg) {
+    margin-top: 4px;
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/icons/Custom.svelte

@@ -1,8 +1,6 @@
 <script>
   export let width = "100"
   export let height = "100"
-  let color =
-    "var(--spectrum-heading-xxs-text-color, var(--spectrum-alias-heading-text-color))"
 </script>
 
 <svg
@@ -11,29 +9,38 @@
   xmlns="http://www.w3.org/2000/svg"
   x="0px"
   y="0px"
-  viewBox="0 0 230.795 230.795"
+  viewBox="0 0 18.43 17.62"
-  style="enable-background:new 0 0 230.795 230.795;"
+  style="enable-background:new 0 0 18.43 17.62;"
   xml:space="preserve"
 >
-  <g>
+  <style type="text/css">
+    .st0 {
+      fill: var(
+        --spectrum-heading-xxs-text-color,
+        var(--spectrum-alias-heading-text-color)
+      );
+    }
+  </style>
+  <ellipse id="Ellipse_9514" class="st0" cx="9.14" cy="2.88" rx="8" ry="2.5" />
   <path
-      d="M60.357,63.289c-2.929-2.929-7.678-2.93-10.606-0.001L2.197,110.836C0.79,112.243,0,114.151,0,116.14
+    id="Path_8355"
-		c0,1.989,0.79,3.896,2.196,5.303l47.348,47.35c1.465,1.465,3.384,2.197,5.304,2.197c1.919,0,3.839-0.732,5.303-2.196
+    class="st0"
-		c2.93-2.929,2.93-7.678,0.001-10.606L18.107,116.14l42.25-42.245C63.286,70.966,63.286,66.217,60.357,63.289z"
+    d="M7.64,12.88c0-0.48,0.06-0.95,0.17-1.41c-2.53-0.16-5.92-0.76-6.67-1.95v4.36
-      fill={color}
+	c0,1.34,3.39,2.43,7.63,2.49C8.03,15.36,7.63,14.13,7.64,12.88z"
   />
   <path
-      d="M228.598,110.639l-47.355-47.352c-2.928-2.928-7.677-2.929-10.606,0.001c-2.929,2.929-2.929,7.678,0.001,10.607
+    id="Path_8356"
-		l42.051,42.048l-42.249,42.243c-2.93,2.929-2.93,7.678-0.001,10.606c1.465,1.465,3.384,2.197,5.304,2.197
+    class="st0"
-		c1.919,0,3.839-0.732,5.303-2.196l47.554-47.547c1.407-1.406,2.197-3.314,2.197-5.304
+    d="M13.64,6.88c1.25,0,2.47,0.39,3.48,1.12c0.01-0.04,0.02-0.08,0.02-0.12V4.51
-		C230.795,113.954,230.005,112.046,228.598,110.639z"
+	c-1.22,1.55-5.53,2-8,2s-7.11-0.58-8-2v3.36c0,1.28,3.09,2.33,7.06,2.48C9.18,8.24,11.3,6.88,13.64,6.88z"
-      fill={color}
   />
   <path
-      d="M155.889,61.302c-3.314-2.484-8.017-1.806-10.498,1.51l-71.994,96.184c-2.482,3.316-1.807,8.017,1.51,10.498
+    id="Path_8353"
-		c1.348,1.01,2.925,1.496,4.488,1.496c2.282,0,4.537-1.038,6.01-3.006L157.398,71.8C159.881,68.484,159.205,63.784,155.889,61.302z"
+    class="st0"
-      fill={color}
+    d="M10.32,14.42c-0.56-0.56-0.56-1.47,0-2.03l1.24-1.24l-0.01-0.01c-0.25-0.25-0.25-0.65,0-0.9l0,0
+	L12,9.79c0.25-0.25,0.65-0.25,0.9,0c0,0,0,0,0,0l0,0l0.45,0.45l1.53-1.53c0.09-0.09,0.24-0.09,0.34,0l0.34,0.34
+	c0.09,0.09,0.09,0.24,0,0.34l-1.53,1.53l1.35,1.35l1.53-1.53c0.09-0.09,0.24-0.09,0.34,0l0.34,0.34c0.09,0.09,0.09,0.24,0,0.34
+	l-1.53,1.53l0.45,0.45c0.25,0.25,0.25,0.65,0,0.9l0,0l-0.45,0.45c-0.25,0.25-0.65,0.25-0.9,0l0,0l-0.01-0.01l-1.24,1.24
+	c-0.56,0.56-1.47,0.56-2.03,0l0,0L10.32,14.42z"
   />
-  </g>
-  <g />
 </svg>

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -18,6 +18,7 @@
   import { createRestDatasource } from "builderStore/datasource"
   import { goto } from "@roxi/routify"
   import ImportRestQueriesModal from "./ImportRestQueriesModal.svelte"
+  import DatasourceCard from "../_components/DatasourceCard.svelte"
 
   export let modal
   let integrations = {}
@@ -49,6 +50,9 @@
       schema: selected.datasource,
       auth: selected.auth,
     }
+    if (selected.friendlyName) {
+      integration.name = selected.friendlyName
+    }
     checkShowImport()
   }
 
@@ -92,14 +96,6 @@
     }
     integrations = newIntegrations
   }
-
-  function getIcon(integrationType, schema) {
-    if (schema.custom) {
-      return ICONS.CUSTOM
-    } else {
-      return ICONS[integrationType]
-    }
-  }
 </script>
 
 <Modal bind:this={internalTableModal}>
@@ -158,28 +154,24 @@
     <Layout noPadding gap="XS">
       <Body size="S">Connect to an external data source</Body>
       <div class="item-list">
-        {#each Object.entries(integrations).filter(([key]) => key !== IntegrationTypes.INTERNAL) as [integrationType, schema]}
+        {#each Object.entries(integrations).filter(([key, val]) => key !== IntegrationTypes.INTERNAL && !val.custom) as [integrationType, schema]}
-          <div
+          <DatasourceCard
-            class:selected={integration.type === integrationType}
+            on:selected={evt => selectIntegration(evt.detail)}
-            on:click={() => selectIntegration(integrationType)}
+            {schema}
-            class="item hoverable"
+            bind:integrationType
-          >
+            {integration}
-            <div class="item-body" class:with-type={!!schema.type}>
-              <svelte:component
-                this={getIcon(integrationType, schema)}
-                height="20"
-                width="20"
           />
-              <div class="text">
-                <Heading size="XXS">{schema.friendlyName}</Heading>
-                {#if schema.type}
-                  <Detail size="S">{schema.type || ""}</Detail>
-                {/if}
-              </div>
-            </div>
-          </div>
         {/each}
       </div>
+      <Body size="S">Custom data source</Body>
+      {#each Object.entries(integrations).filter(entry => entry[1].custom) as [integrationType, schema]}
+        <DatasourceCard
+          on:selected={evt => selectIntegration(evt.detail)}
+          {schema}
+          bind:integrationType
+          {integration}
+        />
+      {/each}
     </Layout>
   </ModalContent>
 </Modal>

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -14,8 +14,14 @@
   let datasource = cloneDeep(integration)
   let skipFetch = false
 
+  $: name =
+    IntegrationNames[datasource.type] || datasource.name || datasource.type
+
   async function saveDatasource() {
     try {
+      if (!datasource.name) {
+        datasource.name = name
+      }
       const resp = await save(datasource, skipFetch)
       $goto(`./datasource/${resp._id}`)
       notifications.success(`Datasource updated successfully.`)
@@ -32,7 +38,7 @@
 </script>
 
 <ModalContent
-  title={`Connect to ${IntegrationNames[datasource.type]}`}
+  title={`Connect to ${name}`}
   onConfirm={() => saveDatasource()}
   onCancel={() => modal.show()}
   confirmText={datasource.plus

packages/server/src/integrations/index.ts

@@ -15,6 +15,7 @@ import redis from "./redis"
 import snowflake from "./snowflake"
 import { getPlugins } from "../api/controllers/plugin"
 import { SourceName, Integration, PluginType } from "@budibase/types"
+import { getDatasourcePlugin } from "../utilities/fileSystem"
 const environment = require("../environment")
 const { cloneDeep } = require("lodash")
 
@@ -65,6 +66,8 @@ if (environment.SELF_HOSTED) {
   DEFINITIONS[SourceName.GOOGLE_SHEETS] = googlesheets.schema
 }
 
+function isIntegrationAvailable(integration: string) {}
+
 module.exports = {
   getDefinitions: async () => {
     const plugins = await getPlugins(PluginType.DATASOURCE)
@@ -82,7 +85,16 @@ module.exports = {
       ...pluginSchemas,
     }
   },
-  getIntegration: async () => {
+  getIntegration: async (integration: string) => {
-    return INTEGRATIONS
+    if (INTEGRATIONS[integration]) {
+      return INTEGRATIONS[integration]
+    }
+    const plugins = await getPlugins(PluginType.DATASOURCE)
+    for (let plugin of plugins) {
+      if (plugin.name === integration) {
+        // need to use commonJS require due to its dynamic runtime nature
+        return getDatasourcePlugin(plugin.name, plugin.jsUrl)
+      }
+    }
   },
 }

packages/server/src/utilities/fileSystem/index.js

@@ -17,6 +17,7 @@ const {
   downloadTarball,
 } = require("./utilities")
 const { updateClientLibrary } = require("./clientLibrary")
+const { checkSlashesInUrl } = require("../")
 const env = require("../../environment")
 const {
   USER_METDATA_PREFIX,
@@ -26,9 +27,11 @@ const {
 const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
 const tar = require("tar")
+const fetch = require("node-fetch")
 
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
 
 /**
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
@@ -348,6 +351,26 @@ exports.extractPluginTarball = async file => {
   return { metadata, directory: path }
 }
 
+exports.getDatasourcePlugin = async (name, url) => {
+  if (!fs.existsSync(DATASOURCE_PATH)) {
+    fs.mkdirSync(DATASOURCE_PATH)
+  }
+  const filename = join(DATASOURCE_PATH, name)
+  if (fs.existsSync(filename)) {
+    return require(filename)
+  }
+  const response = await fetch(checkSlashesInUrl(`${env.MINIO_URL}/${url}`))
+  if (response.status === 200) {
+    const content = await response.text()
+    fs.writeFileSync(filename, content)
+    require(filename)
+  } else {
+    throw new Error(
+      `Unable to retrieve plugin - reason: ${await response.text()}`
+    )
+  }
+}
+
 /**
  * Full function definition for below can be found in the utilities.
  */