From 2aaff7576bcf201ea522fb96054d4ebfb603969a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Aug 2022 10:05:13 +0100 Subject: [PATCH 1/3] Adding full flow for creating custom datasource. --- .../modals/CreateDatasourceModal.svelte | 3 +++ .../modals/DatasourceConfigModal.svelte | 5 ++++- packages/server/src/integrations/index.ts | 16 ++++++++++++-- .../server/src/utilities/fileSystem/index.js | 22 +++++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte index 2d5e6976bf..33953493f6 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte @@ -49,6 +49,9 @@ schema: selected.datasource, auth: selected.auth, } + if (selected.friendlyName) { + integration.name = selected.friendlyName + } checkShowImport() } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 0aaa2b201d..ecfd838b16 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -14,6 +14,9 @@ let datasource = cloneDeep(integration) let skipFetch = false + $: name = + IntegrationNames[datasource.type] || datasource.name || datasource.type + async function saveDatasource() { try { const resp = await save(datasource, skipFetch) @@ -32,7 +35,7 @@ saveDatasource()} onCancel={() => modal.show()} confirmText={datasource.plus diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index bf267e46cc..b9523061a4 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -15,6 +15,7 @@ import redis from "./redis" import snowflake from "./snowflake" import { getPlugins } from "../api/controllers/plugin" import { SourceName, Integration, PluginType } from "@budibase/types" +import { getDatasourcePlugin } from "../utilities/fileSystem" const environment = require("../environment") const { cloneDeep } = require("lodash") @@ -65,6 +66,8 @@ if (environment.SELF_HOSTED) { DEFINITIONS[SourceName.GOOGLE_SHEETS] = googlesheets.schema } +function isIntegrationAvailable(integration: string) {} + module.exports = { getDefinitions: async () => { const plugins = await getPlugins(PluginType.DATASOURCE) @@ -82,7 +85,16 @@ module.exports = { ...pluginSchemas, } }, - getIntegration: async () => { - return INTEGRATIONS + getIntegration: async (integration: string) => { + if (INTEGRATIONS[integration]) { + return INTEGRATIONS[integration] + } + const plugins = await getPlugins(PluginType.DATASOURCE) + for (let plugin of plugins) { + if (plugin.name === integration) { + // need to use commonJS require due to its dynamic runtime nature + return getDatasourcePlugin(plugin.name, plugin.jsUrl) + } + } }, } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 94c5a92375..9b124d6c29 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -26,9 +26,11 @@ const { const MemoryStream = require("memorystream") const { getAppId } = require("@budibase/backend-core/context") const tar = require("tar") +const fetch = require("node-fetch") const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules") +const DATASOURCE_PATH = join(budibaseTempDir(), "datasource") /** * The single stack system (Cloud and Builder) should not make use of the file system where possible, @@ -348,6 +350,26 @@ exports.extractPluginTarball = async file => { return { metadata, directory: path } } +exports.getDatasourcePlugin = async (name, url) => { + if (!fs.existsSync(DATASOURCE_PATH)) { + fs.mkdirSync(DATASOURCE_PATH) + } + const filename = join(DATASOURCE_PATH, name) + if (fs.existsSync(filename)) { + return require(filename) + } + const response = fetch(url) + if (response.status === 200) { + const content = await response.text() + fs.writeFileSync(filename, content) + require(filename) + } else { + throw new Error( + `Unable to retrieve plugin - reason: ${await response.text()}` + ) + } +} + /** * Full function definition for below can be found in the utilities. */ From ee4f72126ad2f3b96b83af4dea6234040d818b09 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Aug 2022 10:46:17 +0100 Subject: [PATCH 2/3] Updating custom datasource SVG - splitting out in UI. --- .../_components/DatasourceCard.svelte | 73 +++++++++++++++++++ .../DatasourceNavigator/icons/Custom.svelte | 57 ++++++++------- .../modals/CreateDatasourceModal.svelte | 45 +++++------- 3 files changed, 122 insertions(+), 53 deletions(-) create mode 100644 packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte new file mode 100644 index 0000000000..2dac2fec79 --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte @@ -0,0 +1,73 @@ + + +
dispatcher("selected", integrationType)} + class="item hoverable" +> +
+ +
+ {schema.friendlyName} + {#if schema.type} + {schema.type || ""} + {/if} +
+
+
+ + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/Custom.svelte b/packages/builder/src/components/backend/DatasourceNavigator/icons/Custom.svelte index 354b7a3358..0d4b7219aa 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/Custom.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/Custom.svelte @@ -1,8 +1,6 @@ - - - - - - + + + + + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte index 33953493f6..945cd574d8 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte @@ -18,6 +18,7 @@ import { createRestDatasource } from "builderStore/datasource" import { goto } from "@roxi/routify" import ImportRestQueriesModal from "./ImportRestQueriesModal.svelte" + import DatasourceCard from "../_components/DatasourceCard.svelte" export let modal let integrations = {} @@ -95,14 +96,6 @@ } integrations = newIntegrations } - - function getIcon(integrationType, schema) { - if (schema.custom) { - return ICONS.CUSTOM - } else { - return ICONS[integrationType] - } - } @@ -161,28 +154,24 @@ Connect to an external data source
- {#each Object.entries(integrations).filter(([key]) => key !== IntegrationTypes.INTERNAL) as [integrationType, schema]} -
selectIntegration(integrationType)} - class="item hoverable" - > -
- -
- {schema.friendlyName} - {#if schema.type} - {schema.type || ""} - {/if} -
-
-
+ {#each Object.entries(integrations).filter(([key, val]) => key !== IntegrationTypes.INTERNAL && !val.custom) as [integrationType, schema]} + selectIntegration(evt.detail)} + {schema} + bind:integrationType + {integration} + /> {/each}
+ Custom data source + {#each Object.entries(integrations).filter(entry => entry[1].custom) as [integrationType, schema]} + selectIntegration(evt.detail)} + {schema} + bind:integrationType + {integration} + /> + {/each} From b6a5ccbf1001b04215400a0e5581b0bd3f6d2cf3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Aug 2022 11:01:25 +0100 Subject: [PATCH 3/3] Adding some controls around datasource config creation and updating minio fetch. --- .../DatasourceNavigator/modals/DatasourceConfigModal.svelte | 3 +++ packages/server/src/utilities/fileSystem/index.js | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index ecfd838b16..c8a5bc96eb 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -19,6 +19,9 @@ async function saveDatasource() { try { + if (!datasource.name) { + datasource.name = name + } const resp = await save(datasource, skipFetch) $goto(`./datasource/${resp._id}`) notifications.success(`Datasource updated successfully.`) diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 9b124d6c29..d65ee48f05 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -17,6 +17,7 @@ const { downloadTarball, } = require("./utilities") const { updateClientLibrary } = require("./clientLibrary") +const { checkSlashesInUrl } = require("../") const env = require("../../environment") const { USER_METDATA_PREFIX, @@ -358,7 +359,7 @@ exports.getDatasourcePlugin = async (name, url) => { if (fs.existsSync(filename)) { return require(filename) } - const response = fetch(url) + const response = await fetch(checkSlashesInUrl(`${env.MINIO_URL}/${url}`)) if (response.status === 200) { const content = await response.text() fs.writeFileSync(filename, content)