From b659060beb4246ad943b4660fb06f7a823d12a3d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Jan 2023 19:16:19 +0000 Subject: [PATCH] Updating datasource API to remove secrets when returning datasources. --- .../portal/settings/environment/index.svelte | 2 +- .../builder/src/stores/backend/datasources.js | 4 +- .../server/src/api/controllers/datasource.ts | 52 +++++++++++-------- .../src/sdk/app/datasources/datasources.ts | 37 ++++++++++++- packages/types/src/sdk/datasources.ts | 13 ++++- 5 files changed, 81 insertions(+), 27 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/environment/index.svelte b/packages/builder/src/pages/builder/portal/settings/environment/index.svelte index 28258ccf4b..9af22ba942 100644 --- a/packages/builder/src/pages/builder/portal/settings/environment/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/environment/index.svelte @@ -53,7 +53,7 @@
- Envrironment Variables + Environment Variables {#if !$licensing.environmentVariablesEnabled} Pro plan diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index bbbf86a41d..ed84bb8ee9 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -28,7 +28,7 @@ export function createDatasourcesStore() { })) } - const updateDatasource = async response => { + const updateDatasource = response => { const { datasource, error } = response store.update(state => { const currentIdx = state.list.findIndex(ds => ds._id === datasource._id) @@ -52,7 +52,7 @@ export function createDatasourcesStore() { datasourceId: datasource?._id, tablesFilter, }) - return await updateDatasource(response) + return updateDatasource(response) } const save = async (body, fetchSchema = false) => { diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 47fde0764b..9568b40243 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -8,18 +8,24 @@ import { } from "../../db/utils" import { destroy as tableDestroy } from "./table/internal" import { BuildSchemaErrors, InvalidColumns } from "../../constants" -import { getIntegration } from "../../integrations" +import { getIntegration, getDefinitions } from "../../integrations" import { getDatasourceAndQuery } from "./row/utils" import { invalidateDynamicVariables } from "../../threads/utils" import { db as dbCore, context, events } from "@budibase/backend-core" -import { BBContext, Datasource, Row } from "@budibase/types" +import { + UserCtx, + Datasource, + Row, + DatasourceFieldType, + PASSWORD_REPLACEMENT, +} from "@budibase/types" import sdk from "../../sdk" -import { cloneDeep } from "lodash/fp" -import { enrich } from "../../sdk/app/datasources/datasources" +import { removeSecrets } from "../../sdk/app/datasources/datasources" -export async function fetch(ctx: BBContext) { +export async function fetch(ctx: UserCtx) { // Get internal tables const db = context.getAppDB() + const definitions = await getDefinitions() const internalTables = await db.allDocs( getTableParams(null, { include_docs: true, @@ -46,23 +52,19 @@ export async function fetch(ctx: BBContext) { ) ).rows.map(row => row.doc) - const allDatasources = [bbInternalDb, ...datasources] + const allDatasources: Datasource[] = [bbInternalDb, ...datasources] for (let datasource of allDatasources) { - if (datasource.config && datasource.config.auth) { - // strip secrets from response so they don't show in the network request - delete datasource.config.auth - } - + datasource = sdk.datasources.removeSecrets(definitions, datasource) if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) { - datasource.entities = internal[datasource._id] + datasource.entities = internal[datasource._id!] } } ctx.body = [bbInternalDb, ...datasources] } -export async function buildSchemaFromDb(ctx: BBContext) { +export async function buildSchemaFromDb(ctx: UserCtx) { const db = context.getAppDB() const datasource = await sdk.datasources.get(ctx.params.datasourceId) const tablesFilter = ctx.request.body.tablesFilter @@ -149,11 +151,12 @@ async function invalidateVariables( await invalidateDynamicVariables(toInvalidate) } -export async function update(ctx: BBContext) { +export async function update(ctx: UserCtx) { const db = context.getAppDB() const datasourceId = ctx.params.datasourceId let datasource = await sdk.datasources.get(datasourceId) const auth = datasource.config?.auth + const definitions = await getDefinitions() await invalidateVariables(datasource, ctx.request.body) const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE @@ -182,13 +185,16 @@ export async function update(ctx: BBContext) { ctx.status = 200 ctx.message = "Datasource saved successfully." - ctx.body = { datasource } + ctx.body = { + datasource: sdk.datasources.removeSecrets(definitions, datasource), + } } -export async function save(ctx: BBContext) { +export async function save(ctx: UserCtx) { const db = context.getAppDB() const plus = ctx.request.body.datasource.plus const fetchSchema = ctx.request.body.fetchSchema + const definitions = await getDefinitions() const datasource = { _id: generateDatasourceID({ plus }), @@ -216,7 +222,9 @@ export async function save(ctx: BBContext) { } } - const response: any = { datasource } + const response: any = { + datasource: sdk.datasources.removeSecrets(definitions, datasource), + } if (schemaError) { response.error = schemaError } @@ -254,7 +262,7 @@ async function destroyInternalTablesBySourceId(datasourceId: string) { } } -export async function destroy(ctx: BBContext) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const datasourceId = ctx.params.datasourceId @@ -282,13 +290,15 @@ export async function destroy(ctx: BBContext) { ctx.status = 200 } -export async function find(ctx: BBContext) { +export async function find(ctx: UserCtx) { const database = context.getAppDB() - ctx.body = await database.get(ctx.params.datasourceId) + const definitions = await getDefinitions() + const datasource = await database.get(ctx.params.datasourceId) + ctx.body = sdk.datasources.removeSecrets(definitions, datasource) } // dynamic query functionality -export async function query(ctx: BBContext) { +export async function query(ctx: UserCtx) { const queryJson = ctx.request.body try { ctx.body = await getDatasourceAndQuery(queryJson) diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 10f9ac889c..52204d7482 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -1,6 +1,11 @@ import { context } from "@budibase/backend-core" -import { processObjectSync } from "@budibase/string-templates" -import { Datasource } from "@budibase/types" +import { processObjectSync, findHBSBlocks } from "@budibase/string-templates" +import { + Datasource, + DatasourceFieldType, + Integration, + PASSWORD_REPLACEMENT, +} from "@budibase/types" import { cloneDeep } from "lodash/fp" import { getEnvironmentVariables } from "../../utils" @@ -37,3 +42,31 @@ export async function getWithEnvVars(datasourceId: string) { const datasource = await appDb.get(datasourceId) return enrichDatasourceWithValues(datasource) } + +export function removeSecrets( + definitions: Record, + datasource: Datasource +) { + const schema = definitions[datasource.source] + if (datasource.config) { + // strip secrets from response, so they don't show in the network request + if (datasource.config.auth) { + delete datasource.config.auth + } + // remove passwords + for (let key of Object.keys(datasource.config)) { + if (typeof datasource.config[key] !== "string") { + continue + } + const blocks = findHBSBlocks(datasource.config[key] as string) + const usesEnvVars = blocks.find(block => block.includes("env.")) != null + if ( + !usesEnvVars && + schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD + ) { + datasource.config[key] = PASSWORD_REPLACEMENT + } + } + } + return datasource +} diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 279a76446f..9d5b001a4b 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -1,5 +1,7 @@ import { Table } from "../documents" +export const PASSWORD_REPLACEMENT = "--secret-value--" + export enum Operation { CREATE = "CREATE", READ = "READ", @@ -104,7 +106,16 @@ export interface Integration { friendlyName: string type?: string iconUrl?: string - datasource: {} + datasource: Record< + string, + { + type: string + display?: string + deprecated?: boolean + default?: any + required?: boolean + } + > query: { [key: string]: QueryDefinition }