Some minor updates after reviewing.

packages/server/src/api/controllers/datasource.ts

@@ -8,24 +8,16 @@ import {
 } from "../../db/utils"
 import { destroy as tableDestroy } from "./table/internal"
 import { BuildSchemaErrors, InvalidColumns } from "../../constants"
-import { getIntegration, getDefinitions } from "../../integrations"
+import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
-import {
-  UserCtx,
-  Datasource,
-  Row,
-  DatasourceFieldType,
-  PASSWORD_REPLACEMENT,
-} from "@budibase/types"
+import { UserCtx, Datasource, Row } from "@budibase/types"
 import sdk from "../../sdk"
-import { removeSecrets } from "../../sdk/app/datasources/datasources"
 
 export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
-  const definitions = await getDefinitions()
   const internalTables = await db.allDocs(
     getTableParams(null, {
       include_docs: true,
@@ -52,10 +44,12 @@ export async function fetch(ctx: UserCtx) {
     )
   ).rows.map(row => row.doc)
 
-  const allDatasources: Datasource[] = [bbInternalDb, ...datasources]
+  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
+    bbInternalDb,
+    ...datasources,
+  ])
 
   for (let datasource of allDatasources) {
-    datasource = sdk.datasources.removeSecrets(definitions, datasource)
     if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
       datasource.entities = internal[datasource._id!]
     }
@@ -156,9 +150,12 @@ export async function update(ctx: UserCtx) {
   const datasourceId = ctx.params.datasourceId
   let datasource = await sdk.datasources.get(datasourceId)
   const auth = datasource.config?.auth
-  const definitions = await getDefinitions()
   await invalidateVariables(datasource, ctx.request.body)
 
+  if (!sdk.datasources.isValid(datasource)) {
+    ctx.throw(400, "Environment variables binding format incorrect")
+  }
+
   const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
 
   const dataSourceBody = isBudibaseSource
@@ -186,7 +183,7 @@ export async function update(ctx: UserCtx) {
   ctx.status = 200
   ctx.message = "Datasource saved successfully."
   ctx.body = {
-    datasource: sdk.datasources.removeSecrets(definitions, datasource),
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
   }
 }
 
@@ -194,7 +191,6 @@ export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
-  const definitions = await getDefinitions()
 
   const datasource = {
     _id: generateDatasourceID({ plus }),
@@ -202,6 +198,10 @@ export async function save(ctx: UserCtx) {
     ...ctx.request.body.datasource,
   }
 
+  if (!sdk.datasources.isValid(datasource)) {
+    ctx.throw(400, "Environment variables binding format incorrect")
+  }
+
   let schemaError = null
   if (fetchSchema) {
     const { tables, error } = await buildSchemaHelper(datasource)
@@ -223,7 +223,7 @@ export async function save(ctx: UserCtx) {
   }
 
   const response: any = {
-    datasource: sdk.datasources.removeSecrets(definitions, datasource),
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
   }
   if (schemaError) {
     response.error = schemaError
@@ -292,9 +292,8 @@ export async function destroy(ctx: UserCtx) {
 
 export async function find(ctx: UserCtx) {
   const database = context.getAppDB()
-  const definitions = await getDefinitions()
   const datasource = await database.get(ctx.params.datasourceId)
-  ctx.body = sdk.datasources.removeSecrets(definitions, datasource)
+  ctx.body = await sdk.datasources.removeSecretSingle(datasource)
 }
 
 // dynamic query functionality

packages/server/src/sdk/app/datasources/datasources.ts

@@ -7,8 +7,10 @@ import {
   PASSWORD_REPLACEMENT,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
-import { env } from "process"
 import { getEnvironmentVariables } from "../../utils"
+import { getDefinitions } from "../../../integrations"
+
+const ENV_VAR_PREFIX = "env."
 
 async function enrichDatasourceWithValues(datasource: Datasource) {
   const cloned = cloneDeep(datasource)
@@ -42,38 +44,50 @@ export async function getWithEnvVars(datasourceId: string) {
   const appDb = context.getAppDB()
   const datasource = await appDb.get(datasourceId)
   const blocks = findHBSBlocks(JSON.stringify(datasource))
-  const usesEnvVars = blocks.find(block => block.includes("env.")) != null
+  const usesEnvVars =
+    blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
   if (usesEnvVars) {
     return enrichDatasourceWithValues(datasource)
   } else {
-    throw new Error("Environment variables binding format incorrect")
+    return datasource
   }
 }
 
-export function removeSecrets(
-  definitions: Record<string, Integration>,
-  datasource: Datasource
-) {
-  const schema = definitions[datasource.source]
-  if (datasource.config) {
-    // strip secrets from response, so they don't show in the network request
-    if (datasource.config.auth) {
-      delete datasource.config.auth
-    }
-    // remove passwords
-    for (let key of Object.keys(datasource.config)) {
-      if (typeof datasource.config[key] !== "string") {
-        continue
+export function isValid(datasource: Datasource) {
+  const blocks = findHBSBlocks(JSON.stringify(datasource))
+  const validList = blocks.filter(block => block.includes(ENV_VAR_PREFIX))
+  return blocks.length === validList.length
+}
+
+export async function removeSecrets(datasources: Datasource[]) {
+  const definitions = await getDefinitions()
+  for (let datasource of datasources) {
+    const schema = definitions[datasource.source]
+    if (datasource.config) {
+      // strip secrets from response, so they don't show in the network request
+      if (datasource.config.auth) {
+        delete datasource.config.auth
       }
-      const blocks = findHBSBlocks(datasource.config[key] as string)
-      const usesEnvVars = blocks.find(block => block.includes("env.")) != null
-      if (
-        !usesEnvVars &&
-        schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD
-      ) {
-        datasource.config[key] = PASSWORD_REPLACEMENT
+      // remove passwords
+      for (let key of Object.keys(datasource.config)) {
+        if (typeof datasource.config[key] !== "string") {
+          continue
+        }
+        const blocks = findHBSBlocks(datasource.config[key] as string)
+        const usesEnvVars =
+          blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+        if (
+          !usesEnvVars &&
+          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD
+        ) {
+          datasource.config[key] = PASSWORD_REPLACEMENT
+        }
       }
     }
   }
-  return datasource
+  return datasources
+}
+
+export async function removeSecretSingle(datasource: Datasource) {
+  return (await removeSecrets([datasource]))[0]
 }