budibase/packages/server/src/api/controllers/datasource.js

const {
  generateDatasourceID,
  getDatasourceParams,
  getQueryParams,
  DocumentType,
  BudibaseInternalDB,
  getTableParams,
} = require("../../db/utils")
const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
const { getIntegration } = require("../../integrations")
const { getDatasourceAndQuery } = require("./row/utils")
const { invalidateDynamicVariables } = require("../../threads/utils")
const { getAppDB } = require("@budibase/backend-core/context")
const { events } = require("@budibase/backend-core")

exports.fetch = async function (ctx) {
  // Get internal tables
  const db = getAppDB()
  const internalTables = await db.allDocs(
    getTableParams(null, {
      include_docs: true,
    })
  )
  const internal = internalTables.rows.map(row => row.doc)

  const bbInternalDb = {
    ...BudibaseInternalDB,
    entities: internal,
  }

  // Get external datasources
  const datasources = (
    await db.allDocs(
      getDatasourceParams(null, {
        include_docs: true,
      })
    )
  ).rows.map(row => row.doc)

  for (let datasource of datasources) {
    if (datasource.config && datasource.config.auth) {
      // strip secrets from response so they don't show in the network request
      delete datasource.config.auth
    }
  }

  ctx.body = [bbInternalDb, ...datasources]
}

exports.buildSchemaFromDb = async function (ctx) {
  const db = getAppDB()
  const datasource = await db.get(ctx.params.datasourceId)
  const tablesFilter = ctx.request.body.tablesFilter

  let { tables, error } = await buildSchemaHelper(datasource)
  if (tablesFilter) {
    if (!datasource.entities) {
      datasource.entities = {}
    }
    for (let key in tables) {
      if (
        tablesFilter.some(filter => filter.toLowerCase() === key.toLowerCase())
      ) {
        datasource.entities[key] = tables[key]
      }
    }
  } else {
    datasource.entities = tables
  }

  setDefaultDisplayColumns(datasource)
  const dbResp = await db.put(datasource)
  datasource._rev = dbResp.rev

  const response = { datasource }
  if (error) {
    response.error = error
  }
  ctx.body = response
}

/**
 * Make sure all datasource entities have a display name selected
 */
const setDefaultDisplayColumns = datasource => {
  //
  for (let entity of Object.values(datasource.entities)) {
    if (entity.primaryDisplay) {
      continue
    }
    const notAutoColumn = Object.values(entity.schema).find(
      schema => !schema.autocolumn
    )
    if (notAutoColumn) {
      entity.primaryDisplay = notAutoColumn.name
    }
  }
}

/**
 * Check for variables that have been updated or removed and invalidate them.
 */
const invalidateVariables = async (existingDatasource, updatedDatasource) => {
  const existingVariables = existingDatasource.config.dynamicVariables
  const updatedVariables = updatedDatasource.config.dynamicVariables
  const toInvalidate = []

  if (!existingVariables) {
    return
  }

  if (!updatedVariables) {
    // invalidate all
    toInvalidate.push(...existingVariables)
  } else {
    // invaldate changed / removed
    existingVariables.forEach(existing => {
      const unchanged = updatedVariables.find(
        updated =>
          existing.name === updated.name &&
          existing.queryId === updated.queryId &&
          existing.value === updated.value
      )
      if (!unchanged) {
        toInvalidate.push(existing)
      }
    })
  }
  await invalidateDynamicVariables(toInvalidate)
}

exports.update = async function (ctx) {
  const db = getAppDB()
  const datasourceId = ctx.params.datasourceId
  let datasource = await db.get(datasourceId)
  const auth = datasource.config.auth
  await invalidateVariables(datasource, ctx.request.body)
  datasource = { ...datasource, ...ctx.request.body }
  if (auth && !ctx.request.body.auth) {
    // don't strip auth config from DB
    datasource.config.auth = auth
  }

  const response = await db.put(datasource)
  await events.datasource.updated(datasource)
  datasource._rev = response.rev

  // Drain connection pools when configuration is changed
  if (datasource.source) {
    const source = await getIntegration(datasource.source)
    if (source && source.pool) {
      await source.pool.end()
    }
  }

  ctx.status = 200
  ctx.message = "Datasource saved successfully."
  ctx.body = { datasource }
}

exports.save = async function (ctx) {
  const db = getAppDB()
  const plus = ctx.request.body.datasource.plus
  const fetchSchema = ctx.request.body.fetchSchema

  const datasource = {
    _id: generateDatasourceID({ plus }),
    type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
    ...ctx.request.body.datasource,
  }

  let schemaError = null
  if (fetchSchema) {
    const { tables, error } = await buildSchemaHelper(datasource)
    schemaError = error
    datasource.entities = tables
    setDefaultDisplayColumns(datasource)
  }

  const dbResp = await db.put(datasource)
  await events.datasource.created(datasource)
  datasource._rev = dbResp.rev

  // Drain connection pools when configuration is changed
  if (datasource.source) {
    const source = await getIntegration(datasource.source)
    if (source && source.pool) {
      await source.pool.end()
    }
  }

  const response = { datasource }
  if (schemaError) {
    response.error = schemaError
  }
  ctx.body = response
}

exports.destroy = async function (ctx) {
  const db = getAppDB()
  const datasourceId = ctx.params.datasourceId

  const datasource = await db.get(datasourceId)
  // Delete all queries for the datasource
  const queries = await db.allDocs(getQueryParams(datasourceId, null))
  await db.bulkDocs(
    queries.rows.map(row => ({
      _id: row.id,
      _rev: row.value.rev,
      _deleted: true,
    }))
  )

  // delete the datasource
  await db.remove(datasourceId, ctx.params.revId)
  await events.datasource.deleted(datasource)

  ctx.message = `Datasource deleted.`
  ctx.status = 200
}

exports.find = async function (ctx) {
  const database = getAppDB()
  ctx.body = await database.get(ctx.params.datasourceId)
}

// dynamic query functionality
exports.query = async function (ctx) {
  const queryJson = ctx.request.body
  try {
    ctx.body = await getDatasourceAndQuery(queryJson)
  } catch (err) {
    ctx.throw(400, err)
  }
}

function getErrorTables(errors, errorType) {
  return Object.entries(errors)
    .filter(entry => entry[1] === errorType)
    .map(([name]) => name)
}

function updateError(error, newError, tables) {
  if (!error) {
    error = ""
  }
  if (error.length > 0) {
    error += "\n"
  }
  error += `${newError} ${tables.join(", ")}`
  return error
}

const buildSchemaHelper = async datasource => {
  const Connector = await getIntegration(datasource.source)

  // Connect to the DB and build the schema
  const connector = new Connector(datasource.config)
  await connector.buildSchema(datasource._id, datasource.entities)

  const errors = connector.schemaErrors
  let error = null
  if (errors && Object.keys(errors).length > 0) {
    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
    if (noKey.length) {
      error = updateError(
        error,
        "No primary key constraint found for the following:",
        noKey
      )
    }
    if (invalidCol.length) {
      const invalidCols = Object.values(InvalidColumns).join(", ")
      error = updateError(
        error,
        `Cannot use columns ${invalidCols} found in following:`,
        invalidCol
      )
    }
  }
  return { tables: connector.tables, error }
}