From aabbbdecfebb91115e6df3a1325352130911545a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 16 Jun 2021 15:45:57 +0100 Subject: [PATCH] consolidate postgres datas sources --- .../[selectedDatasource]/index.svelte | 17 +- .../builder/src/stores/backend/datasources.js | 30 ++- .../server/src/api/controllers/datasource.js | 29 ++- .../server/src/api/controllers/table/index.js | 2 +- packages/server/src/api/routes/datasource.js | 5 + packages/server/src/integrations/index.js | 2 - .../server/src/integrations/plus/postgres.js | 242 +++++++++--------- packages/server/src/integrations/postgres.js | 71 +++++ 8 files changed, 255 insertions(+), 143 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index 4560490b11..166c305d23 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -15,12 +15,22 @@ async function saveDatasource() { try { // Create datasource - await datasources.save(datasource, { refresh: true }) + await datasources.save(datasource) notifications.success(`Datasource ${name} saved successfully.`) unsaved = false + } catch (err) { + notifications.error(`Error saving datasource: ${err}`) + } + } + + async function updateDatasourceSchema() { + try { + await datasources.updateSchema(datasource) + notifications.success(`Datasource ${name} schema saved successfully.`) + unsaved = false await tables.fetch() } catch (err) { - notifications.error(`Error saving datasource: ${err}`) + notifications.error(`Error updating datasource schema: ${err}`) } } @@ -71,7 +81,6 @@ on:change={setUnsaved} /> - {#if !integration.plus}
Queries @@ -86,6 +95,8 @@
{/each} + {#if datasource.plus} + {/if} diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 186068d28c..df669f4fd8 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -28,14 +28,34 @@ export function createDatasourcesStore() { update(state => ({ ...state, selected: datasourceId })) queries.update(state => ({ ...state, selected: null })) }, - save: async (datasource, opts = {}) => { - let url = "/api/datasources" + updateSchema: async (datasource) => { + let url = `/api/datasources/${datasource._id}/schema` - if (datasource.plus && opts.refresh) { - // Pull the latest tables from the datasource - url += "?refresh=1" + const response = await api.post(url) + const json = await response.json() + + if (response.status !== 200) { + throw new Error(json.message) } + update(state => { + const currentIdx = state.list.findIndex(ds => ds._id === json._id) + + const sources = state.list + + if (currentIdx >= 0) { + sources.splice(currentIdx, 1, json) + } else { + sources.push(json) + } + + return { list: sources, selected: json._id } + }) + return json + }, + save: async (datasource) => { + let url = "/api/datasources" + const response = await api.post(url, datasource) const json = await response.json() diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js index 44bcfcd9ee..b6d25b7b83 100644 --- a/packages/server/src/api/controllers/datasource.js +++ b/packages/server/src/api/controllers/datasource.js @@ -8,7 +8,6 @@ const { getTableParams, } = require("../../db/utils") const { integrations } = require("../../integrations") -const plusIntegrations = require("../../integrations/plus") const { makeExternalQuery } = require("./row/utils") exports.fetch = async function (ctx) { @@ -40,6 +39,24 @@ exports.fetch = async function (ctx) { ctx.body = [bbInternalDb, ...datasources] } +exports.buildSchemaFromDb = async function (ctx) { + const db = new CouchDB(ctx.appId) + const datasourceId = ctx.params.datasourceId + const datasource = await db.get(datasourceId) + + const Connector = integrations[datasource.source] + + // Connect to the DB and build the schema + const connector = new Connector(datasource.config) + await connector.buildSchema(datasource._id) + datasource.entities = connector.tables + + const response = await db.post(datasource) + datasource._rev = response.rev + + ctx.body = datasource +} + exports.save = async function (ctx) { const db = new CouchDB(ctx.appId) const plus = ctx.request.body.plus @@ -50,16 +67,6 @@ exports.save = async function (ctx) { ...ctx.request.body, } - // update the schema - if (ctx.query.refresh) { - const PlusConnector = plusIntegrations[datasource.source].integration - - const connector = new PlusConnector(ctx.request.body.config) - await connector.init(datasource._id) - - datasource.entities = connector.tables - } - const response = await db.post(datasource) datasource._rev = response.rev diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index 35736fe399..db9ab4cd56 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -33,7 +33,7 @@ exports.fetch = async function (ctx) { ) const external = externalTables.rows.flatMap(row => { - return Object.values(row.doc.entities).map(entity => ({ + return Object.values(row.doc.entities || {}).map(entity => ({ ...entity, sourceId: row.doc._id, })) diff --git a/packages/server/src/api/routes/datasource.js b/packages/server/src/api/routes/datasource.js index 06964b61fd..e645cccd2b 100644 --- a/packages/server/src/api/routes/datasource.js +++ b/packages/server/src/api/routes/datasource.js @@ -72,6 +72,11 @@ router generateQueryDatasourceSchema(), datasourceController.query ) + .post( + "/api/datasources/:datasourceId/schema", + authorized(BUILDER), + datasourceController.buildSchemaFromDb + ) .post( "/api/datasources", authorized(BUILDER), diff --git a/packages/server/src/integrations/index.js b/packages/server/src/integrations/index.js index 6f90cee16c..bde154548e 100644 --- a/packages/server/src/integrations/index.js +++ b/packages/server/src/integrations/index.js @@ -24,7 +24,6 @@ const DEFINITIONS = { MYSQL: mysql.schema, ARANGODB: arangodb.schema, REST: rest.schema, - POSTGRES_PLUS: postgresPlus.schema, } const INTEGRATIONS = { @@ -39,7 +38,6 @@ const INTEGRATIONS = { MYSQL: mysql.integration, ARANGODB: arangodb.integration, REST: rest.integration, - POSTGRES_PLUS: postgresPlus.integration, } module.exports = { diff --git a/packages/server/src/integrations/plus/postgres.js b/packages/server/src/integrations/plus/postgres.js index 2d5b747935..c7edc56f9f 100644 --- a/packages/server/src/integrations/plus/postgres.js +++ b/packages/server/src/integrations/plus/postgres.js @@ -1,135 +1,135 @@ -const Sql = require("../base/sql") -const { Pool } = require("pg") -const { FieldTypes } = require("../../constants") -const { FIELD_TYPES } = require("../Integration") -const { SEPARATOR } = require("@budibase/auth/db") +// const Sql = require("../base/sql") +// const { Pool } = require("pg") +// const { FieldTypes } = require("../../constants") +// const { FIELD_TYPES } = require("../Integration") +// const { SEPARATOR } = require("@budibase/auth/db") -const TYPE_MAP = { - text: FieldTypes.LONGFORM, - varchar: FieldTypes.STRING, - integer: FieldTypes.NUMBER, - bigint: FieldTypes.NUMBER, - decimal: FieldTypes.NUMBER, - smallint: FieldTypes.NUMBER, - timestamp: FieldTypes.DATETIME, - time: FieldTypes.DATETIME, - boolean: FieldTypes.BOOLEAN, - json: FIELD_TYPES.JSON, -} +// const TYPE_MAP = { +// text: FieldTypes.LONGFORM, +// varchar: FieldTypes.STRING, +// integer: FieldTypes.NUMBER, +// bigint: FieldTypes.NUMBER, +// decimal: FieldTypes.NUMBER, +// smallint: FieldTypes.NUMBER, +// timestamp: FieldTypes.DATETIME, +// time: FieldTypes.DATETIME, +// boolean: FieldTypes.BOOLEAN, +// json: FIELD_TYPES.JSON, +// } -const SCHEMA = { - friendlyName: "PostgreSQL", - description: - "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.", - plus: true, - datasource: { - host: { - type: FIELD_TYPES.STRING, - default: "localhost", - required: true, - }, - port: { - type: FIELD_TYPES.NUMBER, - required: true, - default: 5432, - }, - database: { - type: FIELD_TYPES.STRING, - default: "postgres", - required: true, - }, - user: { - type: FIELD_TYPES.STRING, - default: "root", - required: true, - }, - password: { - type: FIELD_TYPES.PASSWORD, - default: "root", - required: true, - }, - ssl: { - type: FIELD_TYPES.BOOLEAN, - default: false, - required: false, - }, - }, -} +// const SCHEMA = { +// friendlyName: "PostgreSQL", +// description: +// "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.", +// plus: true, +// datasource: { +// host: { +// type: FIELD_TYPES.STRING, +// default: "localhost", +// required: true, +// }, +// port: { +// type: FIELD_TYPES.NUMBER, +// required: true, +// default: 5432, +// }, +// database: { +// type: FIELD_TYPES.STRING, +// default: "postgres", +// required: true, +// }, +// user: { +// type: FIELD_TYPES.STRING, +// default: "root", +// required: true, +// }, +// password: { +// type: FIELD_TYPES.PASSWORD, +// default: "root", +// required: true, +// }, +// ssl: { +// type: FIELD_TYPES.BOOLEAN, +// default: false, +// required: false, +// }, +// }, +// } -class PostgresPlus extends Sql { - static pool - COLUMNS_SQL = - "select * from information_schema.columns where table_schema = 'public'" +// class PostgresPlus extends Sql { +// static pool +// COLUMNS_SQL = +// "select * from information_schema.columns where table_schema = 'public'" - PRIMARY_KEYS_SQL = ` - select tc.table_schema, tc.table_name, kc.column_name as primary_key - from information_schema.table_constraints tc - join - information_schema.key_column_usage kc on kc.table_name = tc.table_name - and kc.table_schema = tc.table_schema - and kc.constraint_name = tc.constraint_name - where tc.constraint_type = 'PRIMARY KEY'; - ` +// PRIMARY_KEYS_SQL = ` +// select tc.table_schema, tc.table_name, kc.column_name as primary_key +// from information_schema.table_constraints tc +// join +// information_schema.key_column_usage kc on kc.table_name = tc.table_name +// and kc.table_schema = tc.table_schema +// and kc.constraint_name = tc.constraint_name +// where tc.constraint_type = 'PRIMARY KEY'; +// ` - constructor(config, datasource) { - super("pg") - this.config = config - this.datasource = datasource +// constructor(config, datasource) { +// super("pg") +// this.config = config +// this.datasource = datasource - if (!this.pool) { - this.pool = new Pool(this.config) - } +// if (!this.pool) { +// this.pool = new Pool(this.config) +// } - this.client = this.pool - } +// this.client = this.pool +// } - async init(datasourceId) { - let keys = [] - try { - const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL) - for (let table of primaryKeysResponse.rows) { - keys.push(table.column_name || table.primary_key) - } - } catch (err) { - // TODO: this try catch method isn't right - keys = ["id"] - } +// async init(datasourceId) { +// let keys = [] +// try { +// const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL) +// for (let table of primaryKeysResponse.rows) { +// keys.push(table.column_name || table.primary_key) +// } +// } catch (err) { +// // TODO: this try catch method isn't right +// keys = ["id"] +// } - const columnsResponse = await this.client.query(this.COLUMNS_SQL) - const tables = {} +// const columnsResponse = await this.client.query(this.COLUMNS_SQL) +// const tables = {} - for (let column of columnsResponse.rows) { - const tableName = column.table_name - const columnName = column.column_name +// for (let column of columnsResponse.rows) { +// const tableName = column.table_name +// const columnName = column.column_name - // table key doesn't exist yet - if (!tables[tableName]) { - tables[tableName] = { - _id: `${datasourceId}${SEPARATOR}${tableName}`, - // TODO: this needs to accommodate composite keys - primary: keys, - name: tableName, - schema: {}, - } - } +// // table key doesn't exist yet +// if (!tables[tableName]) { +// tables[tableName] = { +// _id: `${datasourceId}${SEPARATOR}${tableName}`, +// // TODO: this needs to accommodate composite keys +// primary: keys, +// name: tableName, +// schema: {}, +// } +// } - tables[tableName].schema[columnName] = { - name: columnName, - type: TYPE_MAP[column.data_type] || FIELD_TYPES.STRING, - } - } - this.tables = tables - } +// tables[tableName].schema[columnName] = { +// name: columnName, +// type: TYPE_MAP[column.data_type] || FIELD_TYPES.STRING, +// } +// } +// this.tables = tables +// } - async query(json) { - const operation = this._operation(json).toLowerCase() - const sql = this._query(json) - const response = await this.client.query(sql.sql, sql.bindings) - return response.rows.length ? response.rows : [{ [operation]: true }] - } -} +// async query(json) { +// const operation = this._operation(json).toLowerCase() +// const sql = this._query(json) +// const response = await this.client.query(sql.sql, sql.bindings) +// return response.rows.length ? response.rows : [{ [operation]: true }] +// } +// } -module.exports = { - schema: SCHEMA, - integration: PostgresPlus, -} +// module.exports = { +// schema: SCHEMA, +// integration: PostgresPlus, +// } diff --git a/packages/server/src/integrations/postgres.js b/packages/server/src/integrations/postgres.js index 72b02431be..0cbb4ffa4d 100644 --- a/packages/server/src/integrations/postgres.js +++ b/packages/server/src/integrations/postgres.js @@ -1,9 +1,12 @@ const { Pool } = require("pg") const { FIELD_TYPES } = require("./Integration") const Sql = require("./base/sql") +const { FieldTypes } = require("../constants") +const { SEPARATOR } = require("@budibase/auth/db") const SCHEMA = { docs: "https://node-postgres.com", + plus: true, friendlyName: "PostgreSQL", description: "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.", @@ -55,6 +58,19 @@ const SCHEMA = { }, } +const TYPE_MAP = { + text: FieldTypes.LONGFORM, + varchar: FieldTypes.STRING, + integer: FieldTypes.NUMBER, + bigint: FieldTypes.NUMBER, + decimal: FieldTypes.NUMBER, + smallint: FieldTypes.NUMBER, + timestamp: FieldTypes.DATETIME, + time: FieldTypes.DATETIME, + boolean: FieldTypes.BOOLEAN, + json: FIELD_TYPES.JSON, +} + async function internalQuery(client, sql) { try { return await client.query(sql) @@ -66,6 +82,19 @@ async function internalQuery(client, sql) { class PostgresIntegration extends Sql { static pool + COLUMNS_SQL = + "select * from information_schema.columns where table_schema = 'public'" + + PRIMARY_KEYS_SQL = ` + select tc.table_schema, tc.table_name, kc.column_name as primary_key + from information_schema.table_constraints tc + join + information_schema.key_column_usage kc on kc.table_name = tc.table_name + and kc.table_schema = tc.table_schema + and kc.constraint_name = tc.constraint_name + where tc.constraint_type = 'PRIMARY KEY'; + ` + constructor(config) { super("pg") this.config = config @@ -82,6 +111,48 @@ class PostgresIntegration extends Sql { this.client = this.pool } + /** + * Fetches the tables from the postgres table and assigns them to the datasource. + * @param {*} datasourceId - datasourceId to fetch + */ + async buildSchema(datasourceId) { + let keys = [] + try { + const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL) + for (let table of primaryKeysResponse.rows) { + keys.push(table.column_name || table.primary_key) + } + } catch (err) { + // TODO: this try catch method isn't right + keys = ["id"] + } + + const columnsResponse = await this.client.query(this.COLUMNS_SQL) + const tables = {} + + for (let column of columnsResponse.rows) { + const tableName = column.table_name + const columnName = column.column_name + + // table key doesn't exist yet + if (!tables[tableName]) { + tables[tableName] = { + _id: `${datasourceId}${SEPARATOR}${tableName}`, + // TODO: this needs to accommodate composite keys + primary: keys, + name: tableName, + schema: {}, + } + } + + tables[tableName].schema[columnName] = { + name: columnName, + type: TYPE_MAP[column.data_type] || FIELD_TYPES.STRING, + } + } + this.tables = tables + } + async create({ sql }) { const response = await internalQuery(this.client, sql) return response.rows.length ? response.rows : [{ created: true }]