From 32bdc4a991c6d8f895c59a63067ac3d57c9fc5e9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 24 Sep 2021 18:10:30 +0100 Subject: [PATCH 1/3] Minimal fix for issue involving JSON views and newlines with postgres, this fix couldn't be more over-arching as it risked breaking new lines across the board. Have included a script for setting up the test scenario as well. This fixes issue #2612. --- .../integrations/pg-json/docker-compose.yml | 28 +++++++++++++++++++ .../scripts/integrations/pg-json/init.sql | 22 +++++++++++++++ .../scripts/integrations/pg-json/reset.sh | 3 ++ packages/server/src/integrations/postgres.ts | 12 ++++++++ packages/server/src/utilities/index.js | 11 ++++++++ 5 files changed, 76 insertions(+) create mode 100644 packages/server/scripts/integrations/pg-json/docker-compose.yml create mode 100644 packages/server/scripts/integrations/pg-json/init.sql create mode 100755 packages/server/scripts/integrations/pg-json/reset.sh diff --git a/packages/server/scripts/integrations/pg-json/docker-compose.yml b/packages/server/scripts/integrations/pg-json/docker-compose.yml new file mode 100644 index 0000000000..6bc307a86d --- /dev/null +++ b/packages/server/scripts/integrations/pg-json/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3.8" +services: + db: + container_name: postgres-json + image: postgres + restart: always + environment: + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: main + ports: + - "5432:5432" + volumes: + #- pg_data:/var/lib/postgresql/data/ + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + + pgadmin: + container_name: pgadmin-json + image: dpage/pgadmin4 + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: root@root.com + PGADMIN_DEFAULT_PASSWORD: root + ports: + - "5050:80" + +#volumes: +# pg_data: diff --git a/packages/server/scripts/integrations/pg-json/init.sql b/packages/server/scripts/integrations/pg-json/init.sql new file mode 100644 index 0000000000..06a5b4901d --- /dev/null +++ b/packages/server/scripts/integrations/pg-json/init.sql @@ -0,0 +1,22 @@ +SELECT 'CREATE DATABASE main' +WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec +CREATE TABLE jsonTable ( + id character varying(32), + data jsonb, + text text +); + +INSERT INTO jsonTable (id, data) VALUES ('1', '{"id": 1, "age": 1, "name": "Mike", "newline": "this is text with a\n newline in it"}'); + +CREATE VIEW jsonView AS SELECT + x.id, + x.age, + x.name, + x.newline +FROM + jsonTable c, + LATERAL jsonb_to_record(c.data) x (id character varying(32), + age BIGINT, + name TEXT, + newline TEXT + ); diff --git a/packages/server/scripts/integrations/pg-json/reset.sh b/packages/server/scripts/integrations/pg-json/reset.sh new file mode 100755 index 0000000000..32778bd11f --- /dev/null +++ b/packages/server/scripts/integrations/pg-json/reset.sh @@ -0,0 +1,3 @@ +#!/bin/bash +docker-compose down +docker volume prune -f diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 23a8685648..15a500b67e 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -13,6 +13,9 @@ module PostgresModule { const Sql = require("./base/sql") const { FieldTypes } = require("../constants") const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { escapeDangerousCharacters } = require("../utilities") + + const JSON_REGEX = /'{.*}'::json/s interface PostgresConfig { host: string @@ -94,6 +97,15 @@ module PostgresModule { } async function internalQuery(client: any, query: SqlQuery) { + // need to handle a specific issue with json data types in postgres, + // new lines inside the JSON data will break it + const matches = query.sql.match(JSON_REGEX) + if (matches && matches.length > 0) { + for (let match of matches) { + const escaped = escapeDangerousCharacters(match) + query.sql = query.sql.replace(match, escaped) + } + } try { return await client.query(query.sql, query.bindings || []) } catch (err) { diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index a81f9ddcf5..aac3610515 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -106,3 +106,14 @@ exports.deleteEntityMetadata = async (appId, type, entityId) => { await db.remove(id, rev) } } + +exports.escapeDangerousCharacters = string => { + return string + .replace(/[\\]/g, "\\\\") + .replace(/[\/]/g, "\\/") + .replace(/[\b]/g, "\\b") + .replace(/[\f]/g, "\\f") + .replace(/[\n]/g, "\\n") + .replace(/[\r]/g, "\\r") + .replace(/[\t]/g, "\\t") +} From 009f30b5cc97911287f44cb0ef43fdb4f8e72474 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 27 Sep 2021 12:17:59 +0100 Subject: [PATCH 2/3] Removing useless statement. --- packages/server/src/utilities/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index aac3610515..3aa43976e1 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -110,7 +110,6 @@ exports.deleteEntityMetadata = async (appId, type, entityId) => { exports.escapeDangerousCharacters = string => { return string .replace(/[\\]/g, "\\\\") - .replace(/[\/]/g, "\\/") .replace(/[\b]/g, "\\b") .replace(/[\f]/g, "\\f") .replace(/[\n]/g, "\\n") From 704c8891516e4ff05f0445c494403328445bcdba Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 27 Sep 2021 13:17:31 +0100 Subject: [PATCH 3/3] Fixing issue discovered by test case. --- packages/server/src/integrations/postgres.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 15a500b67e..e06e3936c8 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -99,11 +99,13 @@ module PostgresModule { async function internalQuery(client: any, query: SqlQuery) { // need to handle a specific issue with json data types in postgres, // new lines inside the JSON data will break it - const matches = query.sql.match(JSON_REGEX) - if (matches && matches.length > 0) { - for (let match of matches) { - const escaped = escapeDangerousCharacters(match) - query.sql = query.sql.replace(match, escaped) + if (query && query.sql) { + const matches = query.sql.match(JSON_REGEX) + if (matches && matches.length > 0) { + for (let match of matches) { + const escaped = escapeDangerousCharacters(match) + query.sql = query.sql.replace(match, escaped) + } } } try {