From 10e39abeaeae11f9b3cda3377ba3e7df52ca399f Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 16 Oct 2020 10:31:09 +0300 Subject: [PATCH 01/56] Basic structure --- Dockerfile | 1 + app/preload.php | 1 + app/realtime.php | 40 +++++++++++ bin/realtime | 3 + composer.json | 2 +- composer.lock | 172 ++++++++++++++++++++++----------------------- docker-compose.yml | 50 +++++++++++-- 7 files changed, 177 insertions(+), 92 deletions(-) create mode 100644 app/realtime.php create mode 100644 bin/realtime diff --git a/Dockerfile b/Dockerfile index 75c03c3e0..5f53a82cb 100755 --- a/Dockerfile +++ b/Dockerfile @@ -146,6 +146,7 @@ RUN mkdir -p /storage/uploads && \ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ + chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule && \ chmod +x /usr/local/bin/ssl && \ chmod +x /usr/local/bin/test && \ diff --git a/app/preload.php b/app/preload.php index 63d08c3e1..8ddbffbb7 100644 --- a/app/preload.php +++ b/app/preload.php @@ -26,6 +26,7 @@ include __DIR__.'/controllers/general.php'; ->paths(realpath(__DIR__ . '/../src')) ->ignore(realpath(__DIR__ . '/../vendor/twig/twig')) ->ignore(realpath(__DIR__ . '/../vendor/guzzlehttp/guzzle')) + ->ignore(realpath(__DIR__ . '/../vendor/domnikl')) ->ignore(realpath(__DIR__ . '/../vendor/geoip2')) ->ignore(realpath(__DIR__ . '/../vendor/maxmind')) ->ignore(realpath(__DIR__ . '/../vendor/maxmind-db')) diff --git a/app/realtime.php b/app/realtime.php new file mode 100644 index 000000000..648c6b4e9 --- /dev/null +++ b/app/realtime.php @@ -0,0 +1,40 @@ + } + * - JWT Authentication (in path / or in message) + */ + +$server = new Server("0.0.0.0", 80); + +$server->on("start", function (Server $server) { + echo "Swoole WebSocket Server has started at http://127.0.0.1:3000\n"; +}); + +$server->on('open', function(Server $server, Swoole\Http\Request $request) { + echo "connection open: {$request->fd}\n"; + // $server->tick(1000, function() use ($server, $request) { + // $server->push($request->fd, json_encode(["hello", time()])); + // }); + $server->push($request->fd, json_encode(["hello", time()])); +}); + +$server->on('message', function(Server $server, Frame $frame) { + echo "received message: {$frame->data}\n"; + $server->push($frame->fd, json_encode(["hello", time()])); +}); + +$server->on('close', function(Server $server, int $fd) { + echo "connection close: {$fd}\n"; +}); + +$server->start(); \ No newline at end of file diff --git a/bin/realtime b/bin/realtime new file mode 100644 index 000000000..e43dc269e --- /dev/null +++ b/bin/realtime @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/realtime.php $@ \ No newline at end of file diff --git a/composer.json b/composer.json index f0ce2624c..33ed8a928 100644 --- a/composer.json +++ b/composer.json @@ -53,7 +53,7 @@ "phpmailer/phpmailer": "6.1.7" }, "require-dev": { - "swoole/ide-helper": "4.5.4", + "swoole/ide-helper": "4.5.5", "appwrite/sdk-generator": "master", "phpunit/phpunit": "^9.3" }, diff --git a/composer.lock b/composer.lock index 297b144e8..e459019dc 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "07a5b2d2e742e8651d58889c3253c3b5", + "content-hash": "a59e355840c9762319629e4cc5fc2dbc", "packages": [ { "name": "appwrite/php-clamav", @@ -2300,12 +2300,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "ece0c3ceee73810bd95226401bbfaea9e0f64de7" + "reference": "e33667ac376b7f4dbe97ab556f8e7c8daee383d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/ece0c3ceee73810bd95226401bbfaea9e0f64de7", - "reference": "ece0c3ceee73810bd95226401bbfaea9e0f64de7", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/e33667ac376b7f4dbe97ab556f8e7c8daee383d3", + "reference": "e33667ac376b7f4dbe97ab556f8e7c8daee383d3", "shasum": "" }, "require": { @@ -2365,7 +2365,7 @@ "type": "github" } ], - "time": "2020-10-09T14:34:55+00:00" + "time": "2020-10-15T05:14:52+00:00" }, { "name": "phpunit/php-file-iterator", @@ -2373,12 +2373,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "aa4be8575f26070b100fccb67faabb28f21f66f8" + "reference": "8a1b0bfa74eba894f241e23261febb84c7ffbd8d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/aa4be8575f26070b100fccb67faabb28f21f66f8", - "reference": "aa4be8575f26070b100fccb67faabb28f21f66f8", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/8a1b0bfa74eba894f241e23261febb84c7ffbd8d", + "reference": "8a1b0bfa74eba894f241e23261febb84c7ffbd8d", "shasum": "" }, "require": { @@ -2421,7 +2421,7 @@ "type": "github" } ], - "time": "2020-09-28T05:57:25+00:00" + "time": "2020-10-15T05:05:12+00:00" }, { "name": "phpunit/php-invoker", @@ -2429,12 +2429,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-invoker.git", - "reference": "5a10147d0aaf65b58940a0b72f71c9ac0423cc67" + "reference": "dcc4b2e39d6cb5ba5435a0177ebe947c0c0d05ff" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-invoker/zipball/5a10147d0aaf65b58940a0b72f71c9ac0423cc67", - "reference": "5a10147d0aaf65b58940a0b72f71c9ac0423cc67", + "url": "https://api.github.com/repos/sebastianbergmann/php-invoker/zipball/dcc4b2e39d6cb5ba5435a0177ebe947c0c0d05ff", + "reference": "dcc4b2e39d6cb5ba5435a0177ebe947c0c0d05ff", "shasum": "" }, "require": { @@ -2480,7 +2480,7 @@ "type": "github" } ], - "time": "2020-09-28T05:58:55+00:00" + "time": "2020-10-15T05:05:21+00:00" }, { "name": "phpunit/php-text-template", @@ -2488,12 +2488,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-text-template.git", - "reference": "18c887016e60e52477e54534956d7b47bc52cd84" + "reference": "0b11f04dcd54d149c3904cda577ea8ef8735e377" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/18c887016e60e52477e54534956d7b47bc52cd84", - "reference": "18c887016e60e52477e54534956d7b47bc52cd84", + "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/0b11f04dcd54d149c3904cda577ea8ef8735e377", + "reference": "0b11f04dcd54d149c3904cda577ea8ef8735e377", "shasum": "" }, "require": { @@ -2535,7 +2535,7 @@ "type": "github" } ], - "time": "2020-09-28T06:03:05+00:00" + "time": "2020-10-15T05:06:00+00:00" }, { "name": "phpunit/php-timer", @@ -2543,12 +2543,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-timer.git", - "reference": "c9ff14f493699e2f6adee9fd06a0245b276643b7" + "reference": "7fe57355ba7462b1cd940d93aa003660b4e6db20" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/c9ff14f493699e2f6adee9fd06a0245b276643b7", - "reference": "c9ff14f493699e2f6adee9fd06a0245b276643b7", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/7fe57355ba7462b1cd940d93aa003660b4e6db20", + "reference": "7fe57355ba7462b1cd940d93aa003660b4e6db20", "shasum": "" }, "require": { @@ -2590,7 +2590,7 @@ "type": "github" } ], - "time": "2020-09-28T06:00:25+00:00" + "time": "2020-10-15T05:05:31+00:00" }, { "name": "phpunit/phpunit", @@ -2598,12 +2598,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "8b79c2a70ae855e582cef1ca63a849fe07bdb01d" + "reference": "137f98be3b928d5f9f14e66f7aa5eadcfc668bbe" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/8b79c2a70ae855e582cef1ca63a849fe07bdb01d", - "reference": "8b79c2a70ae855e582cef1ca63a849fe07bdb01d", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/137f98be3b928d5f9f14e66f7aa5eadcfc668bbe", + "reference": "137f98be3b928d5f9f14e66f7aa5eadcfc668bbe", "shasum": "" }, "require": { @@ -2689,7 +2689,7 @@ "type": "github" } ], - "time": "2020-10-11T07:43:20+00:00" + "time": "2020-10-15T05:02:48+00:00" }, { "name": "sebastian/cli-parser", @@ -2697,12 +2697,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/cli-parser.git", - "reference": "442e7c7e687e42adc03470c7b668bc4b2402c0b2" + "reference": "bb13fcea306b784ef38fc1cda21c1395c233f4bc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/442e7c7e687e42adc03470c7b668bc4b2402c0b2", - "reference": "442e7c7e687e42adc03470c7b668bc4b2402c0b2", + "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/bb13fcea306b784ef38fc1cda21c1395c233f4bc", + "reference": "bb13fcea306b784ef38fc1cda21c1395c233f4bc", "shasum": "" }, "require": { @@ -2741,7 +2741,7 @@ "type": "github" } ], - "time": "2020-09-28T06:08:49+00:00" + "time": "2020-10-15T05:19:54+00:00" }, { "name": "sebastian/code-unit", @@ -2749,12 +2749,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/code-unit.git", - "reference": "59236be62b1bb9919e6d7f60b0b832dc05cef9ab" + "reference": "46a6ff3fabc0449fa17ca3ec485c44ab792f65c1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/code-unit/zipball/59236be62b1bb9919e6d7f60b0b832dc05cef9ab", - "reference": "59236be62b1bb9919e6d7f60b0b832dc05cef9ab", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit/zipball/46a6ff3fabc0449fa17ca3ec485c44ab792f65c1", + "reference": "46a6ff3fabc0449fa17ca3ec485c44ab792f65c1", "shasum": "" }, "require": { @@ -2793,7 +2793,7 @@ "type": "github" } ], - "time": "2020-10-02T14:47:54+00:00" + "time": "2020-10-15T05:03:44+00:00" }, { "name": "sebastian/code-unit-reverse-lookup", @@ -2801,12 +2801,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/code-unit-reverse-lookup.git", - "reference": "ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5" + "reference": "a801a24d7681090e8334c631b99181df063ea457" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5", - "reference": "ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/a801a24d7681090e8334c631b99181df063ea457", + "reference": "a801a24d7681090e8334c631b99181df063ea457", "shasum": "" }, "require": { @@ -2844,7 +2844,7 @@ "type": "github" } ], - "time": "2020-09-28T05:30:19+00:00" + "time": "2020-10-15T05:03:53+00:00" }, { "name": "sebastian/comparator", @@ -2852,12 +2852,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "7a8ff306445707539c1a6397372a982a1ec55120" + "reference": "d43148f588efca5b5dd0c3d98da467f5aafdac6b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/7a8ff306445707539c1a6397372a982a1ec55120", - "reference": "7a8ff306445707539c1a6397372a982a1ec55120", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/d43148f588efca5b5dd0c3d98da467f5aafdac6b", + "reference": "d43148f588efca5b5dd0c3d98da467f5aafdac6b", "shasum": "" }, "require": { @@ -2914,7 +2914,7 @@ "type": "github" } ], - "time": "2020-09-30T06:47:25+00:00" + "time": "2020-10-15T05:04:03+00:00" }, { "name": "sebastian/complexity", @@ -2922,12 +2922,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/complexity.git", - "reference": "ba8cc2da0c0bfbc813d03b56406734030c7f1eff" + "reference": "6d4cf3e8224f1e8527ab434b4ba902978af523db" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/ba8cc2da0c0bfbc813d03b56406734030c7f1eff", - "reference": "ba8cc2da0c0bfbc813d03b56406734030c7f1eff", + "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/6d4cf3e8224f1e8527ab434b4ba902978af523db", + "reference": "6d4cf3e8224f1e8527ab434b4ba902978af523db", "shasum": "" }, "require": { @@ -2967,7 +2967,7 @@ "type": "github" } ], - "time": "2020-09-28T06:05:03+00:00" + "time": "2020-10-15T05:06:11+00:00" }, { "name": "sebastian/diff", @@ -2975,12 +2975,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "e3ec6059b3fe483d42fbaf1fe6eefa201f7b4a6d" + "reference": "c25d82b5b776a3ba4e3b232a8688e969477444e0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/e3ec6059b3fe483d42fbaf1fe6eefa201f7b4a6d", - "reference": "e3ec6059b3fe483d42fbaf1fe6eefa201f7b4a6d", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/c25d82b5b776a3ba4e3b232a8688e969477444e0", + "reference": "c25d82b5b776a3ba4e3b232a8688e969477444e0", "shasum": "" }, "require": { @@ -3029,7 +3029,7 @@ "type": "github" } ], - "time": "2020-10-13T11:48:30+00:00" + "time": "2020-10-15T05:04:12+00:00" }, { "name": "sebastian/environment", @@ -3037,12 +3037,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "388b6ced16caa751030f6a69e588299fa09200ac" + "reference": "40fcf803a36737ce1d2c46c489fb5d1ec2db45d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/388b6ced16caa751030f6a69e588299fa09200ac", - "reference": "388b6ced16caa751030f6a69e588299fa09200ac", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/40fcf803a36737ce1d2c46c489fb5d1ec2db45d4", + "reference": "40fcf803a36737ce1d2c46c489fb5d1ec2db45d4", "shasum": "" }, "require": { @@ -3088,7 +3088,7 @@ "type": "github" } ], - "time": "2020-09-28T05:52:38+00:00" + "time": "2020-10-15T05:04:22+00:00" }, { "name": "sebastian/exporter", @@ -3096,12 +3096,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "d89cc98761b8cb5a1a235a6b703ae50d34080e65" + "reference": "dbdc89af25883b35be6ddd4b88a8bc5d22bb819d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/d89cc98761b8cb5a1a235a6b703ae50d34080e65", - "reference": "d89cc98761b8cb5a1a235a6b703ae50d34080e65", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/dbdc89af25883b35be6ddd4b88a8bc5d22bb819d", + "reference": "dbdc89af25883b35be6ddd4b88a8bc5d22bb819d", "shasum": "" }, "require": { @@ -3161,7 +3161,7 @@ "type": "github" } ], - "time": "2020-09-28T05:24:23+00:00" + "time": "2020-10-15T05:04:32+00:00" }, { "name": "sebastian/global-state", @@ -3169,12 +3169,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "ea779cb749a478b22a2564ac41cd7bda79c78dc7" + "reference": "3a606041b47fb201c1d62ae03078d0e8a62569a6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/ea779cb749a478b22a2564ac41cd7bda79c78dc7", - "reference": "ea779cb749a478b22a2564ac41cd7bda79c78dc7", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/3a606041b47fb201c1d62ae03078d0e8a62569a6", + "reference": "3a606041b47fb201c1d62ae03078d0e8a62569a6", "shasum": "" }, "require": { @@ -3221,7 +3221,7 @@ "type": "github" } ], - "time": "2020-09-28T05:54:06+00:00" + "time": "2020-10-15T05:04:42+00:00" }, { "name": "sebastian/lines-of-code", @@ -3229,12 +3229,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/lines-of-code.git", - "reference": "6514b8f21906b8b46f520d1fbd17a4523fa59a54" + "reference": "5835b6f4707e022ae53a2a67420a3388b318715a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/6514b8f21906b8b46f520d1fbd17a4523fa59a54", - "reference": "6514b8f21906b8b46f520d1fbd17a4523fa59a54", + "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/5835b6f4707e022ae53a2a67420a3388b318715a", + "reference": "5835b6f4707e022ae53a2a67420a3388b318715a", "shasum": "" }, "require": { @@ -3274,7 +3274,7 @@ "type": "github" } ], - "time": "2020-09-28T06:07:27+00:00" + "time": "2020-10-15T05:06:21+00:00" }, { "name": "sebastian/object-enumerator", @@ -3282,12 +3282,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/object-enumerator.git", - "reference": "f6f5957013d84725427d361507e13513702888a4" + "reference": "192362c78b33b0231e1e8841678be93ce6f31830" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/f6f5957013d84725427d361507e13513702888a4", - "reference": "f6f5957013d84725427d361507e13513702888a4", + "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/192362c78b33b0231e1e8841678be93ce6f31830", + "reference": "192362c78b33b0231e1e8841678be93ce6f31830", "shasum": "" }, "require": { @@ -3327,7 +3327,7 @@ "type": "github" } ], - "time": "2020-09-28T05:55:06+00:00" + "time": "2020-10-15T05:04:51+00:00" }, { "name": "sebastian/object-reflector", @@ -3335,12 +3335,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/object-reflector.git", - "reference": "d9d0ab3b12acb1768bc1e0a89b23c90d2043cbe5" + "reference": "da3d1ade6fef132f2486d297c35cf61e45930a0b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/d9d0ab3b12acb1768bc1e0a89b23c90d2043cbe5", - "reference": "d9d0ab3b12acb1768bc1e0a89b23c90d2043cbe5", + "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/da3d1ade6fef132f2486d297c35cf61e45930a0b", + "reference": "da3d1ade6fef132f2486d297c35cf61e45930a0b", "shasum": "" }, "require": { @@ -3378,7 +3378,7 @@ "type": "github" } ], - "time": "2020-09-28T05:56:16+00:00" + "time": "2020-10-15T05:05:01+00:00" }, { "name": "sebastian/recursion-context", @@ -3386,12 +3386,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/recursion-context.git", - "reference": "7e70f3d32a3058d4ad5226c1371f2dd4677dc073" + "reference": "3767a68ada0fc1d50b22db067cd2256b1b722faa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/7e70f3d32a3058d4ad5226c1371f2dd4677dc073", - "reference": "7e70f3d32a3058d4ad5226c1371f2dd4677dc073", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/3767a68ada0fc1d50b22db067cd2256b1b722faa", + "reference": "3767a68ada0fc1d50b22db067cd2256b1b722faa", "shasum": "" }, "require": { @@ -3437,7 +3437,7 @@ "type": "github" } ], - "time": "2020-09-28T05:27:00+00:00" + "time": "2020-10-15T05:05:41+00:00" }, { "name": "sebastian/resource-operations", @@ -3496,12 +3496,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/type.git", - "reference": "fa592377f3923946cb90bf1f6a71ba2e5f229909" + "reference": "5eb6a85e349cab8a411886cb52c070393d71bc7e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/fa592377f3923946cb90bf1f6a71ba2e5f229909", - "reference": "fa592377f3923946cb90bf1f6a71ba2e5f229909", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/5eb6a85e349cab8a411886cb52c070393d71bc7e", + "reference": "5eb6a85e349cab8a411886cb52c070393d71bc7e", "shasum": "" }, "require": { @@ -3540,7 +3540,7 @@ "type": "github" } ], - "time": "2020-10-06T08:41:03+00:00" + "time": "2020-10-15T05:05:50+00:00" }, { "name": "sebastian/version", @@ -3593,16 +3593,16 @@ }, { "name": "swoole/ide-helper", - "version": "4.5.4", + "version": "4.5.5", "source": { "type": "git", "url": "https://github.com/swoole/ide-helper.git", - "reference": "3382a1844afb206cac064252f6b8b50115bf72bb" + "reference": "aefd9d15e00cf14b89a5ed87cfa3bd79c9889028" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/swoole/ide-helper/zipball/3382a1844afb206cac064252f6b8b50115bf72bb", - "reference": "3382a1844afb206cac064252f6b8b50115bf72bb", + "url": "https://api.github.com/repos/swoole/ide-helper/zipball/aefd9d15e00cf14b89a5ed87cfa3bd79c9889028", + "reference": "aefd9d15e00cf14b89a5ed87cfa3bd79c9889028", "shasum": "" }, "require-dev": { @@ -3623,7 +3623,7 @@ } ], "description": "IDE help files for Swoole.", - "time": "2020-09-16T00:12:52+00:00" + "time": "2020-10-14T18:05:12+00:00" }, { "name": "symfony/polyfill-ctype", diff --git a/docker-compose.yml b/docker-compose.yml index 19388e463..de9945b55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,8 @@ services: - --providers.file.directory=/storage/config - --providers.file.watch=true - --providers.docker=true - - --entrypoints.web.address=:80 - - --entrypoints.websecure.address=:443 + - --entrypoints.appwrite_web.address=:80 + - --entrypoints.appwrite_websecure.address=:443 - --accesslog=true restart: unless-stopped ports: @@ -41,9 +41,18 @@ services: networks: - appwrite labels: - - traefik.http.routers.appwrite.rule=PathPrefix(`/`) - - traefik.http.routers.appwrite-secure.rule=PathPrefix(`/`) - - traefik.http.routers.appwrite-secure.tls=true + - "traefik.enable=true" + - "traefik.docker.network=appwrite" + - "traefik.http.services.appwrite_api.loadbalancer.server.port=80" + #http + - traefik.http.routers.appwrite_api_http.entrypoints=appwrite_web + - traefik.http.routers.appwrite_api_http.rule=PathPrefix(`/`) + - traefik.http.routers.appwrite_api_http.service=appwrite_api + # https + - traefik.http.routers.appwrite_api_https.entrypoints=appwrite_websecure + - traefik.http.routers.appwrite_api_https.rule=PathPrefix(`/`) + - traefik.http.routers.appwrite_api_https.service=appwrite_api + - traefik.http.routers.appwrite_api_https.tls=true volumes: - /var/run/docker.sock:/var/run/docker.sock - appwrite-uploads:/storage/uploads:rw @@ -87,6 +96,37 @@ services: - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_CONTAINERS + appwrite-socket: + entrypoint: socket + container_name: appwrite-socket + build: + context: . + restart: unless-stopped + ports: + - 9502:80 + labels: + - "traefik.enable=true" + - "traefik.docker.network=appwrite" + - "traefik.http.services.appwrite_socket.loadbalancer.server.port=80" + #ws + - traefik.http.routers.appwrite_socket_ws.entrypoints=appwrite_web + - traefik.http.routers.appwrite_socket_ws.rule=PathPrefix(`/v1/realtime`) + - traefik.http.routers.appwrite_socket_ws.service=appwrite_socket + # wss + - traefik.http.routers.appwrite_socket_wss.entrypoints=appwrite_websecure + - traefik.http.routers.appwrite_socket_wss.rule=PathPrefix(`/v1/realtime`) + - traefik.http.routers.appwrite_socket_wss.service=appwrite_socket + - traefik.http.routers.appwrite_socket_wss.tls=true + - traefik.http.routers.appwrite_socket_wss.tls.certresolver=dns + networks: + - appwrite + depends_on: + - redis + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + appwrite-worker-usage: entrypoint: worker-usage container_name: appwrite-worker-usage From 875c83a137d8f6eca4ebdca8e9cff6462840af49 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 16 Oct 2020 10:50:46 +0300 Subject: [PATCH 02/56] Updated entypoint --- docker-compose.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index de9945b55..4a8c02c64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,9 +96,9 @@ services: - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_CONTAINERS - appwrite-socket: - entrypoint: socket - container_name: appwrite-socket + appwrite-realtime: + entrypoint: realtime + container_name: appwrite-realtime build: context: . restart: unless-stopped @@ -107,17 +107,17 @@ services: labels: - "traefik.enable=true" - "traefik.docker.network=appwrite" - - "traefik.http.services.appwrite_socket.loadbalancer.server.port=80" + - "traefik.http.services.appwrite_realtime.loadbalancer.server.port=80" #ws - - traefik.http.routers.appwrite_socket_ws.entrypoints=appwrite_web - - traefik.http.routers.appwrite_socket_ws.rule=PathPrefix(`/v1/realtime`) - - traefik.http.routers.appwrite_socket_ws.service=appwrite_socket + - traefik.http.routers.appwrite_realtime_ws.entrypoints=appwrite_web + - traefik.http.routers.appwrite_realtime_ws.rule=PathPrefix(`/v1/realtime`) + - traefik.http.routers.appwrite_realtime_ws.service=appwrite_realtime # wss - - traefik.http.routers.appwrite_socket_wss.entrypoints=appwrite_websecure - - traefik.http.routers.appwrite_socket_wss.rule=PathPrefix(`/v1/realtime`) - - traefik.http.routers.appwrite_socket_wss.service=appwrite_socket - - traefik.http.routers.appwrite_socket_wss.tls=true - - traefik.http.routers.appwrite_socket_wss.tls.certresolver=dns + - traefik.http.routers.appwrite_realtime_wss.entrypoints=appwrite_websecure + - traefik.http.routers.appwrite_realtime_wss.rule=PathPrefix(`/v1/realtime`) + - traefik.http.routers.appwrite_realtime_wss.service=appwrite_realtime + - traefik.http.routers.appwrite_realtime_wss.tls=true + - traefik.http.routers.appwrite_realtime_wss.tls.certresolver=dns networks: - appwrite depends_on: From 3cb584430b40530b39ca44ec99d446fe4b09ea41 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 17 Oct 2020 08:48:03 +0300 Subject: [PATCH 03/56] Added docs --- app/realtime.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/realtime.php b/app/realtime.php index 648c6b4e9..4123d741d 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -12,6 +12,10 @@ use Swoole\WebSocket\Frame; * - Limit payload size * - Message structure: { status: "ok"|"error", event: EVENT_NAME, data: } * - JWT Authentication (in path / or in message) + * + * + * - https://github.com/hhxsv5/php-sse + * - https://github.com/shuixn/socket.io-swoole-server */ $server = new Server("0.0.0.0", 80); @@ -25,6 +29,8 @@ $server->on('open', function(Server $server, Swoole\Http\Request $request) { // $server->tick(1000, function() use ($server, $request) { // $server->push($request->fd, json_encode(["hello", time()])); // }); + + var_dump($request->header); $server->push($request->fd, json_encode(["hello", time()])); }); From 5f4532ebbfbad697dd838e2959e9be98e8a6505f Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 18 Oct 2020 14:51:16 +0300 Subject: [PATCH 04/56] Added redis pub/sub --- app/realtime.php | 77 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 7 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 4123d741d..b439ac519 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,8 +1,8 @@ on('message', function(\Swoole\Coroutine\Redis $redis, $rs) use ($server) { +// var_dump($server); +// echo 'redis got message' . PHP_EOL; +// var_dump($rs); +// $server->send(1, $rs); +// }); +// $redis->connect('redis', 6379, function(\Swoole\Coroutine\Redis $redis, $result){ + +// echo 'connected to redis' . PHP_EOL; +// $redis->subscribe('chat'); +// }); + +$server->on("workerStart", function ($server, $workerId) { + Console::success('Worker '.++$workerId.' started succefully'); + + $redis = new Redis(); + $redis->connect('redis', 6379); + + $redis->subscribe(['realtime'], function($redis, $channel, $message) use ($server, $workerId) { + var_dump($redis, $channel, $message); + + $message = 'Message from worker #'.$workerId.'; '.$message; + + foreach($server->connections as $fd) { + if ($server->exist($fd) && $server->isEstablished($fd)) { + $server->push($fd, $message, SWOOLE_WEBSOCKET_OPCODE_TEXT, SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); + } + else { + $server->close($fd); + } + } + }); +}); + +$server->on('BeforeReload', function($serv, $workerId) { + Console::success('Starting reload...'); +}); + +$server->on('AfterReload', function($serv, $workerId) { + Console::success('Reload completed...'); +}); + +// $process = new Process(function($process) use ($server) { +// while (true) { +// $msg = $process->read(); + +// foreach($server->connections as $fd) { +// if ($server->exist($fd) && $server->isEstablished($fd)) { +// $server->push($fd, json_encode(['hey there']), SWOOLE_WEBSOCKET_OPCODE_TEXT, SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); +// } +// } + +// sleep(10); +// } +// }); + +// $server->addProcess($process); + $server->on("start", function (Server $server) { - echo "Swoole WebSocket Server has started at http://127.0.0.1:3000\n"; + Console::success('Server started succefully'); }); $server->on('open', function(Server $server, Swoole\Http\Request $request) { echo "connection open: {$request->fd}\n"; - // $server->tick(1000, function() use ($server, $request) { - // $server->push($request->fd, json_encode(["hello", time()])); - // }); - var_dump($request->header); + foreach($server->connections as $fd) { + if ($server->exist($fd) && $server->isEstablished($fd)) { + $server->push($fd, json_encode(['hey there', count($server->ports[0]->connections), ]), SWOOLE_WEBSOCKET_OPCODE_TEXT, SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); + } + } + $server->push($request->fd, json_encode(["hello", time()])); }); $server->on('message', function(Server $server, Frame $frame) { echo "received message: {$frame->data}\n"; - $server->push($frame->fd, json_encode(["hello", time()])); + $server->push($frame->fd, json_encode(["hello, worker_id:".$server->getWorkerId(), time()])); }); $server->on('close', function(Server $server, int $fd) { From 4dfb91f2ef089b5665f38977e0f72a5fb2644aa8 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 19 Oct 2020 17:49:35 +0300 Subject: [PATCH 05/56] Updated phpredis client --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5f53a82cb..519d2a01c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN composer update --ignore-platform-reqs --optimize-autoloader \ FROM php:7.4-cli-alpine as step1 ENV TZ=Asia/Tel_Aviv \ - PHP_REDIS_VERSION=5.3.0 \ + PHP_REDIS_VERSION=5.3.2RC2 \ PHP_SWOOLE_VERSION=4.5.5 \ PHP_XDEBUG_VERSION=sdebug_2_9-beta @@ -170,6 +170,7 @@ RUN echo extension=redis.so >> /usr/local/etc/php/conf.d/redis.ini RUN echo "opcache.preload_user=www-data" >> /usr/local/etc/php/conf.d/appwrite.ini RUN echo "opcache.preload=/usr/src/code/app/preload.php" >> /usr/local/etc/php/conf.d/appwrite.ini RUN echo "opcache.enable_cli = 1" >> /usr/local/etc/php/conf.d/appwrite.ini +RUN echo "default_socket_timeout = -1" >> /usr/local/etc/php/conf.d/appwrite.ini EXPOSE 80 From b590ecc00ac67aa1fedd5b9b0b601a0da53df7d3 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 19 Oct 2020 17:49:49 +0300 Subject: [PATCH 06/56] Removed unsupported server option --- app/http.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/http.php b/app/http.php index 954037543..e51677f88 100644 --- a/app/http.php +++ b/app/http.php @@ -30,7 +30,6 @@ $http 'open_http2_protocol' => true, // 'document_root' => __DIR__.'/../public', // 'enable_static_handler' => true, - 'timeout' => 7, 'http_compression' => true, 'http_compression_level' => 6, 'package_max_length' => $payloadSize, From 210fa9fd64bb61ff3528bf03100259ef46bf0fc6 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 19 Oct 2020 17:50:12 +0300 Subject: [PATCH 07/56] Fixed redis connection init --- app/init.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 9637c4381..c6a578eba 100644 --- a/app/init.php +++ b/app/init.php @@ -166,8 +166,7 @@ $register->set('statsd', function () { // Register DB connection }); $register->set('cache', function () { // Register cache connection $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', '', 2.5), - App::getEnv('_APP_REDIS_PORT', '')); + $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); return $redis; }); From 3f0595c04aa69babebb8ca0f49e0f8668918570a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 19 Oct 2020 18:09:27 +0300 Subject: [PATCH 08/56] Test realtime server --- app/controllers/api/health.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 5ef4f2b9c..dc14fffff 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -30,6 +30,19 @@ App::get('/v1/health/version') $response->json(['version' => APP_VERSION_STABLE]); }, ['response']); +App::get('/v1/health/realtime') + ->desc('Get Realtime') + ->groups(['api', 'health']) + ->label('scope', 'public') + ->action(function ($response) { + /** @var Utopia\Response $response */ + $redis = new Redis(); + $redis->connect('redis', 6379); + + $redis->publish('realtime', 'I\'m a live message'); + $response->json(['status' => 'OK']); + }, ['response']); + App::get('/v1/health/db') ->desc('Get DB') ->groups(['api', 'health']) From 77e35e708196ab2d051c706da81762fe65338b79 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 19 Oct 2020 18:09:53 +0300 Subject: [PATCH 09/56] Added redis persistent connection --- app/realtime.php | 126 ++++++++++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 62 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index b439ac519..90416aa17 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,6 +1,9 @@ on('message', function(\Swoole\Coroutine\Redis $redis, $rs) use ($server) { -// var_dump($server); -// echo 'redis got message' . PHP_EOL; -// var_dump($rs); -// $server->send(1, $rs); -// }); -// $redis->connect('redis', 6379, function(\Swoole\Coroutine\Redis $redis, $result){ +$connections = []; -// echo 'connected to redis' . PHP_EOL; -// $redis->subscribe('chat'); -// }); - -$server->on("workerStart", function ($server, $workerId) { +$server->on("workerStart", function ($server, $workerId) use (&$connections) { Console::success('Worker '.++$workerId.' started succefully'); - $redis = new Redis(); - $redis->connect('redis', 6379); + $attempts = 0; + $start = time(); + + while ($attempts < 3) { + try { + $redis = new Redis(); + $redis->connect('redis', 6379); + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - $redis->subscribe(['realtime'], function($redis, $channel, $message) use ($server, $workerId) { - var_dump($redis, $channel, $message); - - $message = 'Message from worker #'.$workerId.'; '.$message; - - foreach($server->connections as $fd) { - if ($server->exist($fd) && $server->isEstablished($fd)) { - $server->push($fd, $message, SWOOLE_WEBSOCKET_OPCODE_TEXT, SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); + if($attempts > 0) { + Console::error('Connection lost (lasted '.(time() - $start).' seconds). Attempting restart (attempt #'.$attempts.')'); } - else { - $server->close($fd); + + if($redis->ping('')) { + $attempts = 0; } + + sleep(1); // 1 sec delay between connection attempts + + $redis->subscribe(['realtime'], function($redis, $channel, $message) use ($server, $workerId, &$connections) { + $message = 'Message from worker #'.$workerId.'; '.$message; + + Console::warning('Total connections: '.count($connections)); + + foreach($connections as $fd) { + if ($server->exist($fd) + && $server->isEstablished($fd) + ) { + Console::info('Sending message: '.$message.' (user: '.$fd.', worker: '.$workerId.')'); + + $server->push($fd, $message, SWOOLE_WEBSOCKET_OPCODE_TEXT, + SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); + } + else { + $server->close($fd); + } + } + }); + + $attempts++; + + } catch (\Throwable $th) { + $attempts++; + continue; } - }); + } + + Console::error('Failed to restart connection...'); }); -$server->on('BeforeReload', function($serv, $workerId) { - Console::success('Starting reload...'); -}); - -$server->on('AfterReload', function($serv, $workerId) { - Console::success('Reload completed...'); -}); - -// $process = new Process(function($process) use ($server) { -// while (true) { -// $msg = $process->read(); - -// foreach($server->connections as $fd) { -// if ($server->exist($fd) && $server->isEstablished($fd)) { -// $server->push($fd, json_encode(['hey there']), SWOOLE_WEBSOCKET_OPCODE_TEXT, SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); -// } -// } - -// sleep(10); -// } -// }); - -// $server->addProcess($process); - $server->on("start", function (Server $server) { Console::success('Server started succefully'); }); -$server->on('open', function(Server $server, Swoole\Http\Request $request) { - echo "connection open: {$request->fd}\n"; +$server->on('open', function(Server $server, Request $request) use (&$connections) { + $connections[] = $request->fd; + + Console::info("Connection open (user: {$request->fd}, worker: {$server->getWorkerId()})"); + Console::info('Total connections: '.count($connections)); - foreach($server->connections as $fd) { - if ($server->exist($fd) && $server->isEstablished($fd)) { - $server->push($fd, json_encode(['hey there', count($server->ports[0]->connections), ]), SWOOLE_WEBSOCKET_OPCODE_TEXT, SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); - } - } - - $server->push($request->fd, json_encode(["hello", time()])); + $server->push($request->fd, json_encode(["hello", count($connections)])); }); $server->on('message', function(Server $server, Frame $frame) { - echo "received message: {$frame->data}\n"; + if($frame->data === 'reload') { + $server->reload(); + } + + Console::info('Recieved message: '.$frame->data.' (user: '.$frame->fd.', worker: '.$server->getWorkerId().')'); + $server->push($frame->fd, json_encode(["hello, worker_id:".$server->getWorkerId(), time()])); }); $server->on('close', function(Server $server, int $fd) { - echo "connection close: {$fd}\n"; + Console::error('Connection close: '.$fd); }); $server->start(); \ No newline at end of file From a6cb86639b2ba71ae4a765b9dabacd03c6af58b5 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 19 Oct 2020 21:56:02 +0300 Subject: [PATCH 10/56] Added logs --- app/realtime.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/realtime.php b/app/realtime.php index 90416aa17..40ee8f572 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -46,6 +46,10 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { if($redis->ping('')) { $attempts = 0; + Console::success('Connection established'); + } + else { + Console::error('Connection failed'); } sleep(1); // 1 sec delay between connection attempts From a52de551a1070831eec037f4f245a1f66c2b2c0b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 19 Oct 2020 23:38:49 +0300 Subject: [PATCH 11/56] Fixed connection retry --- app/realtime.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 40ee8f572..5bfb207bd 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -34,17 +34,18 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { $attempts = 0; $start = time(); - while ($attempts < 3) { + while ($attempts < 300) { try { + if($attempts > 0) { + Console::error('Connection lost (lasted '.(time() - $start).' seconds). Attempting restart in 5 seconds (attempt #'.$attempts.')'); + sleep(5); // 1 sec delay between connection attempts + } + $redis = new Redis(); $redis->connect('redis', 6379); $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - if($attempts > 0) { - Console::error('Connection lost (lasted '.(time() - $start).' seconds). Attempting restart (attempt #'.$attempts.')'); - } - - if($redis->ping('')) { + if($redis->ping(true)) { $attempts = 0; Console::success('Connection established'); } @@ -52,8 +53,6 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { Console::error('Connection failed'); } - sleep(1); // 1 sec delay between connection attempts - $redis->subscribe(['realtime'], function($redis, $channel, $message) use ($server, $workerId, &$connections) { $message = 'Message from worker #'.$workerId.'; '.$message; @@ -74,12 +73,13 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { } }); - $attempts++; - } catch (\Throwable $th) { + Console::error('Connection error: '.$th->getMessage()); $attempts++; continue; } + + $attempts++; } Console::error('Failed to restart connection...'); From 4ccab4f9554f5a799cb41f484a801ccac57b0322 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 20 Oct 2020 16:22:46 +0300 Subject: [PATCH 12/56] POC --- app/realtime.php | 27 +++++++++++++++++++++------ docker-compose.yml | 7 +++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 5bfb207bd..cbe1da219 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -4,8 +4,11 @@ require_once __DIR__.'/../vendor/autoload.php'; use Swoole\WebSocket\Server; use Swoole\Http\Request; +use Swoole\Process; use Swoole\WebSocket\Frame; +use Utopia\App; use Utopia\CLI\Console; +use Utopia\Route; /** * TODO List @@ -16,9 +19,11 @@ use Utopia\CLI\Console; * - Message structure: { status: "ok"|"error", event: EVENT_NAME, data: } * - JWT Authentication (in path / or in message) * - * - * - https://github.com/hhxsv5/php-sse - * - https://github.com/shuixn/socket.io-swoole-server + * Protocols Support: + * - Websocket support: https://www.swoole.co.uk/docs/modules/swoole-websocket-server + * - MQTT support: https://www.swoole.co.uk/docs/modules/swoole-mqtt-server + * - SSE support: https://github.com/hhxsv5/php-sse + * - Socket.io support: https://github.com/shuixn/socket.io-swoole-server */ ini_set('default_socket_timeout', -1); @@ -55,9 +60,7 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { $redis->subscribe(['realtime'], function($redis, $channel, $message) use ($server, $workerId, &$connections) { $message = 'Message from worker #'.$workerId.'; '.$message; - - Console::warning('Total connections: '.count($connections)); - + foreach($connections as $fd) { if ($server->exist($fd) && $server->isEstablished($fd) @@ -87,6 +90,14 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { $server->on("start", function (Server $server) { Console::success('Server started succefully'); + + Console::info("Master pid {$server->master_pid}, manager pid {$server->manager_pid}"); + + // listen ctrl + c + Process::signal(2, function () use ($server) { + Console::log('Stop by Ctrl+C'); + $server->shutdown(); + }); }); $server->on('open', function(Server $server, Request $request) use (&$connections) { @@ -95,6 +106,10 @@ $server->on('open', function(Server $server, Request $request) use (&$connection Console::info("Connection open (user: {$request->fd}, worker: {$server->getWorkerId()})"); Console::info('Total connections: '.count($connections)); + $app = new App('Asia/Tel_Aviv'); + + var_dump($app->getResource('user')); + $server->push($request->fd, json_encode(["hello", count($connections)])); }); diff --git a/docker-compose.yml b/docker-compose.yml index 4a8c02c64..11c66d75b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -124,8 +124,15 @@ services: - redis environment: - _APP_ENV + - _APP_OPTIONS_ABUSE + - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS appwrite-worker-usage: entrypoint: worker-usage From f1df0e7071fba01c7d145f62753735aeafd08ca0 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 21 Oct 2020 13:49:56 +0300 Subject: [PATCH 13/56] Updated FW version --- app/controllers/web/home.php | 2 +- composer.json | 2 +- composer.lock | 36 ++++++++++++++++++------------------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/app/controllers/web/home.php b/app/controllers/web/home.php index a6c2491c0..781abd89a 100644 --- a/app/controllers/web/home.php +++ b/app/controllers/web/home.php @@ -437,7 +437,7 @@ App::get('/open-api-2.json') ]; foreach ($route->getParams() as $name => $param) { - $validator = (\is_callable($param['validator'])) ? call_user_func_array($param['validator'], $utopia->getResources($param['resources'])) : $param['validator']; /* @var $validator \Utopia\Validator */ + $validator = (\is_callable($param['validator'])) ? call_user_func_array($param['validator'], App::getResources($param['resources'])) : $param['validator']; /* @var $validator \Utopia\Validator */ $node = [ 'name' => $name, diff --git a/composer.json b/composer.json index c11009687..e1df44b44 100644 --- a/composer.json +++ b/composer.json @@ -33,7 +33,7 @@ "appwrite/php-clamav": "1.0.*", - "utopia-php/framework": "0.9.1", + "utopia-php/framework": "0.9.2", "utopia-php/abuse": "0.2.*", "utopia-php/audit": "0.3.*", "utopia-php/cache": "0.2.*", diff --git a/composer.lock b/composer.lock index 922593c99..f13d8d1bb 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "19f7cef86ddc98623cd3ffffa2be2cae", + "content-hash": "f51a86b9521acc0e934814d05b2927d2", "packages": [ { "name": "appwrite/php-clamav", @@ -514,12 +514,12 @@ "source": { "type": "git", "url": "https://github.com/guzzle/promises.git", - "reference": "60d379c243457e073cff02bc323a2a86cb355631" + "reference": "ddfeedfff2a52661429437da0702979f708e6ac6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/promises/zipball/60d379c243457e073cff02bc323a2a86cb355631", - "reference": "60d379c243457e073cff02bc323a2a86cb355631", + "url": "https://api.github.com/repos/guzzle/promises/zipball/ddfeedfff2a52661429437da0702979f708e6ac6", + "reference": "ddfeedfff2a52661429437da0702979f708e6ac6", "shasum": "" }, "require": { @@ -557,7 +557,7 @@ "keywords": [ "promise" ], - "time": "2020-09-30T07:37:28+00:00" + "time": "2020-10-19T16:50:15+00:00" }, { "name": "guzzlehttp/psr7", @@ -1522,23 +1522,23 @@ }, { "name": "utopia-php/framework", - "version": "0.9.1", + "version": "0.9.2", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "1c33b92b9188fb11b2ae70a4b7cf51844e07aa2e" + "reference": "82d1d0c913f0002e79557ae2ac9da991b2256e4d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/1c33b92b9188fb11b2ae70a4b7cf51844e07aa2e", - "reference": "1c33b92b9188fb11b2ae70a4b7cf51844e07aa2e", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/82d1d0c913f0002e79557ae2ac9da991b2256e4d", + "reference": "82d1d0c913f0002e79557ae2ac9da991b2256e4d", "shasum": "" }, "require": { - "php": ">=7.0.0" + "php": ">=7.3.0" }, "require-dev": { - "phpunit/phpunit": "^7.0" + "phpunit/phpunit": "^9.4" }, "type": "library", "autoload": { @@ -1562,7 +1562,7 @@ "php", "upf" ], - "time": "2020-09-09T19:50:26+00:00" + "time": "2020-10-21T04:58:48+00:00" }, { "name": "utopia-php/locale", @@ -1713,7 +1713,7 @@ "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator", - "reference": "a57b3cd56c4bfe1538276cfc77456cf95d8835cb" + "reference": "ad1ee55f61967546c0889d377b628e244182311e" }, "require": { "ext-curl": "*", @@ -1743,7 +1743,7 @@ } ], "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", - "time": "2020-10-14T12:07:25+00:00" + "time": "2020-10-20T10:23:43+00:00" }, { "name": "doctrine/instantiator", @@ -2646,12 +2646,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "338bf27b4510498c4b0ab70c7cbc292a591dc0df" + "reference": "0534ed70e3a792e4891f32ff78055648dfac4a3d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/338bf27b4510498c4b0ab70c7cbc292a591dc0df", - "reference": "338bf27b4510498c4b0ab70c7cbc292a591dc0df", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/0534ed70e3a792e4891f32ff78055648dfac4a3d", + "reference": "0534ed70e3a792e4891f32ff78055648dfac4a3d", "shasum": "" }, "require": { @@ -2737,7 +2737,7 @@ "type": "github" } ], - "time": "2020-10-19T09:25:00+00:00" + "time": "2020-10-21T04:49:22+00:00" }, { "name": "sebastian/cli-parser", From 619781a4ad567b6def0add82e8a9fcf9cb48f83a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 21 Oct 2020 13:50:11 +0300 Subject: [PATCH 14/56] Handle user & project channels --- app/init.php | 9 +++++---- app/realtime.php | 41 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/app/init.php b/app/init.php index c6a578eba..7f5fe1e3d 100644 --- a/app/init.php +++ b/app/init.php @@ -380,10 +380,10 @@ App::setResource('user', function($mode, $project, $console, $request, $response $request->getHeader('x-appwrite-key', '')))); // Get API Key // Get fallback session from clients who block 3rd-party cookies - $response->addHeader('X-Debug-Fallback', 'false'); + if($response) $response->addHeader('X-Debug-Fallback', 'false'); if(empty($session['id']) && empty($session['secret'])) { - $response->addHeader('X-Debug-Fallback', 'true'); + if($response) $response->addHeader('X-Debug-Fallback', 'true'); $fallback = $request->getHeader('x-fallback-cookies', ''); $fallback = \json_decode($fallback, true); $session = Auth::decodeSession(((isset($fallback[Auth::$cookieName])) ? $fallback[Auth::$cookieName] : '')); @@ -397,7 +397,7 @@ App::setResource('user', function($mode, $project, $console, $request, $response } else { $user = $consoleDB->getDocument(Auth::$unique); - + $user ->setAttribute('$id', 'admin-'.$user->getAttribute('$id')) ; @@ -412,7 +412,8 @@ App::setResource('user', function($mode, $project, $console, $request, $response if (APP_MODE_ADMIN === $mode) { if (!empty($user->search('teamId', $project->getAttribute('teamId'), $user->getAttribute('memberships')))) { Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users. - } else { + } + else { $user = new Document(['$id' => '', '$collection' => Database::SYSTEM_COLLECTION_USERS]); } } diff --git a/app/realtime.php b/app/realtime.php index cbe1da219..497d71a8e 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,7 +1,9 @@ on("workerStart", function ($server, $workerId) use (&$connections) { Console::success('Worker '.++$workerId.' started succefully'); @@ -71,7 +74,7 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); } else { - $server->close($fd); + $server->close($fd); } } }); @@ -106,11 +109,43 @@ $server->on('open', function(Server $server, Request $request) use (&$connection Console::info("Connection open (user: {$request->fd}, worker: {$server->getWorkerId()})"); Console::info('Total connections: '.count($connections)); + $connection = $request->fd; $app = new App('Asia/Tel_Aviv'); + $request = new SwooleRequest($request); - var_dump($app->getResource('user')); + App::setResource('request', function () use ($request) { + return $request; + }); - $server->push($request->fd, json_encode(["hello", count($connections)])); + App::setResource('response', function () { + return null; + }); + + $user = App::getResource('user'); + $project = App::getResource('project'); + + /** @var Appwrite\Database\Document $user */ + /** @var Appwrite\Database\Document $project */ + + var_dump($project->getId()); + var_dump($project->getAttribute('name')); + var_dump($user->getId()); + var_dump($user->getAttribute('name')); + + if(!isset($subscriptions[$project->getId()])) { // Init Project + $subscriptions[$project->getId()] = []; + } + + if(isset($subscriptions[$project->getId()][$user->getId()])) { // Close previous connection + $server->close($subscriptions[$project->getId()][$user->getId()]['connection']); + } + + $subscriptions[$project->getId()][$user->getId()] = [ + 'channels' => [], + 'connection' => $connection, + ]; + + $server->push($connection, json_encode(["hello", count($connections)])); }); $server->on('message', function(Server $server, Frame $frame) { From c491b781144327f5a76f70da4584c18838abf433 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 21 Oct 2020 15:03:50 +0300 Subject: [PATCH 15/56] Added connection open/close logic --- app/realtime.php | 85 +++++++++++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 497d71a8e..48ea45092 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -32,11 +32,10 @@ ini_set('default_socket_timeout', -1); Swoole\Runtime::enableCoroutine(SWOOLE_HOOK_ALL); $server = new Server("0.0.0.0", 80); - -$connections = []; $subscriptions = []; +$connections = []; -$server->on("workerStart", function ($server, $workerId) use (&$connections) { +$server->on("workerStart", function ($server, $workerId) use (&$subscriptions) { Console::success('Worker '.++$workerId.' started succefully'); $attempts = 0; @@ -45,7 +44,7 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { while ($attempts < 300) { try { if($attempts > 0) { - Console::error('Connection lost (lasted '.(time() - $start).' seconds). Attempting restart in 5 seconds (attempt #'.$attempts.')'); + Console::error('Pub/sub connection lost (lasted '.(time() - $start).' seconds). Attempting restart in 5 seconds (attempt #'.$attempts.')'); sleep(5); // 1 sec delay between connection attempts } @@ -55,32 +54,32 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { if($redis->ping(true)) { $attempts = 0; - Console::success('Connection established'); + Console::success('Pub/sub connection established'); } else { - Console::error('Connection failed'); + Console::error('Pub/sub failed'); } - $redis->subscribe(['realtime'], function($redis, $channel, $message) use ($server, $workerId, &$connections) { + $redis->subscribe(['realtime'], function($redis, $channel, $message) use ($server, $workerId) { $message = 'Message from worker #'.$workerId.'; '.$message; - foreach($connections as $fd) { - if ($server->exist($fd) - && $server->isEstablished($fd) - ) { - Console::info('Sending message: '.$message.' (user: '.$fd.', worker: '.$workerId.')'); + // foreach($connections as $fd) { + // if ($server->exist($fd) + // && $server->isEstablished($fd) + // ) { + // Console::info('Sending message: '.$message.' (user: '.$fd.', worker: '.$workerId.')'); - $server->push($fd, $message, SWOOLE_WEBSOCKET_OPCODE_TEXT, - SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); - } - else { - $server->close($fd); - } - } + // $server->push($fd, $message, SWOOLE_WEBSOCKET_OPCODE_TEXT, + // SWOOLE_WEBSOCKET_FLAG_FIN | SWOOLE_WEBSOCKET_FLAG_COMPRESS); + // } + // else { + // $server->close($fd); + // } + // } }); } catch (\Throwable $th) { - Console::error('Connection error: '.$th->getMessage()); + Console::error('Pub/sub error: '.$th->getMessage()); $attempts++; continue; } @@ -88,7 +87,7 @@ $server->on("workerStart", function ($server, $workerId) use (&$connections) { $attempts++; } - Console::error('Failed to restart connection...'); + Console::error('Failed to restart pub/sub...'); }); $server->on("start", function (Server $server) { @@ -103,14 +102,10 @@ $server->on("start", function (Server $server) { }); }); -$server->on('open', function(Server $server, Request $request) use (&$connections) { - $connections[] = $request->fd; - +$server->on('open', function(Server $server, Request $request) use (&$connections, &$subscriptions) { Console::info("Connection open (user: {$request->fd}, worker: {$server->getWorkerId()})"); - Console::info('Total connections: '.count($connections)); $connection = $request->fd; - $app = new App('Asia/Tel_Aviv'); $request = new SwooleRequest($request); App::setResource('request', function () use ($request) { @@ -121,6 +116,7 @@ $server->on('open', function(Server $server, Request $request) use (&$connection return null; }); + $channels = array_flip($request->getQuery('channels', [])); $user = App::getResource('user'); $project = App::getResource('project'); @@ -136,16 +132,20 @@ $server->on('open', function(Server $server, Request $request) use (&$connection $subscriptions[$project->getId()] = []; } - if(isset($subscriptions[$project->getId()][$user->getId()])) { // Close previous connection - $server->close($subscriptions[$project->getId()][$user->getId()]['connection']); + if(!isset($subscriptions[$project->getId()][$user->getId()])) { // Add user first connection + $subscriptions[$project->getId()][$user->getId()] = []; } - $subscriptions[$project->getId()][$user->getId()] = [ - 'channels' => [], - 'connection' => $connection, + foreach ($channels as $channel => $list) { + $subscriptions[$project->getId()][$user->getId()][$channel][$connection] = true; + } + + $connections[$connection] = [ + 'projectId' => $project->getId(), + 'userId' => $user->getId() ]; - $server->push($connection, json_encode(["hello", count($connections)])); + $server->push($connection, json_encode($subscriptions)); }); $server->on('message', function(Server $server, Frame $frame) { @@ -158,8 +158,27 @@ $server->on('message', function(Server $server, Frame $frame) { $server->push($frame->fd, json_encode(["hello, worker_id:".$server->getWorkerId(), time()])); }); -$server->on('close', function(Server $server, int $fd) { +$server->on('close', function(Server $server, int $fd) use (&$connections, &$subscriptions) { Console::error('Connection close: '.$fd); + + $projectId = $connections[$fd]['projectId'] ?? ''; + $userId = $connections[$fd]['userId'] ?? ''; + + foreach ($subscriptions[$projectId][$userId] as $channel => $list) { + unset($subscriptions[$projectId][$userId][$channel][$fd]); // Remove connection + + if(empty($list)) { + unset($subscriptions[$projectId][$userId][$channel]); // Remove channel + } + } + + if(empty($subscriptions[$projectId][$userId])) { + unset($subscriptions[$projectId][$userId]); // Remove user + } + + unset($connections[$fd]); + + var_dump($subscriptions); }); $server->start(); \ No newline at end of file From 437f5f1eb9cceafd98fb32d4285759fb1c92c5df Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 21 Oct 2020 18:03:18 +0300 Subject: [PATCH 16/56] Updated FW --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index e1df44b44..9af01f840 100644 --- a/composer.json +++ b/composer.json @@ -33,7 +33,7 @@ "appwrite/php-clamav": "1.0.*", - "utopia-php/framework": "0.9.2", + "utopia-php/framework": "0.9.4", "utopia-php/abuse": "0.2.*", "utopia-php/audit": "0.3.*", "utopia-php/cache": "0.2.*", From dabff52271deac01baac2af7cbe3ccd11469ba95 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 21 Oct 2020 18:03:33 +0300 Subject: [PATCH 17/56] Updated spec --- docs/specs/realtime.drawio.svg | 503 ++++++++++++++++++++++++++++++++- 1 file changed, 502 insertions(+), 1 deletion(-) diff --git a/docs/specs/realtime.drawio.svg b/docs/specs/realtime.drawio.svg index 93788ddd2..e2f07ef7f 100644 --- a/docs/specs/realtime.drawio.svg +++ b/docs/specs/realtime.drawio.svg @@ -1 +1,502 @@ -
Appwrite API
Appwrite API
Send all events
Send all events
Messaging Worker
Messaging Worker
Redis - Message Log
Redis - Message Log
Expire Every x seconds
Expire Every x secon...
Websocket 1
Websocket 1
Connections
Connections
Events
Events
Authorization
Authorization
Websocket 2
Websocket 2
Websocket 3
Websocket 3
Pull
Pull
Connections
Connections
Events
Events
Authorization
Authorization
Connections
Connections
Events
Events
Authorization
Authorization
Clients
Clients
LoadBalancer
LoadBalancer

Real Time
Specification for adding, a sclabale, realtime stream to trasmit Appwrite system and user generated events to any platform.

Real Time...Viewer does not support full SVG 1.1 \ No newline at end of file + + + + + + + + + + + +
+
+
+ Appwrite API +
+
+
+ + + Appwrite API + + + + + + + +
+
+
+ Send all events +
+
+
+ + + Send all events + + + + + + + + + +
+
+
+ Messaging Worker +
+
+
+ + + Messaging Worker + + + + + + + +
+
+
+ Redis - PUB/SUB +
+
+
+ + + Redis - PUB/SUB + + + + + + + + + + + +
+
+
+ Websocket 1 +
+
+
+ + + Websocket 1 + + + + + + + + + +
+
+
+ Connections +
+
+
+ + + Connections + + + + + + + + + +
+
+
+ Events +
+
+
+ + + Events + + + + + + + +
+
+
+ Authorization +
+
+
+ + + Authorization + + + + + + + + + + + +
+
+
+ Websocket 2 +
+
+
+ + + Websocket 2 + + + + + + + + + + + +
+
+
+ Websocket 3 +
+
+
+ + + Websocket 3 + + + + + + + +
+
+
+ Pull +
+
+
+ + + Pull + + + + + + + +
+
+
+ Connections +
+
+
+ + + Connections + + + + + + + +
+
+
+ Events +
+
+
+ + + Events + + + + + + + +
+
+
+ Authorization +
+
+
+ + + Authorization + + + + + + + +
+
+
+ Connections +
+
+
+ + + Connections + + + + + + + +
+
+
+ Events +
+
+
+ + + Events + + + + + + + +
+
+
+ Authorization +
+
+
+ + + Authorization + + + + + + + + + + + + + + + + + +
+
+
+ Clients +
+
+
+ + + Clients + + + + + + + + + + + + + +
+
+
+ LoadBalancer +
+
+
+ + + LoadBalancer + + + + + + + +
+
+
+

+ + Real Time +
+ + + Specification for adding, a sclabale, realtime stream to trasmit Appwrite system and user generated events to any platform. + +
+

+
+
+
+ + + Real Time... + + + + + + + +
+
+
+ Pull +
+
+
+ + + Pull + + + + + + + +
+
+
+ Pull +
+
+
+ + + Pull + + + + + + + +
+
+
+ Websocket +
+
+
+ + + Websocket + + + + + + + +
+
+
+ MQTT +
+
+
+ + + MQTT + + + + + + + +
+
+
+ Socket.io +
+
+
+ + + Socket.io + + + + + + + +
+
+
+ SSE +
+
+
+ + + SSE + + + + + + + + + Viewer does not support full SVG 1.1 + + + + \ No newline at end of file From f4f66256f4d74ba9f11401855d38f81be6e2650b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Thu, 22 Oct 2020 10:16:40 +0300 Subject: [PATCH 18/56] Now tracking all user roles --- app/realtime.php | 58 +++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 48ea45092..9c3927cc8 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -3,6 +3,7 @@ require_once __DIR__.'/init.php'; require_once __DIR__.'/../vendor/autoload.php'; +use Appwrite\Auth\Auth; use Appwrite\Swoole\Request as SwooleRequest; use Swoole\WebSocket\Server; use Swoole\Http\Request; @@ -44,7 +45,8 @@ $server->on("workerStart", function ($server, $workerId) use (&$subscriptions) { while ($attempts < 300) { try { if($attempts > 0) { - Console::error('Pub/sub connection lost (lasted '.(time() - $start).' seconds). Attempting restart in 5 seconds (attempt #'.$attempts.')'); + Console::error('Pub/sub connection lost (lasted '.(time() - $start).' seconds, worker: '.$workerId.'). + Attempting restart in 5 seconds (attempt #'.$attempts.')'); sleep(5); // 1 sec delay between connection attempts } @@ -54,10 +56,10 @@ $server->on("workerStart", function ($server, $workerId) use (&$subscriptions) { if($redis->ping(true)) { $attempts = 0; - Console::success('Pub/sub connection established'); + Console::success('Pub/sub connection established (worker: '.$workerId.')'); } else { - Console::error('Pub/sub failed'); + Console::error('Pub/sub failed (worker: '.$workerId.')'); } $redis->subscribe(['realtime'], function($redis, $channel, $message) use ($server, $workerId) { @@ -105,6 +107,7 @@ $server->on("start", function (Server $server) { $server->on('open', function(Server $server, Request $request) use (&$connections, &$subscriptions) { Console::info("Connection open (user: {$request->fd}, worker: {$server->getWorkerId()})"); + $app = new App(''); $connection = $request->fd; $request = new SwooleRequest($request); @@ -117,8 +120,9 @@ $server->on('open', function(Server $server, Request $request) use (&$connection }); $channels = array_flip($request->getQuery('channels', [])); - $user = App::getResource('user'); - $project = App::getResource('project'); + $user = $app->getResource('user'); + $project = $app->getResource('project'); + $roles = ['user:'.$user->getId(), 'role:'.(($user->isEmpty()) ? Auth::USER_ROLE_GUEST : Auth::USER_ROLE_MEMBER)]; /** @var Appwrite\Database\Document $user */ /** @var Appwrite\Database\Document $project */ @@ -128,21 +132,33 @@ $server->on('open', function(Server $server, Request $request) use (&$connection var_dump($user->getId()); var_dump($user->getAttribute('name')); + \array_map(function ($node) use (&$roles) { + if (isset($node['teamId']) && isset($node['roles'])) { + $roles[] = 'team:'.$node['teamId']; + + foreach ($node['roles'] as $nodeRole) { // Set all team roles + $roles[] = 'team:'.$node['teamId'].'/'.$nodeRole; + } + } + }, $user->getAttribute('memberships', [])); + if(!isset($subscriptions[$project->getId()])) { // Init Project $subscriptions[$project->getId()] = []; } - if(!isset($subscriptions[$project->getId()][$user->getId()])) { // Add user first connection - $subscriptions[$project->getId()][$user->getId()] = []; - } - - foreach ($channels as $channel => $list) { - $subscriptions[$project->getId()][$user->getId()][$channel][$connection] = true; + foreach ($roles as $key => $role) { + if(!isset($subscriptions[$project->getId()][$role])) { // Add user first connection + $subscriptions[$project->getId()][$role] = []; + } + + foreach ($channels as $channel => $list) { + $subscriptions[$project->getId()][$role][$channel][$connection] = true; + } } $connections[$connection] = [ 'projectId' => $project->getId(), - 'userId' => $user->getId() + 'roles' => $roles, ]; $server->push($connection, json_encode($subscriptions)); @@ -162,18 +178,20 @@ $server->on('close', function(Server $server, int $fd) use (&$connections, &$sub Console::error('Connection close: '.$fd); $projectId = $connections[$fd]['projectId'] ?? ''; - $userId = $connections[$fd]['userId'] ?? ''; + $roles = $connections[$fd]['roles'] ?? []; - foreach ($subscriptions[$projectId][$userId] as $channel => $list) { - unset($subscriptions[$projectId][$userId][$channel][$fd]); // Remove connection + foreach ($roles as $key => $role) { + foreach ($subscriptions[$projectId][$role] as $channel => $list) { + unset($subscriptions[$projectId][$role][$channel][$fd]); // Remove connection - if(empty($list)) { - unset($subscriptions[$projectId][$userId][$channel]); // Remove channel + if(empty($subscriptions[$projectId][$role][$channel])) { + unset($subscriptions[$projectId][$role][$channel]); // Remove channel + } } - } - if(empty($subscriptions[$projectId][$userId])) { - unset($subscriptions[$projectId][$userId]); // Remove user + if(empty($subscriptions[$projectId][$role])) { + unset($subscriptions[$projectId][$role]); // Remove role + } } unset($connections[$fd]); From c8be0a0e040a3f82c12c55c615c33ef55c80e778 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Thu, 22 Oct 2020 15:31:41 +0300 Subject: [PATCH 19/56] Updated Redis client --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 519d2a01c..d413f2a37 100755 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN composer update --ignore-platform-reqs --optimize-autoloader \ FROM php:7.4-cli-alpine as step1 ENV TZ=Asia/Tel_Aviv \ - PHP_REDIS_VERSION=5.3.2RC2 \ + PHP_REDIS_VERSION=5.3.2 \ PHP_SWOOLE_VERSION=4.5.5 \ PHP_XDEBUG_VERSION=sdebug_2_9-beta From 081943ce0350e319c9cce5d287b1bd6f59c5574b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 9 Jan 2021 23:58:21 +0200 Subject: [PATCH 20/56] Updated port --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 553b82c41..177dee3cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -125,9 +125,10 @@ services: context: . restart: unless-stopped ports: - - 9502:80 + - 9505:80 labels: - "traefik.enable=true" + - "traefik.constraint-label-stack=appwrite" - "traefik.docker.network=appwrite" - "traefik.http.services.appwrite_realtime.loadbalancer.server.port=80" #ws From 98dddd487ca6042ce3a7015732a08c3eb9fad11d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 29 Jan 2021 08:28:09 +0545 Subject: [PATCH 21/56] redis auth environment var --- app/config/variables.php | 8 ++++++++ app/init.php | 6 +++++- app/views/install/compose.phtml | 11 +++++++++++ docker-compose.yml | 11 +++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/app/config/variables.php b/app/config/variables.php index 011164db2..578e28a8f 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -133,6 +133,14 @@ return [ 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_REDIS_AUTH', + 'description' => 'Redis server auth.', + 'introduction' => '', + 'default' => '', + 'required' => false, + 'question' => '', + ], ], ], [ diff --git a/app/init.php b/app/init.php index 977efd94d..5feb354fc 100644 --- a/app/init.php +++ b/app/init.php @@ -91,7 +91,7 @@ Config::load('storage-inputs', __DIR__.'/config/storage/inputs.php'); Config::load('storage-outputs', __DIR__.'/config/storage/outputs.php'); Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '') - .':'.App::getEnv('_APP_REDIS_PORT', '')); + .':'.App::getEnv('_APP_REDIS_PORT', ''), 0, App::getEnv('_APP_REDIS_AUTH',null)); /** * DB Filters @@ -175,6 +175,10 @@ $register->set('statsd', function () { // Register DB connection $register->set('cache', function () { // Register cache connection $redis = new Redis(); $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); + $auth = App::getEnv('_APP_REDIS_AUTH',null); + if($auth != null) { + $redis->auth($auth); + } $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); return $redis; diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 9a182cc62..cbb2394f2 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -66,6 +66,7 @@ services: - _APP_DOMAIN_TARGET - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -101,6 +102,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_STATSD_HOST - _APP_STATSD_PORT @@ -118,6 +120,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -139,6 +142,7 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -160,6 +164,7 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -183,6 +188,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -207,6 +213,7 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -231,6 +238,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -259,6 +267,7 @@ services: - _APP_SYSTEM_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -279,6 +288,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_ABUSE @@ -298,6 +308,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH mariadb: image: appwrite/mariadb:1.2.0 # fix issues when upgrading using: mysql_upgrade -u root -p diff --git a/docker-compose.yml b/docker-compose.yml index 6366e2ad2..b64310486 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,6 +86,7 @@ services: - _APP_DOMAIN_TARGET - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -127,6 +128,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_STATSD_HOST - _APP_STATSD_PORT @@ -147,6 +149,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -172,6 +175,7 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -196,6 +200,7 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -222,6 +227,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -248,6 +254,7 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -275,6 +282,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -307,6 +315,7 @@ services: - _APP_SYSTEM_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -329,6 +338,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_ABUSE @@ -350,6 +360,7 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_REDIS_AUTH mariadb: image: appwrite/mariadb:1.2.0 # fix issues when upgrading using: mysql_upgrade -u root -p From 4023e5efcac565c9479398087c48eb9fcf85b717 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 31 Jan 2021 11:09:38 +0545 Subject: [PATCH 22/56] supporting redis auth --- app/init.php | 16 ++++++++++++---- bin/schedule | 11 ++++++++++- bin/worker-audits | 12 +++++++++++- bin/worker-certificates | 12 +++++++++++- bin/worker-deletes | 12 +++++++++++- bin/worker-functions | 12 +++++++++++- bin/worker-mails | 12 +++++++++++- bin/worker-tasks | 12 +++++++++++- bin/worker-usage | 12 +++++++++++- bin/worker-webhooks | 12 +++++++++++- 10 files changed, 110 insertions(+), 13 deletions(-) diff --git a/app/init.php b/app/init.php index 5feb354fc..660425637 100644 --- a/app/init.php +++ b/app/init.php @@ -90,9 +90,12 @@ Config::load('storage-mimes', __DIR__.'/config/storage/mimes.php'); Config::load('storage-inputs', __DIR__.'/config/storage/inputs.php'); Config::load('storage-outputs', __DIR__.'/config/storage/outputs.php'); -Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '') - .':'.App::getEnv('_APP_REDIS_PORT', ''), 0, App::getEnv('_APP_REDIS_AUTH',null)); - +$auth = App::getEnv('_APP_REDIS_AUTH',null); +if($auth != null) { + Resque::setBackend('redis://'.App::getEnv('_APP_REDIS_AUTH', '').'@'.App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', '')); +} else { + Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', '')); +} /** * DB Filters */ @@ -177,7 +180,12 @@ $register->set('cache', function () { // Register cache connection $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); $auth = App::getEnv('_APP_REDIS_AUTH',null); if($auth != null) { - $redis->auth($auth); + $auth = explode(':',$auth); + if(!empty(trim($auth[0]))) { + $redis->auth($auth); + }else{ + $redis->auth([$auth[1]]); + } } $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); diff --git a/bin/schedule b/bin/schedule index 74da3fe1e..c3162bc54 100644 --- a/bin/schedule +++ b/bin/schedule @@ -1,3 +1,12 @@ #!/bin/sh -REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" RESQUE_PHP='/usr/src/code/vendor/autoload.php' php /usr/src/code/vendor/bin/resque-scheduler \ No newline at end of file +if [ -z "$_APP_REDIS_AUTH" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +RESQUE_PHP='/usr/src/code/vendor/autoload.php' + +php /usr/src/code/vendor/bin/resque-scheduler \ No newline at end of file diff --git a/bin/worker-audits b/bin/worker-audits index 99b3eac2e..4ee658a6b 100644 --- a/bin/worker-audits +++ b/bin/worker-audits @@ -1,3 +1,13 @@ #!/bin/sh -QUEUE='v1-audits' APP_INCLUDE='/usr/src/code/app/workers/audits.php' REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +if [ -z "$_APP_REDIS_AUTH" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +QUEUE='v1-audits' +APP_INCLUDE='/usr/src/code/app/workers/audits.php' + +php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-certificates b/bin/worker-certificates index 6d360aff3..354da4d4b 100644 --- a/bin/worker-certificates +++ b/bin/worker-certificates @@ -1,3 +1,13 @@ #!/bin/sh -QUEUE='v1-certificates' APP_INCLUDE='/usr/src/code/app/workers/certificates.php' REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +if [ -z "$_APP_REDIS_AUTH" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +QUEUE='v1-certificates' +APP_INCLUDE='/usr/src/code/app/workers/certificates.php' + +php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-deletes b/bin/worker-deletes index d053381db..283b0058f 100644 --- a/bin/worker-deletes +++ b/bin/worker-deletes @@ -1,3 +1,13 @@ #!/bin/sh -QUEUE='v1-deletes' APP_INCLUDE='/usr/src/code/app/workers/deletes.php' REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +if [ -z "$_APP_REDIS_AUTH" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +QUEUE='v1-deletes' +APP_INCLUDE='/usr/src/code/app/workers/deletes.php' + +php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-functions b/bin/worker-functions index 787addab0..e62dd41ce 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,13 @@ #!/bin/sh -QUEUE='v1-functions' APP_INCLUDE='/usr/src/code/app/workers/functions.php' REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +if [ -z "$_APP_REDIS_AUTH" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +QUEUE='v1-functions' +APP_INCLUDE='/usr/src/code/app/workers/functions.php' + +php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-mails b/bin/worker-mails index 8a1a9de97..53e907bb3 100644 --- a/bin/worker-mails +++ b/bin/worker-mails @@ -1,3 +1,13 @@ #!/bin/sh -QUEUE='v1-mails' APP_INCLUDE='/usr/src/code/app/workers/mails.php' REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +if [ -z "$_APP_REDIS_AUTH" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +QUEUE='v1-mails' +APP_INCLUDE='/usr/src/code/app/workers/mails.php' + +php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-tasks b/bin/worker-tasks index 236b34cb7..d3c7b0123 100644 --- a/bin/worker-tasks +++ b/bin/worker-tasks @@ -1,3 +1,13 @@ #!/bin/sh -QUEUE='v1-tasks' APP_INCLUDE='/usr/src/code/app/workers/tasks.php' REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +if [ -z "$_APP_REDIS_AUTH" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +QUEUE='v1-tasks' +APP_INCLUDE='/usr/src/code/app/workers/tasks.php' + +php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-usage b/bin/worker-usage index 2b967f8f0..6e07542e3 100644 --- a/bin/worker-usage +++ b/bin/worker-usage @@ -1,3 +1,13 @@ #!/bin/sh -QUEUE='v1-usage' APP_INCLUDE='/usr/src/code/app/workers/usage.php' REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +if [ -z "$_APP_REDIS_AUTH" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +QUEUE='v1-usage' +APP_INCLUDE='/usr/src/code/app/workers/usage.php' + +php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-webhooks b/bin/worker-webhooks index d09a4a39a..ecd34c3cb 100644 --- a/bin/worker-webhooks +++ b/bin/worker-webhooks @@ -1,3 +1,13 @@ #!/bin/sh -QUEUE='v1-webhooks' APP_INCLUDE='/usr/src/code/app/workers/webhooks.php' REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +if [ -z "$_APP_REDIS_AUTH" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +QUEUE='v1-webhooks' +APP_INCLUDE='/usr/src/code/app/workers/webhooks.php' + +php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file From 26500f82efd667c9fc9a3a6038bcb9335feb51b8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 31 Jan 2021 20:41:06 +0545 Subject: [PATCH 23/56] updating redis auth environment --- app/config/variables.php | 14 +++++++++++--- app/init.php | 19 ++++++++++--------- app/views/install/compose.phtml | 33 ++++++++++++++++++++++----------- bin/schedule | 4 ++-- bin/worker-audits | 4 ++-- bin/worker-certificates | 4 ++-- bin/worker-deletes | 4 ++-- bin/worker-functions | 4 ++-- bin/worker-mails | 4 ++-- bin/worker-tasks | 4 ++-- bin/worker-usage | 4 ++-- bin/worker-webhooks | 4 ++-- docker-compose.yml | 33 ++++++++++++++++++++++----------- 13 files changed, 83 insertions(+), 52 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 578e28a8f..db0a66f65 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -134,9 +134,17 @@ return [ 'question' => '', ], [ - 'name' => '_APP_REDIS_AUTH', - 'description' => 'Redis server auth.', - 'introduction' => '', + 'name' => '_APP_REDIS_USER', + 'description' => 'Redis server user.', + 'introduction' => '0.7', + 'default' => '', + 'required' => false, + 'question' => '', + ], + [ + 'name' => '_APP_REDIS_PASS', + 'description' => 'Redis server password.', + 'introduction' => '0.7', 'default' => '', 'required' => false, 'question' => '', diff --git a/app/init.php b/app/init.php index 660425637..e9a81f8fe 100644 --- a/app/init.php +++ b/app/init.php @@ -90,9 +90,10 @@ Config::load('storage-mimes', __DIR__.'/config/storage/mimes.php'); Config::load('storage-inputs', __DIR__.'/config/storage/inputs.php'); Config::load('storage-outputs', __DIR__.'/config/storage/outputs.php'); -$auth = App::getEnv('_APP_REDIS_AUTH',null); -if($auth != null) { - Resque::setBackend('redis://'.App::getEnv('_APP_REDIS_AUTH', '').'@'.App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', '')); +$user = App::getEnv('_APP_REDIS_USER',''); +$pass = App::getEnv('_APP_REDIS_PASS',''); +if(!empty($pass)) { + Resque::setBackend('redis://'.$user.':'.$pass.'@'.App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', '')); } else { Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', '')); } @@ -178,13 +179,13 @@ $register->set('statsd', function () { // Register DB connection $register->set('cache', function () { // Register cache connection $redis = new Redis(); $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $auth = App::getEnv('_APP_REDIS_AUTH',null); - if($auth != null) { - $auth = explode(':',$auth); - if(!empty(trim($auth[0]))) { - $redis->auth($auth); + $user = App::getEnv('_APP_REDIS_USER','') ?? ''; + $pass = App::getEnv('_APP_REDIS_PASS','') ?? ''; + if(!empty($pass)) { + if(!empty(trim($user))) { + $redis->auth([$user,$pass]); }else{ - $redis->auth([$auth[1]]); + $redis->auth([$pass]); } } $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index cbb2394f2..e74f22274 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -66,7 +66,8 @@ services: - _APP_DOMAIN_TARGET - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -102,7 +103,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_STATSD_HOST - _APP_STATSD_PORT @@ -120,7 +122,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -142,7 +145,8 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -164,7 +168,8 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -188,7 +193,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -213,7 +219,8 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -238,7 +245,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -267,7 +275,8 @@ services: - _APP_SYSTEM_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -288,7 +297,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_ABUSE @@ -308,7 +318,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS mariadb: image: appwrite/mariadb:1.2.0 # fix issues when upgrading using: mysql_upgrade -u root -p diff --git a/bin/schedule b/bin/schedule index c3162bc54..c7a63c33b 100644 --- a/bin/schedule +++ b/bin/schedule @@ -1,10 +1,10 @@ #!/bin/sh -if [ -z "$_APP_REDIS_AUTH" ] +if [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else - REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi RESQUE_PHP='/usr/src/code/vendor/autoload.php' diff --git a/bin/worker-audits b/bin/worker-audits index 4ee658a6b..f5df25207 100644 --- a/bin/worker-audits +++ b/bin/worker-audits @@ -1,10 +1,10 @@ #!/bin/sh -if [ -z "$_APP_REDIS_AUTH" ] +if [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else - REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi QUEUE='v1-audits' diff --git a/bin/worker-certificates b/bin/worker-certificates index 354da4d4b..435895e7b 100644 --- a/bin/worker-certificates +++ b/bin/worker-certificates @@ -1,10 +1,10 @@ #!/bin/sh -if [ -z "$_APP_REDIS_AUTH" ] +if [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else - REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi QUEUE='v1-certificates' diff --git a/bin/worker-deletes b/bin/worker-deletes index 283b0058f..ec8f59818 100644 --- a/bin/worker-deletes +++ b/bin/worker-deletes @@ -1,10 +1,10 @@ #!/bin/sh -if [ -z "$_APP_REDIS_AUTH" ] +if [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else - REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi QUEUE='v1-deletes' diff --git a/bin/worker-functions b/bin/worker-functions index e62dd41ce..c27f2191a 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,10 +1,10 @@ #!/bin/sh -if [ -z "$_APP_REDIS_AUTH" ] +if [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else - REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi QUEUE='v1-functions' diff --git a/bin/worker-mails b/bin/worker-mails index 53e907bb3..e9396afd4 100644 --- a/bin/worker-mails +++ b/bin/worker-mails @@ -1,10 +1,10 @@ #!/bin/sh -if [ -z "$_APP_REDIS_AUTH" ] +if [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else - REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi QUEUE='v1-mails' diff --git a/bin/worker-tasks b/bin/worker-tasks index d3c7b0123..c6f87200d 100644 --- a/bin/worker-tasks +++ b/bin/worker-tasks @@ -1,10 +1,10 @@ #!/bin/sh -if [ -z "$_APP_REDIS_AUTH" ] +if [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else - REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi QUEUE='v1-tasks' diff --git a/bin/worker-usage b/bin/worker-usage index 6e07542e3..5f86bc3b1 100644 --- a/bin/worker-usage +++ b/bin/worker-usage @@ -1,10 +1,10 @@ #!/bin/sh -if [ -z "$_APP_REDIS_AUTH" ] +if [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else - REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi QUEUE='v1-usage' diff --git a/bin/worker-webhooks b/bin/worker-webhooks index ecd34c3cb..8a51947db 100644 --- a/bin/worker-webhooks +++ b/bin/worker-webhooks @@ -1,10 +1,10 @@ #!/bin/sh -if [ -z "$_APP_REDIS_AUTH" ] +if [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else - REDIS_BACKEND="redis://${_APP_REDIS_AUTH}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi QUEUE='v1-webhooks' diff --git a/docker-compose.yml b/docker-compose.yml index b64310486..347a89475 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,7 +86,8 @@ services: - _APP_DOMAIN_TARGET - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -128,7 +129,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_STATSD_HOST - _APP_STATSD_PORT @@ -149,7 +151,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -175,7 +178,8 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -200,7 +204,8 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -227,7 +232,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -254,7 +260,8 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -282,7 +289,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -315,7 +323,8 @@ services: - _APP_SYSTEM_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -338,7 +347,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_ABUSE @@ -360,7 +370,8 @@ services: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - - _APP_REDIS_AUTH + - _APP_REDIS_USER + - _APP_REDIS_PASS mariadb: image: appwrite/mariadb:1.2.0 # fix issues when upgrading using: mysql_upgrade -u root -p From 7ac826b355fdf54c62f2687850ef98edef0d9f19 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 1 Feb 2021 12:06:42 +0545 Subject: [PATCH 24/56] updating checks for both user and pass --- app/init.php | 19 +++++++++++-------- bin/schedule | 2 +- bin/worker-audits | 2 +- bin/worker-certificates | 2 +- bin/worker-deletes | 2 +- bin/worker-functions | 2 +- bin/worker-mails | 2 +- bin/worker-tasks | 2 +- bin/worker-usage | 2 +- bin/worker-webhooks | 2 +- 10 files changed, 20 insertions(+), 17 deletions(-) diff --git a/app/init.php b/app/init.php index e9a81f8fe..ad41f3cfe 100644 --- a/app/init.php +++ b/app/init.php @@ -92,7 +92,7 @@ Config::load('storage-outputs', __DIR__.'/config/storage/outputs.php'); $user = App::getEnv('_APP_REDIS_USER',''); $pass = App::getEnv('_APP_REDIS_PASS',''); -if(!empty($pass)) { +if(!empty($user) || !empty($pass)) { Resque::setBackend('redis://'.$user.':'.$pass.'@'.App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', '')); } else { Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '').':'.App::getEnv('_APP_REDIS_PORT', '')); @@ -179,14 +179,17 @@ $register->set('statsd', function () { // Register DB connection $register->set('cache', function () { // Register cache connection $redis = new Redis(); $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $user = App::getEnv('_APP_REDIS_USER','') ?? ''; - $pass = App::getEnv('_APP_REDIS_PASS','') ?? ''; + $user = App::getEnv('_APP_REDIS_USER',''); + $pass = App::getEnv('_APP_REDIS_PASS',''); + $auth = []; + if(!empty($user)) { + $auth["user"] = $user; + } if(!empty($pass)) { - if(!empty(trim($user))) { - $redis->auth([$user,$pass]); - }else{ - $redis->auth([$pass]); - } + $auth["pass"] = $pass; + } + if(!empty($auth)) { + $redis->auth($auth); } $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); diff --git a/bin/schedule b/bin/schedule index c7a63c33b..ed76c3e5b 100644 --- a/bin/schedule +++ b/bin/schedule @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "$_APP_REDIS_PASS" ] +if [ -z "$_APP_REDIS_USER" ] || [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else diff --git a/bin/worker-audits b/bin/worker-audits index f5df25207..57cad2685 100644 --- a/bin/worker-audits +++ b/bin/worker-audits @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "$_APP_REDIS_PASS" ] +if [ -z "$_APP_REDIS_USER" ] || [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else diff --git a/bin/worker-certificates b/bin/worker-certificates index 435895e7b..018a96e21 100644 --- a/bin/worker-certificates +++ b/bin/worker-certificates @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "$_APP_REDIS_PASS" ] +if [ -z "$_APP_REDIS_USER" ] || [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else diff --git a/bin/worker-deletes b/bin/worker-deletes index ec8f59818..61b95c778 100644 --- a/bin/worker-deletes +++ b/bin/worker-deletes @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "$_APP_REDIS_PASS" ] +if [ -z "$_APP_REDIS_USER" ] || [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else diff --git a/bin/worker-functions b/bin/worker-functions index c27f2191a..4c28ad119 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "$_APP_REDIS_PASS" ] +if [ -z "$_APP_REDIS_USER" ] || [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else diff --git a/bin/worker-mails b/bin/worker-mails index e9396afd4..71e41012c 100644 --- a/bin/worker-mails +++ b/bin/worker-mails @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "$_APP_REDIS_PASS" ] +if [ -z "$_APP_REDIS_USER" ] || [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else diff --git a/bin/worker-tasks b/bin/worker-tasks index c6f87200d..9d990efe1 100644 --- a/bin/worker-tasks +++ b/bin/worker-tasks @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "$_APP_REDIS_PASS" ] +if [ -z "$_APP_REDIS_USER" ] || [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else diff --git a/bin/worker-usage b/bin/worker-usage index 5f86bc3b1..92e9e576b 100644 --- a/bin/worker-usage +++ b/bin/worker-usage @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "$_APP_REDIS_PASS" ] +if [ -z "$_APP_REDIS_USER" ] || [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else diff --git a/bin/worker-webhooks b/bin/worker-webhooks index 8a51947db..495a1b680 100644 --- a/bin/worker-webhooks +++ b/bin/worker-webhooks @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "$_APP_REDIS_PASS" ] +if [ -z "$_APP_REDIS_USER" ] || [ -z "$_APP_REDIS_PASS" ] then REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" else From 19c870b2aa951cfaf1714723785d6500f373e211 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 1 Feb 2021 18:09:25 +0545 Subject: [PATCH 25/56] fix issue with QUEUE not being set --- bin/schedule | 4 +--- bin/worker-audits | 5 +---- bin/worker-certificates | 5 +---- bin/worker-deletes | 5 +---- bin/worker-functions | 5 +---- bin/worker-mails | 5 +---- bin/worker-tasks | 5 +---- bin/worker-usage | 5 +---- bin/worker-webhooks | 5 +---- 9 files changed, 9 insertions(+), 35 deletions(-) mode change 100644 => 100755 bin/worker-certificates diff --git a/bin/schedule b/bin/schedule index ed76c3e5b..08a9e9a08 100644 --- a/bin/schedule +++ b/bin/schedule @@ -7,6 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -RESQUE_PHP='/usr/src/code/vendor/autoload.php' - -php /usr/src/code/vendor/bin/resque-scheduler \ No newline at end of file +RESQUE_PHP='/usr/src/code/vendor/autoload.php' php /usr/src/code/vendor/bin/resque-scheduler \ No newline at end of file diff --git a/bin/worker-audits b/bin/worker-audits index 57cad2685..e411aa386 100644 --- a/bin/worker-audits +++ b/bin/worker-audits @@ -7,7 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-audits' -APP_INCLUDE='/usr/src/code/app/workers/audits.php' - -php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +QUEUE='v1-audits' APP_INCLUDE='/usr/src/code/app/workers/audits.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-certificates b/bin/worker-certificates old mode 100644 new mode 100755 index 018a96e21..7d474e1ea --- a/bin/worker-certificates +++ b/bin/worker-certificates @@ -7,7 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-certificates' -APP_INCLUDE='/usr/src/code/app/workers/certificates.php' - -php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +QUEUE='v1-certificates' APP_INCLUDE='/usr/src/code/app/workers/certificates.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-deletes b/bin/worker-deletes index 61b95c778..9c2fb3118 100644 --- a/bin/worker-deletes +++ b/bin/worker-deletes @@ -7,7 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-deletes' -APP_INCLUDE='/usr/src/code/app/workers/deletes.php' - -php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +QUEUE='v1-deletes' APP_INCLUDE='/usr/src/code/app/workers/deletes.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-functions b/bin/worker-functions index 4c28ad119..b41c01994 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -7,7 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-functions' -APP_INCLUDE='/usr/src/code/app/workers/functions.php' - -php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +QUEUE='v1-functions' APP_INCLUDE='/usr/src/code/app/workers/functions.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-mails b/bin/worker-mails index 71e41012c..b29eacdab 100644 --- a/bin/worker-mails +++ b/bin/worker-mails @@ -7,7 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-mails' -APP_INCLUDE='/usr/src/code/app/workers/mails.php' - -php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +QUEUE='v1-mails' APP_INCLUDE='/usr/src/code/app/workers/mails.php' _APP_REDIS_USERphp /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-tasks b/bin/worker-tasks index 9d990efe1..fc395b34a 100644 --- a/bin/worker-tasks +++ b/bin/worker-tasks @@ -7,7 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-tasks' -APP_INCLUDE='/usr/src/code/app/workers/tasks.php' - -php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +QUEUE='v1-tasks' APP_INCLUDE='/usr/src/code/app/workers/tasks.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-usage b/bin/worker-usage index 92e9e576b..4176f54b8 100644 --- a/bin/worker-usage +++ b/bin/worker-usage @@ -7,7 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-usage' -APP_INCLUDE='/usr/src/code/app/workers/usage.php' - -php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +QUEUE='v1-usage' APP_INCLUDE='/usr/src/code/app/workers/usage.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/bin/worker-webhooks b/bin/worker-webhooks index 495a1b680..13029ecbd 100644 --- a/bin/worker-webhooks +++ b/bin/worker-webhooks @@ -7,7 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-webhooks' -APP_INCLUDE='/usr/src/code/app/workers/webhooks.php' - -php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +QUEUE='v1-webhooks' APP_INCLUDE='/usr/src/code/app/workers/webhooks.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file From 3a7285a8454ebe81b9c14c2a5ce564eef5654b3a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 7 Feb 2021 13:42:15 +0545 Subject: [PATCH 26/56] fix worker schedule not starting issue --- bin/schedule | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/schedule b/bin/schedule index 08a9e9a08..857ec9f6f 100644 --- a/bin/schedule +++ b/bin/schedule @@ -7,4 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -RESQUE_PHP='/usr/src/code/vendor/autoload.php' php /usr/src/code/vendor/bin/resque-scheduler \ No newline at end of file +REDIS_BACKEND=$REDIS_BACKEND RESQUE_PHP='/usr/src/code/vendor/autoload.php' php /usr/src/code/vendor/bin/resque-scheduler From 0475d87a7297f3c56d62976cce0d94c8154e4bf9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 15 Feb 2021 13:22:13 +0545 Subject: [PATCH 27/56] fix issue with mails worker --- bin/worker-mails | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/worker-mails b/bin/worker-mails index b29eacdab..64d10f609 100644 --- a/bin/worker-mails +++ b/bin/worker-mails @@ -7,4 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-mails' APP_INCLUDE='/usr/src/code/app/workers/mails.php' _APP_REDIS_USERphp /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +QUEUE='v1-mails' APP_INCLUDE='/usr/src/code/app/workers/mails.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file From ebae23499fa377722dfc3b9cbebf9f02a91c2b4a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 17 Feb 2021 11:44:31 +0100 Subject: [PATCH 28/56] fixes function upload powershell snippet --- app/views/console/functions/function.phtml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/console/functions/function.phtml b/app/views/console/functions/function.phtml index 49e2fffc4..74960eba4 100644 --- a/app/views/console/functions/function.phtml +++ b/app/views/console/functions/function.phtml @@ -596,9 +596,9 @@ $usageStatsEnabled = $this->getParam('usageStatsEnabled',true);

PowerShell

-
From 92d35cff1841cc533fdda5ab39855a8cd1843b18 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 18 Feb 2021 17:42:30 +0545 Subject: [PATCH 29/56] fix missing port information in URL in console --- app/controllers/shared/web.php | 19 ++++++++++++------- app/views/console/database/collection.phtml | 2 +- .../console/database/search/documents.phtml | 2 +- app/views/console/database/search/files.phtml | 2 +- app/views/console/settings/index.phtml | 6 +++--- app/views/console/storage/index.phtml | 8 ++++---- app/views/console/users/index.phtml | 2 +- app/views/console/users/team.phtml | 2 +- app/views/home/auth/recovery.phtml | 2 +- app/views/layouts/default.phtml | 4 +++- 10 files changed, 28 insertions(+), 21 deletions(-) diff --git a/app/controllers/shared/web.php b/app/controllers/shared/web.php index 43eaf74d0..526a4dbb6 100644 --- a/app/controllers/shared/web.php +++ b/app/controllers/shared/web.php @@ -11,13 +11,18 @@ App::init(function ($utopia, $request, $response, $layout) { /* AJAX check */ if (!empty($request->getQuery('version', ''))) { - $layout->setPath(__DIR__.'/../../views/layouts/empty.phtml'); + $layout->setPath(__DIR__ . '/../../views/layouts/empty.phtml'); } - + + $port = $request->getPort(); + $protocol = $request->getProtocol(); + $domain = $request->getHostname(); + $layout ->setParam('title', APP_NAME) - ->setParam('protocol', $request->getProtocol()) - ->setParam('domain', $request->getHostname()) + ->setParam('protocol', $protocol) + ->setParam('domain', $domain) + ->setParam('endpoint', $protocol . '://' . $domain . ($port != 80 && $port != 443 ? ':' . $port : '')) ->setParam('home', App::getEnv('_APP_HOME')) ->setParam('setup', App::getEnv('_APP_SETUP')) ->setParam('class', 'unknown') @@ -34,10 +39,10 @@ App::init(function ($utopia, $request, $response, $layout) { $time = (60 * 60 * 24 * 45); // 45 days cache $response - ->addHeader('Cache-Control', 'public, max-age='.$time) - ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $time).' GMT') // 45 days cache + ->addHeader('Cache-Control', 'public, max-age=' . $time) + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $time) . ' GMT') // 45 days cache ->addHeader('X-Frame-Options', 'SAMEORIGIN') // Avoid console and homepage from showing in iframes - ->addHeader('X-XSS-Protection', '1; mode=block; report=/v1/xss?url='.\urlencode($request->getURI())) + ->addHeader('X-XSS-Protection', '1; mode=block; report=/v1/xss?url=' . \urlencode($request->getURI())) ->addHeader('X-UA-Compatible', 'IE=Edge') // Deny IE browsers from going into quirks mode ; diff --git a/app/views/console/database/collection.phtml b/app/views/console/database/collection.phtml index afe0365ee..45d3287cf 100644 --- a/app/views/console/database/collection.phtml +++ b/app/views/console/database/collection.phtml @@ -114,7 +114,7 @@ $maxCells = 10; - + {...} diff --git a/app/views/console/database/search/documents.phtml b/app/views/console/database/search/documents.phtml index 1b656dc4e..6fb67feab 100644 --- a/app/views/console/database/search/documents.phtml +++ b/app/views/console/database/search/documents.phtml @@ -72,7 +72,7 @@ $rules = $collection->getAttribute('rules', []); - + {...} diff --git a/app/views/console/database/search/files.phtml b/app/views/console/database/search/files.phtml index f8c39f4a8..ac9f76e0e 100644 --- a/app/views/console/database/search/files.phtml +++ b/app/views/console/database/search/files.phtml @@ -54,7 +54,7 @@ - + diff --git a/app/views/console/settings/index.phtml b/app/views/console/settings/index.phtml index 5eedb6388..45f039d38 100644 --- a/app/views/console/settings/index.phtml +++ b/app/views/console/settings/index.phtml @@ -135,7 +135,7 @@ $customDomainsTarget = $this->getParam('customDomainsTarget', false);
- +
    @@ -453,7 +453,7 @@ $customDomainsTarget = $this->getParam('customDomainsTarget', false); data-failure-param-alert-classname="error"> - + @@ -493,7 +493,7 @@ $customDomainsTarget = $this->getParam('customDomainsTarget', false); data-failure-param-alert-classname="error"> - + diff --git a/app/views/console/storage/index.phtml b/app/views/console/storage/index.phtml index 934189f93..bd3969959 100644 --- a/app/views/console/storage/index.phtml +++ b/app/views/console/storage/index.phtml @@ -113,7 +113,7 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0); - +
    @@ -180,15 +180,15 @@ $fileLimitHuman = $this->getParam('fileLimitHuman', 0);
    File Preview
    - +
    - New Window + New Window
    - Download + Download
    diff --git a/app/views/console/users/index.phtml b/app/views/console/users/index.phtml index 8debcd9b4..b41730baf 100644 --- a/app/views/console/users/index.phtml +++ b/app/views/console/users/index.phtml @@ -361,7 +361,7 @@ $providers = $this->getParam('providers', []);

    To complete set up, add this OAuth2 redirect URI to your escape(ucfirst($provider)); ?> app configuration.

    - +
    diff --git a/app/views/console/users/team.phtml b/app/views/console/users/team.phtml index 659fe3bd5..ad6d263fa 100644 --- a/app/views/console/users/team.phtml +++ b/app/views/console/users/team.phtml @@ -145,7 +145,7 @@ data-failure-param-alert-classname="error"> - + diff --git a/app/views/home/auth/recovery.phtml b/app/views/home/auth/recovery.phtml index 575227a15..6f633b9a0 100644 --- a/app/views/home/auth/recovery.phtml +++ b/app/views/home/auth/recovery.phtml @@ -23,7 +23,7 @@ - + diff --git a/app/views/layouts/default.phtml b/app/views/layouts/default.phtml index 1f0c68649..8c2641f18 100644 --- a/app/views/layouts/default.phtml +++ b/app/views/layouts/default.phtml @@ -2,6 +2,7 @@ $protocol = $this->getParam('protocol', ''); $domain = $this->getParam('domain', ''); +$endpoint = $this->getParam('endpoint', ''); $platforms = $this->getParam('platforms', []); $version = $this->getParam('version', '0.0.0'); $isDev = $this->getParam('isDev', false); @@ -56,7 +57,7 @@ if(!empty($platforms)) { - +