From 5201e488152b860501ab4e42db5ff8e4dc32dd80 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 7 Jun 2021 12:02:46 +0545 Subject: [PATCH 01/17] adding image crop gravity support --- app/controllers/api/storage.php | 5 +++-- composer.json | 2 +- composer.lock | 38 ++++++++++++++++----------------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index b3060eaf4b..d7b214dd22 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -242,6 +242,7 @@ App::get('/v1/storage/files/:fileId/preview') ->param('fileId', '', new UID(), 'File unique ID') ->param('width', 0, new Range(0, 4000), 'Resize preview image width, Pass an integer between 0 to 4000.', true) ->param('height', 0, new Range(0, 4000), 'Resize preview image height, Pass an integer between 0 to 4000.', true) + ->param('gravity', Image::GRAVITY_CENTER, new Range(0, 8), 'Image crop gravity', true) ->param('quality', 100, new Range(0, 100), 'Preview image quality. Pass an integer between 0 to 100. Defaults to 100.', true) ->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true) ->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true) @@ -254,7 +255,7 @@ App::get('/v1/storage/files/:fileId/preview') ->inject('response') ->inject('project') ->inject('projectDB') - ->action(function ($fileId, $width, $height, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $projectDB) { + ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $projectDB) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Appwrite\Database\Document $project */ @@ -342,7 +343,7 @@ App::get('/v1/storage/files/:fileId/preview') $image = new Image($source); - $image->crop((int) $width, (int) $height); + $image->crop((int) $width, (int) $height, (int) $gravity); if (!empty($opacity) || $opacity==0) { $image->setOpacity($opacity); diff --git a/composer.json b/composer.json index ee9942a9f3..c27cf8276a 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/domains": "1.1.*", "utopia-php/swoole": "0.2.*", "utopia-php/storage": "0.5.*", - "utopia-php/image": "0.2.*", + "utopia-php/image": "0.3.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "4.2.2", "dragonmantank/cron-expression": "3.1.0", diff --git a/composer.lock b/composer.lock index 1a89b21c8b..82b95e2d26 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9a72955402438e63ec6101723c33f544", + "content-hash": "2d32e708dd70ab32d06b912e74b1194a", "packages": [ { "name": "adhocore/jwt", @@ -1324,16 +1324,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.4.0", + "version": "0.4.1", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "2b8cc40a67c045c137b44d1a11326f494acf50a4" + "reference": "8b7973aae4b02489bd22ffea45b985608f13b6d9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/2b8cc40a67c045c137b44d1a11326f494acf50a4", - "reference": "2b8cc40a67c045c137b44d1a11326f494acf50a4", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/8b7973aae4b02489bd22ffea45b985608f13b6d9", + "reference": "8b7973aae4b02489bd22ffea45b985608f13b6d9", "shasum": "" }, "require": { @@ -1370,9 +1370,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.4.0" + "source": "https://github.com/utopia-php/abuse/tree/0.4.1" }, - "time": "2021-03-17T20:21:24+00:00" + "time": "2021-06-05T14:31:33+00:00" }, { "name": "utopia-php/analytics", @@ -1742,16 +1742,16 @@ }, { "name": "utopia-php/image", - "version": "0.2.1", + "version": "0.3.0", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "0754955a165483852184d1215cc3bf659432d23a" + "reference": "7761ff565e505bb3ddb9cfa05b7e1efddf3bebaa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/0754955a165483852184d1215cc3bf659432d23a", - "reference": "0754955a165483852184d1215cc3bf659432d23a", + "url": "https://api.github.com/repos/utopia-php/image/zipball/7761ff565e505bb3ddb9cfa05b7e1efddf3bebaa", + "reference": "7761ff565e505bb3ddb9cfa05b7e1efddf3bebaa", "shasum": "" }, "require": { @@ -1789,9 +1789,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.2.1" + "source": "https://github.com/utopia-php/image/tree/0.3.0" }, - "time": "2021-04-13T07:47:24+00:00" + "time": "2021-06-02T07:08:04+00:00" }, { "name": "utopia-php/locale", @@ -4823,16 +4823,16 @@ }, { "name": "sebastian/type", - "version": "2.3.1", + "version": "2.3.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/type.git", - "reference": "81cd61ab7bbf2de744aba0ea61fae32f721df3d2" + "reference": "0d1c587401514d17e8f9258a27e23527cb1b06c1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/81cd61ab7bbf2de744aba0ea61fae32f721df3d2", - "reference": "81cd61ab7bbf2de744aba0ea61fae32f721df3d2", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/0d1c587401514d17e8f9258a27e23527cb1b06c1", + "reference": "0d1c587401514d17e8f9258a27e23527cb1b06c1", "shasum": "" }, "require": { @@ -4867,7 +4867,7 @@ "homepage": "https://github.com/sebastianbergmann/type", "support": { "issues": "https://github.com/sebastianbergmann/type/issues", - "source": "https://github.com/sebastianbergmann/type/tree/2.3.1" + "source": "https://github.com/sebastianbergmann/type/tree/2.3.2" }, "funding": [ { @@ -4875,7 +4875,7 @@ "type": "github" } ], - "time": "2020-10-26T13:18:59+00:00" + "time": "2021-06-04T13:02:07+00:00" }, { "name": "sebastian/version", From 4b9be0f741d1bcfcfee863f12c6275bf4a5b8c67 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 8 Jun 2021 12:25:09 +0545 Subject: [PATCH 02/17] composer update --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 92e91a62a7..26b8c6c85b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6704f7df5ffe0baac3633dc8e683ed78", + "content-hash": "399d2426ca92e04b6d6fb84a91c316c3", "packages": [ { "name": "adhocore/jwt", From 4fb54ac87b2d793e8dbb88690bd71815cfc1abbd Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 9 Jun 2021 12:26:39 +0545 Subject: [PATCH 03/17] fix crop gravity param type --- app/controllers/api/storage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index d7b214dd22..f4f00cbea6 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -242,7 +242,7 @@ App::get('/v1/storage/files/:fileId/preview') ->param('fileId', '', new UID(), 'File unique ID') ->param('width', 0, new Range(0, 4000), 'Resize preview image width, Pass an integer between 0 to 4000.', true) ->param('height', 0, new Range(0, 4000), 'Resize preview image height, Pass an integer between 0 to 4000.', true) - ->param('gravity', Image::GRAVITY_CENTER, new Range(0, 8), 'Image crop gravity', true) + ->param('gravity', Image::GRAVITY_CENTER, new WhiteList(Image::GRAVITY_CENTER, Image::GRAVITY_NORTH, Image::GRAVITY_NORTHWEST, Image::GRAVITY_NORTHEAST, Image::GRAVITY_WEST, Image::GRAVITY_EAST, Image::GRAVITY_SOUTHWEST, Image::GRAVITY_SOUTH, Image::GRAVITY_SOUTHEAST), 'Image crop gravity', true) ->param('quality', 100, new Range(0, 100), 'Preview image quality. Pass an integer between 0 to 100. Defaults to 100.', true) ->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true) ->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true) From 3c29e0da95d5b942c033b91c665752b8f1c66133 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 9 Jun 2021 13:06:49 +0545 Subject: [PATCH 04/17] update image library --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index 26b8c6c85b..053de4bab1 100644 --- a/composer.lock +++ b/composer.lock @@ -1742,16 +1742,16 @@ }, { "name": "utopia-php/image", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "7761ff565e505bb3ddb9cfa05b7e1efddf3bebaa" + "reference": "20849a3a55790bd6eb3decde9f9708f04d58e489" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/7761ff565e505bb3ddb9cfa05b7e1efddf3bebaa", - "reference": "7761ff565e505bb3ddb9cfa05b7e1efddf3bebaa", + "url": "https://api.github.com/repos/utopia-php/image/zipball/20849a3a55790bd6eb3decde9f9708f04d58e489", + "reference": "20849a3a55790bd6eb3decde9f9708f04d58e489", "shasum": "" }, "require": { @@ -1789,9 +1789,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.3.0" + "source": "https://github.com/utopia-php/image/tree/0.3.1" }, - "time": "2021-06-02T07:08:04+00:00" + "time": "2021-06-09T07:12:35+00:00" }, { "name": "utopia-php/locale", From e1133b665d2ff2afb55581a21ae233108e81c3e0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 9 Jun 2021 14:22:20 +0545 Subject: [PATCH 05/17] fix whitelist issue --- app/controllers/api/storage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f4f00cbea6..68edc774ee 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -242,7 +242,7 @@ App::get('/v1/storage/files/:fileId/preview') ->param('fileId', '', new UID(), 'File unique ID') ->param('width', 0, new Range(0, 4000), 'Resize preview image width, Pass an integer between 0 to 4000.', true) ->param('height', 0, new Range(0, 4000), 'Resize preview image height, Pass an integer between 0 to 4000.', true) - ->param('gravity', Image::GRAVITY_CENTER, new WhiteList(Image::GRAVITY_CENTER, Image::GRAVITY_NORTH, Image::GRAVITY_NORTHWEST, Image::GRAVITY_NORTHEAST, Image::GRAVITY_WEST, Image::GRAVITY_EAST, Image::GRAVITY_SOUTHWEST, Image::GRAVITY_SOUTH, Image::GRAVITY_SOUTHEAST), 'Image crop gravity', true) + ->param('gravity', Image::GRAVITY_CENTER, new WhiteList([Image::GRAVITY_CENTER, Image::GRAVITY_NORTH, Image::GRAVITY_NORTHWEST, Image::GRAVITY_NORTHEAST, Image::GRAVITY_WEST, Image::GRAVITY_EAST, Image::GRAVITY_SOUTHWEST, Image::GRAVITY_SOUTH, Image::GRAVITY_SOUTHEAST]), 'Image crop gravity', true) ->param('quality', 100, new Range(0, 100), 'Preview image quality. Pass an integer between 0 to 100. Defaults to 100.', true) ->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true) ->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true) From 1f1b68f0cada5692926fae6c3c2bf6c9a26c72ef Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Jun 2021 15:16:03 +0545 Subject: [PATCH 06/17] update dependency --- composer.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/composer.lock b/composer.lock index 053de4bab1..9e7affe29c 100644 --- a/composer.lock +++ b/composer.lock @@ -1742,16 +1742,16 @@ }, { "name": "utopia-php/image", - "version": "0.3.1", + "version": "0.3.2", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "20849a3a55790bd6eb3decde9f9708f04d58e489" + "reference": "2044fdd44d87c4253cfe929cca975fd037461b00" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/20849a3a55790bd6eb3decde9f9708f04d58e489", - "reference": "20849a3a55790bd6eb3decde9f9708f04d58e489", + "url": "https://api.github.com/repos/utopia-php/image/zipball/2044fdd44d87c4253cfe929cca975fd037461b00", + "reference": "2044fdd44d87c4253cfe929cca975fd037461b00", "shasum": "" }, "require": { @@ -1789,9 +1789,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.3.1" + "source": "https://github.com/utopia-php/image/tree/0.3.2" }, - "time": "2021-06-09T07:12:35+00:00" + "time": "2021-06-10T09:16:11+00:00" }, { "name": "utopia-php/locale", @@ -6025,5 +6025,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.0.0" + "plugin-api-version": "2.1.0" } From 4ea0d627f486965a0a842afa93da531ef194118c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 11 Jun 2021 18:11:04 +0545 Subject: [PATCH 07/17] remove int conversion --- app/controllers/api/storage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 68edc774ee..c4fe8c8494 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -343,7 +343,7 @@ App::get('/v1/storage/files/:fileId/preview') $image = new Image($source); - $image->crop((int) $width, (int) $height, (int) $gravity); + $image->crop((int) $width, (int) $height, $gravity); if (!empty($opacity) || $opacity==0) { $image->setOpacity($opacity); From 3f199e1232e5243890d3a7204e872ea8845dd89a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 11 Jun 2021 16:20:18 +0200 Subject: [PATCH 08/17] refactor(workers): create abstract worker class --- app/controllers/api/health.php | 1 - app/workers/audits.php | 12 ++++++------ app/workers/certificates.php | 21 ++++++++++----------- app/workers/deletes.php | 14 ++++++-------- app/workers/functions.php | 11 ++++++----- app/workers/mails.php | 13 ++++++------- app/workers/tasks.php | 25 ++++++++++++------------- app/workers/usage.php | 19 +++++++++---------- app/workers/webhooks.php | 13 ++++++------- src/Appwrite/Resque/Worker.php | 33 +++++++++++++++++++++++++++++++++ 10 files changed, 94 insertions(+), 68 deletions(-) create mode 100644 src/Appwrite/Resque/Worker.php diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 0885c0c40b..dd1d4a0d69 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -6,7 +6,6 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; use Appwrite\ClamAV\Network; use Appwrite\Event\Event; -use RuntimeException; App::get('/v1/health') ->desc('Get HTTP') diff --git a/app/workers/audits.php b/app/workers/audits.php index d6027dd2e3..17327a3bb8 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -1,5 +1,6 @@ log($userId, $event, $resource, $userAgent, $ip, '', $data); } - public function tearDown(): void + public function shutdown(): void { // ... Remove environment for this job } -} +} \ No newline at end of file diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 90907204a3..0cb0e63c6c 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -1,30 +1,30 @@ args['projectId']) ? $this->args['projectId'] : ''; $type = $this->args['type']; @@ -82,9 +81,8 @@ class DeletesV1 } } - public function tearDown(): void + public function shutdown(): void { - // ... Remove environment for this job } protected function deleteDocuments(Document $document, $projectId) diff --git a/app/workers/functions.php b/app/workers/functions.php index 8bf719e607..cbe8ff4f50 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -6,6 +6,7 @@ use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Validator\Authorization; use Appwrite\Event\Event; +use Appwrite\Resque\Worker; use Cron\CronExpression; use Swoole\Runtime; use Utopia\App; @@ -125,17 +126,17 @@ Console::info(count($list)." functions listed in " . ($executionEnd - $execution //TODO aviod scheduled execution if delay is bigger than X offest -class FunctionsV1 +class FunctionsV1 extends Worker { public $args = []; public $allowed = []; - public function setUp(): void + public function init(): void { } - public function perform() + public function run(): void { global $register; @@ -579,7 +580,7 @@ class FunctionsV1 return $output; } - public function tearDown(): void + public function shutdown(): void { } -} +} \ No newline at end of file diff --git a/app/workers/mails.php b/app/workers/mails.php index 11f41e4446..e28d619811 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -1,26 +1,26 @@ getAttribute('updated') !== $updated) { // Task have already been rescheduled by owner - return false; + return; } if ($task->getAttribute('status') !== 'play') { // Skip task and don't schedule again - return false; + return; } // Reschedule @@ -202,11 +202,10 @@ class TasksV1 // Send alert if needed (use SMTP as default for now) - return true; + return; } - public function tearDown(): void + public function shutdown(): void { - // ... Remove environment for this job } -} +} \ No newline at end of file diff --git a/app/workers/usage.php b/app/workers/usage.php index 605b8d3f4c..bf948c0388 100644 --- a/app/workers/usage.php +++ b/app/workers/usage.php @@ -1,29 +1,30 @@ get('statsd', true); $projectId = $this->args['projectId'] ?? ''; @@ -36,12 +37,12 @@ class UsageV1 $httpMethod = $this->args['httpMethod'] ?? ''; $httpRequest = $this->args['httpRequest'] ?? 0; - $functionId = $this->args['functionId']; + $functionId = $this->args['functionId'] ?? ''; $functionExecution = $this->args['functionExecution'] ?? 0; $functionExecutionTime = $this->args['functionExecutionTime'] ?? 0; $functionStatus = $this->args['functionStatus'] ?? ''; - $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN').''; + $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) $statsd->setNamespace('appwrite.usage'); @@ -52,7 +53,6 @@ class UsageV1 if($functionExecution >= 1) { $statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus); - var_dump($tags.',functionId='.$functionId.',functionStatus='.$functionStatus); $statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime); } @@ -65,8 +65,7 @@ class UsageV1 } } - public function tearDown(): void + public function shutdown(): void { - // ... Remove environment for this job } -} +} \ No newline at end of file diff --git a/app/workers/webhooks.php b/app/workers/webhooks.php index 6acc38f646..a73dbb6c79 100644 --- a/app/workers/webhooks.php +++ b/app/workers/webhooks.php @@ -1,23 +1,23 @@ init(); + } + + public function perform() + { + $this->run(); + } + + public function tearDown(): void + { + $this->shutdown(); + } +} \ No newline at end of file From d96635ec23d69f2a13fe16fbe8178f22c4b2d4c7 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 11 Jun 2021 16:39:44 +0200 Subject: [PATCH 09/17] fix(worker): remove unnecessary imports --- src/Appwrite/Resque/Worker.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 0707d2f87f..db8dc91ce7 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -2,10 +2,6 @@ namespace Appwrite\Resque; -use Swoole\Runtime; - -use function Swoole\Coroutine\run; - abstract class Worker { public $args = []; From 665d57106eb9ea8bb64bf131448ee83a30df8878 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 11 Jun 2021 18:09:46 +0200 Subject: [PATCH 10/17] feat(server): database connection pools --- app/http.php | 31 ++++++++----- app/init.php | 58 ++++++++++++------------ app/workers.php | 32 +++++++++++++ app/workers/audits.php | 2 +- app/workers/certificates.php | 2 +- app/workers/deletes.php | 2 +- app/workers/functions.php | 2 +- app/workers/mails.php | 2 +- app/workers/tasks.php | 2 +- app/workers/usage.php | 2 +- app/workers/webhooks.php | 2 +- composer.json | 2 +- composer.lock | 26 +++++------ src/Appwrite/Database/Pool.php | 20 ++++++++ src/Appwrite/Database/Pool/PDOPool.php | 49 ++++++++++++++++++++ src/Appwrite/Database/Pool/RedisPool.php | 42 +++++++++++++++++ 16 files changed, 213 insertions(+), 63 deletions(-) create mode 100644 app/workers.php create mode 100644 src/Appwrite/Database/Pool.php create mode 100644 src/Appwrite/Database/Pool/PDOPool.php create mode 100644 src/Appwrite/Database/Pool/RedisPool.php diff --git a/app/http.php b/app/http.php index 89ff646d70..2fced63f4d 100644 --- a/app/http.php +++ b/app/http.php @@ -12,16 +12,6 @@ use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Utopia\App; use Utopia\CLI\Console; -use Utopia\Config\Config; -use Utopia\Domains\Domain; - -// xdebug_start_trace('/tmp/trace'); - -ini_set('memory_limit','512M'); -ini_set('display_errors', 1); -ini_set('display_startup_errors', 1); -ini_set('default_socket_timeout', -1); -error_reporting(E_ALL); $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); @@ -68,7 +58,7 @@ Files::load(__DIR__ . '/../public'); include __DIR__ . '/controllers/general.php'; -$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) { +$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) { $request = new Request($swooleRequest); $response = new Response($swooleResponse); @@ -85,6 +75,17 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo return; } + $db = $register->get('dbPool')->get(); + $redis = $register->get('redisPool')->get(); + + $register->set('db', function () use (&$db) { + return $db; + }); + + $register->set('cache', function () use (&$redis) { + return $redis; + }); + $app = new App('UTC'); try { @@ -104,6 +105,14 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo else { $swooleResponse->end('500: Server Error'); } + } finally { + /** @var PDOPool $dbPool */ + $dbPool = $register->get('dbPool'); + $dbPool->put($db); + + /** @var RedisPool $redisPool */ + $redisPool = $register->get('redisPool'); + $redisPool->put($redis); } }); diff --git a/app/init.php b/app/init.php index e628b56da7..8ece86da2c 100644 --- a/app/init.php +++ b/app/init.php @@ -11,6 +11,12 @@ if (\file_exists(__DIR__.'/../vendor/autoload.php')) { require_once __DIR__.'/../vendor/autoload.php'; } +ini_set('memory_limit','512M'); +ini_set('display_errors', 1); +ini_set('display_startup_errors', 1); +ini_set('default_socket_timeout', -1); +error_reporting(E_ALL); + use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; @@ -18,6 +24,8 @@ use Appwrite\Database\Database; use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Document; +use Appwrite\Database\Pool\PDOPool; +use Appwrite\Database\Pool\RedisPool; use Appwrite\Database\Validator\Authorization; use Appwrite\Event\Event; use Appwrite\Extend\PDO; @@ -145,23 +153,32 @@ Database::addFilter('encrypt', /* * Registry */ -$register->set('db', function () { // Register DB connection +$register->set('dbPool', function () { // Register DB connection $dbHost = App::getEnv('_APP_DB_HOST', ''); $dbUser = App::getEnv('_APP_DB_USER', ''); $dbPass = App::getEnv('_APP_DB_PASS', ''); $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + $pool = new PDOPool(10, $dbHost, $dbScheme, $dbUser, $dbPass); - $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDONative::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', - PDONative::ATTR_TIMEOUT => 3, // Seconds - PDONative::ATTR_PERSISTENT => true - )); + return $pool; +}); +$register->set('redisPool', function () { + $redisHost = App::getEnv('_APP_REDIS_HOST', ''); + $redisPort = App::getEnv('_APP_REDIS_PORT', ''); + $redisUser = App::getEnv('_APP_REDIS_USER', ''); + $redisPass = App::getEnv('_APP_REDIS_PASS', ''); + $redisAuth = []; - // Connection settings - $pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC); // Return arrays - $pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION); // Handle all errors with exceptions + if ($redisUser) { + $redisAuth[] = $redisUser; + } + if ($redisPass) { + $redisAuth[] = $redisPass; + } - return $pdo; + $pool = new RedisPool(10, $redisHost, $redisPort, $redisAuth); + + return $pool; }); $register->set('influxdb', function () { // Register DB connection $host = App::getEnv('_APP_INFLUXDB_HOST', ''); @@ -185,25 +202,6 @@ $register->set('statsd', function () { // Register DB connection return $statsd; }); -$register->set('cache', function () { // Register cache connection - $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $user = App::getEnv('_APP_REDIS_USER',''); - $pass = App::getEnv('_APP_REDIS_PASS',''); - $auth = []; - if(!empty($user)) { - $auth["user"] = $user; - } - if(!empty($pass)) { - $auth["pass"] = $pass; - } - if(!empty($auth)) { - $redis->auth($auth); - } - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); $register->set('smtp', function () { $mail = new PHPMailer(true); @@ -387,7 +385,7 @@ App::setResource('user', function($mode, $project, $console, $request, $response /** @var Appwrite\Database\Document $project */ /** @var Appwrite\Database\Database $consoleDB */ /** @var Appwrite\Database\Database $projectDB */ - /** @var bool $mode */ + /** @var string $mode */ Authorization::setDefaultStatus(true); diff --git a/app/workers.php b/app/workers.php new file mode 100644 index 0000000000..ff5ba1dbd3 --- /dev/null +++ b/app/workers.php @@ -0,0 +1,32 @@ +set('db', function () { + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + + $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + )); + + return $pdo; +}); +$register->set('cache', function () { // Register cache connection + $redis = new Redis(); + $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); \ No newline at end of file diff --git a/app/workers/audits.php b/app/workers/audits.php index 17327a3bb8..152934a6f6 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -5,7 +5,7 @@ use Utopia\Audit\Audit; use Utopia\Audit\Adapters\MySQL as AuditAdapter; use Utopia\CLI\Console; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Console::title('Audits V1 Worker'); Console::success(APP_NAME.' audits worker v1 has started'); diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 0cb0e63c6c..b44b2e44d9 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -11,7 +11,7 @@ use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Domains\Domain; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Console::title('Certificates V1 Worker'); Console::success(APP_NAME.' certificates worker v1 has started'); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 0568327962..63551e90f9 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -14,7 +14,7 @@ use Utopia\Config\Config; use Utopia\Audit\Audit; use Utopia\Audit\Adapters\MySQL as AuditAdapter; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Console::title('Deletes V1 Worker'); Console::success(APP_NAME.' deletes worker v1 has started'."\n"); diff --git a/app/workers/functions.php b/app/workers/functions.php index cbe8ff4f50..1fb9892a29 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -13,7 +13,7 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Runtime::enableCoroutine(0); diff --git a/app/workers/mails.php b/app/workers/mails.php index e28d619811..aa4fb8db80 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\App; use Utopia\CLI\Console; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Console::title('Mails V1 Worker'); Console::success(APP_NAME.' mails worker v1 has started'."\n"); diff --git a/app/workers/tasks.php b/app/workers/tasks.php index 12a37c67a7..f9a9b6f5d9 100644 --- a/app/workers/tasks.php +++ b/app/workers/tasks.php @@ -10,7 +10,7 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Console::title('Tasks V1 Worker'); Console::success(APP_NAME.' tasks worker v1 has started'); diff --git a/app/workers/usage.php b/app/workers/usage.php index bf948c0388..b5a3f885af 100644 --- a/app/workers/usage.php +++ b/app/workers/usage.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\App; use Utopia\CLI\Console; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Console::title('Usage V1 Worker'); Console::success(APP_NAME.' usage worker v1 has started'); diff --git a/app/workers/webhooks.php b/app/workers/webhooks.php index a73dbb6c79..00307bdda2 100644 --- a/app/workers/webhooks.php +++ b/app/workers/webhooks.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\App; use Utopia\CLI\Console; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Console::title('Webhooks V1 Worker'); Console::success(APP_NAME.' webhooks worker v1 has started'); diff --git a/composer.json b/composer.json index 306e9d06e2..c36ee333eb 100644 --- a/composer.json +++ b/composer.json @@ -46,7 +46,7 @@ "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", "utopia-php/locale": "0.3.*", - "utopia-php/registry": "0.4.*", + "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", "utopia-php/swoole": "0.2.*", diff --git a/composer.lock b/composer.lock index e53d9a82b1..d38db66866 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6704f7df5ffe0baac3633dc8e683ed78", + "content-hash": "19426f6ef8c8da256a0ff0fba2c11051", "packages": [ { "name": "adhocore/jwt", @@ -1899,16 +1899,16 @@ }, { "name": "utopia-php/registry", - "version": "0.4.0", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/registry.git", - "reference": "7aebbc6c5f3f04ff7a35ac3dad39fa91c9bd7c2d" + "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/registry/zipball/7aebbc6c5f3f04ff7a35ac3dad39fa91c9bd7c2d", - "reference": "7aebbc6c5f3f04ff7a35ac3dad39fa91c9bd7c2d", + "url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d", + "reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d", "shasum": "" }, "require": { @@ -1945,9 +1945,9 @@ ], "support": { "issues": "https://github.com/utopia-php/registry/issues", - "source": "https://github.com/utopia-php/registry/tree/0.4.0" + "source": "https://github.com/utopia-php/registry/tree/0.5.0" }, - "time": "2021-03-10T06:50:09+00:00" + "time": "2021-03-10T10:45:22+00:00" }, { "name": "utopia-php/storage", @@ -4472,16 +4472,16 @@ }, { "name": "sebastian/global-state", - "version": "5.0.2", + "version": "5.0.3", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "a90ccbddffa067b51f574dea6eb25d5680839455" + "reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/a90ccbddffa067b51f574dea6eb25d5680839455", - "reference": "a90ccbddffa067b51f574dea6eb25d5680839455", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/23bd5951f7ff26f12d4e3242864df3e08dec4e49", + "reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49", "shasum": "" }, "require": { @@ -4524,7 +4524,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.2" + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.3" }, "funding": [ { @@ -4532,7 +4532,7 @@ "type": "github" } ], - "time": "2020-10-26T15:55:19+00:00" + "time": "2021-06-11T13:31:12+00:00" }, { "name": "sebastian/lines-of-code", diff --git a/src/Appwrite/Database/Pool.php b/src/Appwrite/Database/Pool.php new file mode 100644 index 0000000000..f6ab232664 --- /dev/null +++ b/src/Appwrite/Database/Pool.php @@ -0,0 +1,20 @@ +available = false; + while (!$this->pool->isEmpty()) { + $this->pool->pop(); + } + } +} diff --git a/src/Appwrite/Database/Pool/PDOPool.php b/src/Appwrite/Database/Pool/PDOPool.php new file mode 100644 index 0000000000..75da61e6eb --- /dev/null +++ b/src/Appwrite/Database/Pool/PDOPool.php @@ -0,0 +1,49 @@ +pool = new SplQueue; + $this->size = $size; + for ($i = 0; $i < $this->size; $i++) { + $pdo = new PDO( + "mysql:" . + "host={$host};" . + "dbname={$schema};" . + "charset={$charset}", + $user, + $pass, + [ + PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true + ] + ); + $this->pool->enqueue($pdo); + } + } + + public function put(PDO $pdo) + { + $this->pool->enqueue($pdo); + } + + public function get(): PDO + { + if ($this->available && count($this->pool) > 0) { + return $this->pool->dequeue(); + } + sleep(0.01); + return $this->get(); + } +} diff --git a/src/Appwrite/Database/Pool/RedisPool.php b/src/Appwrite/Database/Pool/RedisPool.php new file mode 100644 index 0000000000..08e102abb0 --- /dev/null +++ b/src/Appwrite/Database/Pool/RedisPool.php @@ -0,0 +1,42 @@ +pool = new SplQueue; + $this->size = $size; + for ($i = 0; $i < $this->size; $i++) { + $redis = new Redis(); + $redis->pconnect($host, $port); + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + if ($auth) { + $redis->auth($auth); + } + + $this->pool->enqueue($redis); + } + } + + public function put(Redis $redis) + { + $this->pool->enqueue($redis); + } + + public function get(): Redis + { + if ($this->available && !$this->pool->isEmpty()) { + return $this->pool->dequeue(); + } + sleep(0.1); + return $this->get(); + } +} From 967afad6968762ea0cb8dae426c57d6f1e9aed35 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 12 Jun 2021 13:57:32 +0300 Subject: [PATCH 11/17] Fixed extra space on modal window --- app/views/console/home/index.phtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/console/home/index.phtml b/app/views/console/home/index.phtml index 1c8578ec37..021cf89928 100644 --- a/app/views/console/home/index.phtml +++ b/app/views/console/home/index.phtml @@ -332,7 +332,7 @@ $usageStatsEnabled = $this->getParam('usageStatsEnabled',true);