From 8e492a9797be231687d771975a46d6f0b7c12b53 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 28 Oct 2022 17:45:16 +0300 Subject: [PATCH 01/15] Tests --- .env | 2 + app/config/collections.php | 2 +- app/http.php | 14 ++- app/init.php | 221 ++++++++++++++++++++++++++++++++++++- app/tasks/specs.php | 2 +- 5 files changed, 232 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 950fd44821..4a29d7eb37 100644 --- a/.env +++ b/.env @@ -25,6 +25,8 @@ _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mariadb:3306/appwrite +#_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://doadmin:AVNS_sgtjg6ZSHBdg66422x-@db-mysql-fra1-shmuel-test-do-user-10204879-0.b.db.ondigitalocean.com:25060/console +#_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://doadmin:AVNS_sgtjg6ZSHBdg66422x-@db-mysql-fra1-shmuel-test-do-user-10204879-0.b.db.ondigitalocean.com:25060/projects _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 diff --git a/app/config/collections.php b/app/config/collections.php index 4d88518737..5d5b8890b0 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -1003,7 +1003,7 @@ $collections = [ '$id' => ID::custom('secret'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 512, // var_dump of \bin2hex(\random_bytes(128)) => string(256) doubling for encryption + 'size' => 512, // Output of \bin2hex(\random_bytes(128)) => string(256) doubling for encryption 'signed' => true, 'required' => true, 'default' => null, diff --git a/app/http.php b/app/http.php index 7b4ac39cef..0c9597933f 100644 --- a/app/http.php +++ b/app/http.php @@ -20,6 +20,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Swoole\Files; use Appwrite\Utopia\Request; +use Swoole\Coroutine\Channel; use Utopia\Logger\Log; use Utopia\Logger\Log\User; use Utopia\Pools\Group; @@ -59,9 +60,13 @@ include __DIR__ . '/controllers/general.php'; $http->on('start', function (Server $http) use ($payloadSize, $register) { $app = new App('UTC'); + global $http; + var_dump('Test size is:'.$register->get('test')->length()); + + var_dump('Startup is served by:'.$http->worker_id); go(function () use ($register, $app) { - + var_dump('Test 2 size is:'.$register->get('test')->length()); $pools = $register->get('pools'); /** @var Group $pools */ App::setResource('pools', fn() => $pools); @@ -119,7 +124,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { /** * Skip to prevent 0.16 migration issues. */ - if (in_array($key, ['cache', 'variables']) && $dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'bucket_1')) { + if (in_array($key, ['cache', 'variables']) && $dbForConsole->exists($dbForConsole->getDefaultDatabase(), 'bucket_1')) { continue; } @@ -155,7 +160,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection($key, $attributes, $indexes); } - if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'bucket_1')) { + if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists($dbForConsole->getDefaultDatabase(), 'bucket_1')) { Console::success('[Setup] - Creating default bucket...'); $dbForConsole->createDocument('buckets', new Document([ '$id' => ID::custom('default'), @@ -233,6 +238,9 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) { $request = new Request($swooleRequest); $response = new Response($swooleResponse); + var_dump('Test 3 size is:'.$register->get('test')->length()); + global $http; + var_dump('Served by:'.$http->worker_id); if (Files::isFileLoaded($request->getURI())) { $time = (60 * 60 * 24 * 365 * 2); // 45 days cache diff --git a/app/init.php b/app/init.php index 3a05ea5ddf..0f1ef4b4ec 100644 --- a/app/init.php +++ b/app/init.php @@ -1,5 +1,206 @@ name = $name; + $this->size = $size; + $this->init = $init; + $this->channel = new Channel($size); + } + + /** + * @return string + */ + public function getName(): string + { + return $this->name; + } + + /** + * @return int + */ + public function getSize(): int + { + return $this->size; + } + + /** + * @return int + */ + public function getReconnectAttempts(): int + { + return $this->reconnectAttempts; + } + + /** + * @var int $reconnectAttempts + * @return self + */ + public function setReconnectAttempts(int $reconnectAttempts): self + { + $this->reconnectAttempts = $reconnectAttempts; + return $this; + } + + /** + * @return int + */ + public function getReconnectSleep(): int + { + return $this->reconnectSleep; + } + + /** + * @var int $reconnectSleep + * @return self + */ + public function setReconnectSleep(int $reconnectSleep): self + { + $this->reconnectSleep = $reconnectSleep; + return $this; + } + + /** + * @return self + */ + public function fill(): self + { + if(!$this->channel->isEmpty()) { + return $this; + } + for ($i=0; $i < $this->size; $i++) { + $attempts = 0; + + do { + try { + $attempts++; + $connection = new Connection(($this->init)()); + break; // leave loop if successful + } catch (\Exception $e) { + if ($attempts >= $this->getReconnectAttempts()) { + throw new \Exception('Failed to create connection: ' . $e->getMessage()); + } + sleep($this->getReconnectSleep()); + } + } while ($attempts < $this->getReconnectAttempts()); + + $connection->setID($this->getName().'-'.$i); + + $this->channel->push($connection); + } + + return $this; + } + + /** + * @return Connection + */ + public function pop(): Connection + { + if ($this->channel->isEmpty()) { + throw new Exception('Pool is empty'); + } + + $connection = $this->channel->pop(); + $this->active[$connection->getID()] = $connection; + + return $connection; + } + + /** + * @param Connection $connection + * @return self + */ + public function push(Connection $connection): self + { + $this->channel->push($connection); + unset($this->active[$connection->getID()]); + + return $this; + } + + /** + * @return int + */ + public function count(): int + { + return $this->channel->length(); + } + + /** + * @return self + */ + public function reclaim(): self + { + foreach ($this->active as $connection) { + $this->push($connection); + } + return $this; + } + + /** + * @return bool + */ + public function isEmpty(): bool + { + return $this->channel->isEmpty(); + } + + /** + * @return bool + */ + public function isFull(): bool + { + return $this->channel->isFull(); + } +} + /** * Init * @@ -69,7 +270,6 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\MySQL; use Utopia\Pools\Group; -use Utopia\Pools\Pool; use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use MaxMind\Db\Reader; @@ -498,7 +698,8 @@ $register->set('logger', function () { return new Logger($adapter); }); $register->set('pools', function () { - + Console::info('Initializing pools'); + $group = new Group(); $fallbackForDB = AppwriteURL::unparse([ @@ -622,6 +823,9 @@ $register->set('pools', function () { break; } + + + $pool = new Pool($name, 64, function () use ($type, $resource, $dsn) { // Get Adapter $adapter = null; @@ -634,7 +838,6 @@ $register->set('pools', function () { default => null }; - var_dump($dsn->getDatabase()); $adapter->setDefaultDatabase($dsn->getDatabase()); break; @@ -658,7 +861,7 @@ $register->set('pools', function () { return $adapter; }); - +var_dump($pool->channel->length()); $group->add($pool); } @@ -667,12 +870,22 @@ $register->set('pools', function () { try { $group->fill(); + Console::success('Pools have been filled succefully'); } catch (\Throwable $th) { Console::error('Connection failure: ' . $th->getMessage()); } return $group; }); +$register->set('test', function () { + var_dump('[[init test!!]]'); + $test = new Channel(5); + $test->push(1); + $test->push(1); + $test->push(1); + $test->push(1); + return $test; +}); $register->set('influxdb', function () { // Register DB connection $host = App::getEnv('_APP_INFLUXDB_HOST', ''); diff --git a/app/tasks/specs.php b/app/tasks/specs.php index 8d1bc5039f..b3ac763b39 100644 --- a/app/tasks/specs.php +++ b/app/tasks/specs.php @@ -206,7 +206,7 @@ $cli unset($models[$key]); } } - // var_dump($models); + $arguments = [new App('UTC'), $services, $routes, $models, $keys[$platform], $authCounts[$platform] ?? 0]; foreach (['swagger2', 'open-api3'] as $format) { $formatInstance = match ($format) { From ae877ea59769de2c0d399336d7a3f68b892ba286 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 28 Oct 2022 17:53:21 +0300 Subject: [PATCH 02/15] cleanup --- .env | 2 -- 1 file changed, 2 deletions(-) diff --git a/.env b/.env index 4a29d7eb37..950fd44821 100644 --- a/.env +++ b/.env @@ -25,8 +25,6 @@ _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mariadb:3306/appwrite -#_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://doadmin:AVNS_sgtjg6ZSHBdg66422x-@db-mysql-fra1-shmuel-test-do-user-10204879-0.b.db.ondigitalocean.com:25060/console -#_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://doadmin:AVNS_sgtjg6ZSHBdg66422x-@db-mysql-fra1-shmuel-test-do-user-10204879-0.b.db.ondigitalocean.com:25060/projects _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 From 835c5de073f06af3b1e287fbc14ae235a214160f Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 29 Oct 2022 16:54:05 +0300 Subject: [PATCH 03/15] Updated dependencies --- composer.json | 4 +-- composer.lock | 75 ++++++++++++++++++++++----------------------------- 2 files changed, 34 insertions(+), 45 deletions(-) diff --git a/composer.json b/composer.json index 957810ef0e..dc49794ac3 100644 --- a/composer.json +++ b/composer.json @@ -43,7 +43,7 @@ "ext-sockets": "*", "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.11.*", - "utopia-php/framework": "0.21.*", + "utopia-php/framework": "0.23.*", "utopia-php/logger": "0.3.*", "utopia-php/abuse": "0.14.*", "utopia-php/analytics": "0.2.*", @@ -56,7 +56,7 @@ "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", - "utopia-php/swoole": "0.3.*", + "utopia-php/swoole": "0.5.*", "utopia-php/storage": "0.11.*", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", diff --git a/composer.lock b/composer.lock index 262b6d1846..0965b0d0ed 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8e777148ca3643e8186eec24ad045eb1", + "content-hash": "35b9599e05d47387586904cd2b6234ae", "packages": [ { "name": "adhocore/jwt", @@ -620,16 +620,16 @@ }, { "name": "guzzlehttp/psr7", - "version": "2.4.2", + "version": "2.4.3", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "3148458748274be1546f8f2809a6c09fe66f44aa" + "reference": "67c26b443f348a51926030c83481b85718457d3d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/3148458748274be1546f8f2809a6c09fe66f44aa", - "reference": "3148458748274be1546f8f2809a6c09fe66f44aa", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/67c26b443f348a51926030c83481b85718457d3d", + "reference": "67c26b443f348a51926030c83481b85718457d3d", "shasum": "" }, "require": { @@ -719,7 +719,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.4.2" + "source": "https://github.com/guzzle/psr7/tree/2.4.3" }, "funding": [ { @@ -735,7 +735,7 @@ "type": "tidelift" } ], - "time": "2022-10-25T13:49:28+00:00" + "time": "2022-10-26T14:07:24+00:00" }, { "name": "influxdb/influxdb-php", @@ -1775,12 +1775,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "cdc81747329f562c05d4843ad736811e20a81d34" + "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/cdc81747329f562c05d4843ad736811e20a81d34", - "reference": "cdc81747329f562c05d4843ad736811e20a81d34", + "url": "https://api.github.com/repos/utopia-php/database/zipball/8ba52f4ce40674ed9f6e8f72e300d5d9a4900590", + "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590", "shasum": "" }, "require": { @@ -1821,7 +1821,7 @@ "issues": "https://github.com/utopia-php/database/issues", "source": "https://github.com/utopia-php/database/tree/feat-update-create" }, - "time": "2022-10-26T09:10:13+00:00" + "time": "2022-10-28T12:01:54+00:00" }, { "name": "utopia-php/domains", @@ -1879,24 +1879,24 @@ }, { "name": "utopia-php/framework", - "version": "0.21.1", + "version": "0.23.2", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "c81789b87a917da2daf336738170ebe01f50ea18" + "reference": "847cfd48fa93f25797dea0be5692f5c3735c9b2f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/c81789b87a917da2daf336738170ebe01f50ea18", - "reference": "c81789b87a917da2daf336738170ebe01f50ea18", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/847cfd48fa93f25797dea0be5692f5c3735c9b2f", + "reference": "847cfd48fa93f25797dea0be5692f5c3735c9b2f", "shasum": "" }, "require": { "php": ">=8.0.0" }, "require-dev": { - "phpunit/phpunit": "^9.5.10", - "vimeo/psalm": "4.13.1" + "phpunit/phpunit": "^9.5.25", + "vimeo/psalm": "^4.27.0" }, "type": "library", "autoload": { @@ -1908,12 +1908,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple, light and advanced PHP framework", "keywords": [ "framework", @@ -1922,9 +1916,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.21.1" + "source": "https://github.com/utopia-php/framework/tree/0.23.2" }, - "time": "2022-09-07T09:56:28+00:00" + "time": "2022-10-25T15:05:52+00:00" }, { "name": "utopia-php/image", @@ -2365,16 +2359,16 @@ }, { "name": "utopia-php/swoole", - "version": "0.3.3", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "8312df69233b5dcd3992de88f131f238002749de" + "reference": "c2a3a4f944a2f22945af3cbcb95b13f0769628b1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/8312df69233b5dcd3992de88f131f238002749de", - "reference": "8312df69233b5dcd3992de88f131f238002749de", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/c2a3a4f944a2f22945af3cbcb95b13f0769628b1", + "reference": "c2a3a4f944a2f22945af3cbcb95b13f0769628b1", "shasum": "" }, "require": { @@ -2383,6 +2377,7 @@ "utopia-php/framework": "0.*.*" }, "require-dev": { + "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", "swoole/ide-helper": "4.8.3", "vimeo/psalm": "4.15.0" @@ -2397,12 +2392,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "team@appwrite.io" - } - ], "description": "An extension for Utopia Framework to work with PHP Swoole as a PHP FPM alternative", "keywords": [ "framework", @@ -2415,9 +2404,9 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.3.3" + "source": "https://github.com/utopia-php/swoole/tree/0.5.0" }, - "time": "2022-01-20T09:58:43+00:00" + "time": "2022-10-19T22:19:07+00:00" }, { "name": "utopia-php/system", @@ -3295,16 +3284,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.17", + "version": "9.2.18", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8" + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/aa94dc41e8661fe90c7316849907cba3007b10d8", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/12fddc491826940cf9b7e88ad9664cf51f0f6d0a", + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a", "shasum": "" }, "require": { @@ -3360,7 +3349,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.17" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.18" }, "funding": [ { @@ -3368,7 +3357,7 @@ "type": "github" } ], - "time": "2022-08-30T12:24:04+00:00" + "time": "2022-10-27T13:35:33+00:00" }, { "name": "phpunit/php-file-iterator", From 5eb39b1ccd189b4791f5b52bdaeef9db0dd11368 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 29 Oct 2022 16:59:03 +0300 Subject: [PATCH 04/15] No need to delegate DB errors --- app/controllers/general.php | 5 ----- app/init.php | 11 +---------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index e56357869d..fd7599595a 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -403,11 +403,6 @@ App::error() $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->match($request); - /** Delegate PDO exceptions to the global handler so the database connection can be returned to the pool */ - if ($error instanceof PDOException) { - throw $error; - } - if ($logger) { if ($error->getCode() >= 500 || $error->getCode() === 0) { try { diff --git a/app/init.php b/app/init.php index 0f1ef4b4ec..24083c801f 100644 --- a/app/init.php +++ b/app/init.php @@ -861,7 +861,7 @@ $register->set('pools', function () { return $adapter; }); -var_dump($pool->channel->length()); + $group->add($pool); } @@ -877,15 +877,6 @@ var_dump($pool->channel->length()); return $group; }); -$register->set('test', function () { - var_dump('[[init test!!]]'); - $test = new Channel(5); - $test->push(1); - $test->push(1); - $test->push(1); - $test->push(1); - return $test; -}); $register->set('influxdb', function () { // Register DB connection $host = App::getEnv('_APP_INFLUXDB_HOST', ''); From 766c19b165b68525a4899d0a9760005bc98c49e1 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 29 Oct 2022 17:04:35 +0300 Subject: [PATCH 05/15] Removed logs --- app/init.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/init.php b/app/init.php index 24083c801f..384e8159cd 100644 --- a/app/init.php +++ b/app/init.php @@ -698,8 +698,6 @@ $register->set('logger', function () { return new Logger($adapter); }); $register->set('pools', function () { - Console::info('Initializing pools'); - $group = new Group(); $fallbackForDB = AppwriteURL::unparse([ @@ -870,7 +868,6 @@ $register->set('pools', function () { try { $group->fill(); - Console::success('Pools have been filled succefully'); } catch (\Throwable $th) { Console::error('Connection failure: ' . $th->getMessage()); } From fdf88d83553abb7d74645e682af8e628d36e1e18 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 29 Oct 2022 17:05:26 +0300 Subject: [PATCH 06/15] HTTP Tests --- app/http.php | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/http.php b/app/http.php index 0c9597933f..5a8f16f232 100644 --- a/app/http.php +++ b/app/http.php @@ -20,15 +20,18 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Swoole\Files; use Appwrite\Utopia\Request; -use Swoole\Coroutine\Channel; +use Swoole\Runtime; use Utopia\Logger\Log; use Utopia\Logger\Log\User; use Utopia\Pools\Group; +Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL); + $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); $payloadSize = 6 * (1024 * 1024); // 6MB $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); +$workerNumber = 1; $http ->set([ @@ -60,13 +63,8 @@ include __DIR__ . '/controllers/general.php'; $http->on('start', function (Server $http) use ($payloadSize, $register) { $app = new App('UTC'); - global $http; - var_dump('Test size is:'.$register->get('test')->length()); - - var_dump('Startup is served by:'.$http->worker_id); go(function () use ($register, $app) { - var_dump('Test 2 size is:'.$register->get('test')->length()); $pools = $register->get('pools'); /** @var Group $pools */ App::setResource('pools', fn() => $pools); @@ -238,9 +236,6 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) { $request = new Request($swooleRequest); $response = new Response($swooleResponse); - var_dump('Test 3 size is:'.$register->get('test')->length()); - global $http; - var_dump('Served by:'.$http->worker_id); if (Files::isFileLoaded($request->getURI())) { $time = (60 * 60 * 24 * 365 * 2); // 45 days cache @@ -254,6 +249,15 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo return; } + $pools = $register->get('pools'); /** @var Group $pools */ + var_dump('current console connection'); + var_dump($pools->get('console')->pop()->getID()); + var_dump('current pool size'); + var_dump($pools->get('console')->count()); + // Console::log('sleep start'); + // System::sleep(3); + // Console::log('sleep end'); + $app = new App('UTC'); $pools = $register->get('pools'); From de610855fe88bd8c985aa325e1fdf77382b60cac Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Thu, 3 Nov 2022 17:19:55 +0200 Subject: [PATCH 07/15] Updates --- app/http.php | 5 -- app/init.php | 206 +-------------------------------------------- docker-compose.yml | 3 - 3 files changed, 2 insertions(+), 212 deletions(-) diff --git a/app/http.php b/app/http.php index 5a8f16f232..9bd9501fb5 100644 --- a/app/http.php +++ b/app/http.php @@ -249,11 +249,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo return; } - $pools = $register->get('pools'); /** @var Group $pools */ - var_dump('current console connection'); - var_dump($pools->get('console')->pop()->getID()); - var_dump('current pool size'); - var_dump($pools->get('console')->count()); // Console::log('sleep start'); // System::sleep(3); // Console::log('sleep end'); diff --git a/app/init.php b/app/init.php index 384e8159cd..9782af7265 100644 --- a/app/init.php +++ b/app/init.php @@ -1,205 +1,5 @@ name = $name; - $this->size = $size; - $this->init = $init; - $this->channel = new Channel($size); - } - - /** - * @return string - */ - public function getName(): string - { - return $this->name; - } - - /** - * @return int - */ - public function getSize(): int - { - return $this->size; - } - - /** - * @return int - */ - public function getReconnectAttempts(): int - { - return $this->reconnectAttempts; - } - - /** - * @var int $reconnectAttempts - * @return self - */ - public function setReconnectAttempts(int $reconnectAttempts): self - { - $this->reconnectAttempts = $reconnectAttempts; - return $this; - } - - /** - * @return int - */ - public function getReconnectSleep(): int - { - return $this->reconnectSleep; - } - - /** - * @var int $reconnectSleep - * @return self - */ - public function setReconnectSleep(int $reconnectSleep): self - { - $this->reconnectSleep = $reconnectSleep; - return $this; - } - - /** - * @return self - */ - public function fill(): self - { - if(!$this->channel->isEmpty()) { - return $this; - } - for ($i=0; $i < $this->size; $i++) { - $attempts = 0; - - do { - try { - $attempts++; - $connection = new Connection(($this->init)()); - break; // leave loop if successful - } catch (\Exception $e) { - if ($attempts >= $this->getReconnectAttempts()) { - throw new \Exception('Failed to create connection: ' . $e->getMessage()); - } - sleep($this->getReconnectSleep()); - } - } while ($attempts < $this->getReconnectAttempts()); - - $connection->setID($this->getName().'-'.$i); - - $this->channel->push($connection); - } - - return $this; - } - - /** - * @return Connection - */ - public function pop(): Connection - { - if ($this->channel->isEmpty()) { - throw new Exception('Pool is empty'); - } - - $connection = $this->channel->pop(); - $this->active[$connection->getID()] = $connection; - - return $connection; - } - - /** - * @param Connection $connection - * @return self - */ - public function push(Connection $connection): self - { - $this->channel->push($connection); - unset($this->active[$connection->getID()]); - - return $this; - } - - /** - * @return int - */ - public function count(): int - { - return $this->channel->length(); - } - - /** - * @return self - */ - public function reclaim(): self - { - foreach ($this->active as $connection) { - $this->push($connection); - } - return $this; - } - - /** - * @return bool - */ - public function isEmpty(): bool - { - return $this->channel->isEmpty(); - } - - /** - * @return bool - */ - public function isFull(): bool - { - return $this->channel->isFull(); - } -} /** * Init @@ -270,6 +70,7 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\MySQL; use Utopia\Pools\Group; +use Utopia\Pools\Pool; use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use MaxMind\Db\Reader; @@ -821,10 +622,7 @@ $register->set('pools', function () { break; } - - - - $pool = new Pool($name, 64, function () use ($type, $resource, $dsn) { + $pool = new Pool($name, 5, function () use ($type, $resource, $dsn) { // Get Adapter $adapter = null; diff --git a/docker-compose.yml b/docker-compose.yml index f7229ce27b..34a9d5993c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,6 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/local/dev - - ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework depends_on: - mariadb - redis @@ -361,7 +360,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis - mariadb @@ -619,7 +617,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis environment: From 9bba91fcb8b51677d13bc20722f7f733d153175f Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Thu, 3 Nov 2022 18:07:53 +0200 Subject: [PATCH 08/15] Update pools library version --- app/init.php | 6 ------ composer.json | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/app/init.php b/app/init.php index 9782af7265..bff40c5a81 100644 --- a/app/init.php +++ b/app/init.php @@ -664,12 +664,6 @@ $register->set('pools', function () { Config::setParam('pools-' . $key, $config); } - try { - $group->fill(); - } catch (\Throwable $th) { - Console::error('Connection failure: ' . $th->getMessage()); - } - return $group; }); $register->set('influxdb', function () { diff --git a/composer.json b/composer.json index dc49794ac3..79c8313e5f 100644 --- a/composer.json +++ b/composer.json @@ -61,7 +61,7 @@ "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.6.*", - "utopia-php/pools": "0.1.*", + "utopia-php/pools": "0.2.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", From be8c145e845209bc660bfb7b850221b333f43658 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Thu, 3 Nov 2022 18:26:31 +0200 Subject: [PATCH 09/15] Fix for linter --- app/init.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/init.php b/app/init.php index bff40c5a81..e78d03b478 100644 --- a/app/init.php +++ b/app/init.php @@ -1,6 +1,5 @@ Date: Thu, 3 Nov 2022 18:27:41 +0200 Subject: [PATCH 10/15] Removed enableCoroutine support --- app/http.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/http.php b/app/http.php index 9bd9501fb5..2448eb1b77 100644 --- a/app/http.php +++ b/app/http.php @@ -20,13 +20,10 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Swoole\Files; use Appwrite\Utopia\Request; -use Swoole\Runtime; use Utopia\Logger\Log; use Utopia\Logger\Log\User; use Utopia\Pools\Group; -Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL); - $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); $payloadSize = 6 * (1024 * 1024); // 6MB From 1d59b8bd54535ab0c3bb76a2d2c31561bc020d1c Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Thu, 3 Nov 2022 18:28:28 +0200 Subject: [PATCH 11/15] Leftovers --- app/http.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/http.php b/app/http.php index 2448eb1b77..b497c466f3 100644 --- a/app/http.php +++ b/app/http.php @@ -28,7 +28,6 @@ $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); $payloadSize = 6 * (1024 * 1024); // 6MB $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); -$workerNumber = 1; $http ->set([ @@ -246,10 +245,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo return; } - // Console::log('sleep start'); - // System::sleep(3); - // Console::log('sleep end'); - $app = new App('UTC'); $pools = $register->get('pools'); From c28edaaa9623b058447375c9a05fe46afde5c84a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Thu, 3 Nov 2022 18:30:55 +0200 Subject: [PATCH 12/15] Enabled 64 connections --- app/init.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index e78d03b478..09d5ce0589 100644 --- a/app/init.php +++ b/app/init.php @@ -65,7 +65,6 @@ use Utopia\Storage\Device\Wasabi; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; -use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\MySQL; use Utopia\Pools\Group; @@ -621,7 +620,7 @@ $register->set('pools', function () { break; } - $pool = new Pool($name, 5, function () use ($type, $resource, $dsn) { + $pool = new Pool($name, 64, function () use ($type, $resource, $dsn) { // Get Adapter $adapter = null; From 541c6772f2b1f81a00c30b19393806066396f21a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Thu, 3 Nov 2022 21:00:00 +0200 Subject: [PATCH 13/15] Updated dependencies --- composer.json | 2 +- composer.lock | 38 +++++++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/composer.json b/composer.json index 79c8313e5f..65286e5952 100644 --- a/composer.json +++ b/composer.json @@ -61,7 +61,7 @@ "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.6.*", - "utopia-php/pools": "0.2.*", + "utopia-php/pools": "dev-feat-optimize-filling as 0.3.0", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", diff --git a/composer.lock b/composer.lock index 0965b0d0ed..dcbf3d84d6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "35b9599e05d47387586904cd2b6234ae", + "content-hash": "ef123ba6e9ee89e8355c42b590c272d9", "packages": [ { "name": "adhocore/jwt", @@ -1879,16 +1879,16 @@ }, { "name": "utopia-php/framework", - "version": "0.23.2", + "version": "0.23.4", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "847cfd48fa93f25797dea0be5692f5c3735c9b2f" + "reference": "97f64aa1732af92b967c3576f16967dc762ad47b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/847cfd48fa93f25797dea0be5692f5c3735c9b2f", - "reference": "847cfd48fa93f25797dea0be5692f5c3735c9b2f", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/97f64aa1732af92b967c3576f16967dc762ad47b", + "reference": "97f64aa1732af92b967c3576f16967dc762ad47b", "shasum": "" }, "require": { @@ -1916,9 +1916,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.23.2" + "source": "https://github.com/utopia-php/framework/tree/0.23.4" }, - "time": "2022-10-25T15:05:52+00:00" + "time": "2022-10-31T11:57:14+00:00" }, { "name": "utopia-php/image", @@ -2146,23 +2146,24 @@ }, { "name": "utopia-php/pools", - "version": "0.1.0", + "version": "dev-feat-optimize-filling", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805" + "reference": "be603898142f55df9db5058f81a610ead7769d7d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/5a467a569a80aefc846a97dc195b4adc2fd71805", - "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/be603898142f55df9db5058f81a610ead7769d7d", + "reference": "be603898142f55df9db5058f81a610ead7769d7d", "shasum": "" }, "require": { "ext-mongodb": "*", "ext-pdo": "*", "ext-redis": "*", - "php": ">=8.0" + "php": ">=8.0", + "utopia-php/cli": "^0.13.0" }, "require-dev": { "phpunit/phpunit": "^9.4", @@ -2193,9 +2194,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.1.0" + "source": "https://github.com/utopia-php/pools/tree/feat-optimize-filling" }, - "time": "2022-10-11T19:31:07+00:00" + "time": "2022-11-03T18:50:28+00:00" }, { "name": "utopia-php/preloader", @@ -5112,11 +5113,18 @@ "version": "dev-feat-update-create", "alias": "0.26.1", "alias_normalized": "0.26.1.0" + }, + { + "package": "utopia-php/pools", + "version": "dev-feat-optimize-filling", + "alias": "0.3.0", + "alias_normalized": "0.3.0.0" } ], "minimum-stability": "stable", "stability-flags": { - "utopia-php/database": 20 + "utopia-php/database": 20, + "utopia-php/pools": 20 }, "prefer-stable": false, "prefer-lowest": false, From f6ae1fedd24bab210e8000dffddbb9231ca186e6 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Thu, 3 Nov 2022 22:24:03 +0200 Subject: [PATCH 14/15] Fixed broken call --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 5656dbc64e..e3a1374f76 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1574,7 +1574,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') Query::equal('databaseInternalId', [$db->getInternalId()]) ], 61); - $limit = 64 - MariaDB::getCountOfDefaultIndexes(); + $limit = $dbForProject->getLimitForIndexes(); if ($count >= $limit) { throw new Exception(Exception::INDEX_LIMIT_EXCEEDED, 'Index limit exceeded'); From d31a3ab088d52ce419089c0f7937603efeed4b13 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 4 Nov 2022 07:26:56 +0200 Subject: [PATCH 15/15] Updated dependencies --- app/realtime.php | 1 - composer.lock | 49 +++++++++++++++++++++--------------------------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 8cd28c193f..f0a2b6e4a2 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -32,7 +32,6 @@ require_once __DIR__ . '/init.php'; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); - function getConsoleDB(): Database { global $register; diff --git a/composer.lock b/composer.lock index dcbf3d84d6..369daf8093 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ef123ba6e9ee89e8355c42b590c272d9", + "content-hash": "44c45e61fcc2cf8837fbae7df6360a4f", "packages": [ { "name": "adhocore/jwt", @@ -1462,23 +1462,23 @@ }, { "name": "utopia-php/abuse", - "version": "0.14.0", + "version": "0.16.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "1a5da248e74c1bfc39bc440fa949de6935acceeb" + "reference": "6370d9150425460416583feba0990504ac789e98" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/1a5da248e74c1bfc39bc440fa949de6935acceeb", - "reference": "1a5da248e74c1bfc39bc440fa949de6935acceeb", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/6370d9150425460416583feba0990504ac789e98", + "reference": "6370d9150425460416583feba0990504ac789e98", "shasum": "" }, "require": { "ext-curl": "*", "ext-pdo": "*", "php": ">=8.0", - "utopia-php/database": "0.26.*" + "utopia-php/database": "0.28.*" }, "require-dev": { "phpunit/phpunit": "^9.4", @@ -1510,9 +1510,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.14.0" + "source": "https://github.com/utopia-php/abuse/tree/0.16.0" }, - "time": "2022-10-14T11:26:39+00:00" + "time": "2022-10-31T14:46:41+00:00" }, { "name": "utopia-php/analytics", @@ -1571,22 +1571,22 @@ }, { "name": "utopia-php/audit", - "version": "0.15.0", + "version": "0.17.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "937ffd13e7a5ac9ad220b329247569ef2a4881d9" + "reference": "455471bd4de8d74026809e843f8c9740eb32922c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/937ffd13e7a5ac9ad220b329247569ef2a4881d9", - "reference": "937ffd13e7a5ac9ad220b329247569ef2a4881d9", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/455471bd4de8d74026809e843f8c9740eb32922c", + "reference": "455471bd4de8d74026809e843f8c9740eb32922c", "shasum": "" }, "require": { "ext-pdo": "*", "php": ">=8.0", - "utopia-php/database": "0.26.*" + "utopia-php/database": "0.28.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -1612,9 +1612,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.15.0" + "source": "https://github.com/utopia-php/audit/tree/0.17.0" }, - "time": "2022-10-14T11:39:18+00:00" + "time": "2022-10-31T14:44:52+00:00" }, { "name": "utopia-php/cache", @@ -1771,16 +1771,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-update-create", + "version": "0.28.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590" + "reference": "ef6506af1c09c22f5dc1e7859159d323f7fafa94" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/8ba52f4ce40674ed9f6e8f72e300d5d9a4900590", - "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590", + "url": "https://api.github.com/repos/utopia-php/database/zipball/ef6506af1c09c22f5dc1e7859159d323f7fafa94", + "reference": "ef6506af1c09c22f5dc1e7859159d323f7fafa94", "shasum": "" }, "require": { @@ -1819,9 +1819,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-update-create" + "source": "https://github.com/utopia-php/database/tree/0.28.0" }, - "time": "2022-10-28T12:01:54+00:00" + "time": "2022-10-31T09:58:46+00:00" }, { "name": "utopia-php/domains", @@ -5108,12 +5108,6 @@ } ], "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-update-create", - "alias": "0.26.1", - "alias_normalized": "0.26.1.0" - }, { "package": "utopia-php/pools", "version": "dev-feat-optimize-filling", @@ -5123,7 +5117,6 @@ ], "minimum-stability": "stable", "stability-flags": { - "utopia-php/database": 20, "utopia-php/pools": 20 }, "prefer-stable": false,