From 561e7b43e11d053d5c4f3e08f22e999af5987ce5 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 3 Aug 2022 00:26:45 +0530 Subject: [PATCH] feat: review comments --- app/controllers/api/projects.php | 4 +- app/http.php | 20 ++--- app/init.php | 37 +++++---- app/realtime.php | 15 ++-- src/Appwrite/Database/DatabasePool.php | 101 ++++++++++++------------- src/Appwrite/Database/PDOPool.php | 46 +++++++++++ 6 files changed, 132 insertions(+), 91 deletions(-) create mode 100644 src/Appwrite/Database/PDOPool.php diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 64c0454bd1..a4e62e95ac 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -85,7 +85,7 @@ App::post('/v1/projects') throw new Exception("'console' is a reserved project.", 400, Exception::PROJECT_RESERVED_PROJECT); } - [$dbForProject, $returnDB, $dbName] = $dbPool->getAnyFromPool($cache); + [$dbForProject, $dbName] = $dbPool->getAnyFromPool($cache); $project = $dbForConsole->createDocument('projects', new Document([ '$id' => $projectId, @@ -161,8 +161,6 @@ App::post('/v1/projects') $dbForProject->createCollection($key, $attributes, $indexes); } - call_user_func($returnDB); - $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($project, Response::MODEL_PROJECT); }); diff --git a/app/http.php b/app/http.php index 53990811e3..492b338811 100644 --- a/app/http.php +++ b/app/http.php @@ -2,6 +2,7 @@ require_once __DIR__ . '/../vendor/autoload.php'; +use Appwrite\Database\DatabasePool; use Appwrite\Utopia\Response; use Swoole\Process; use Swoole\Http\Server; @@ -63,8 +64,9 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { App::setResource('cache', fn() => $redis); $dbPool = $register->get('dbPool'); - [$dbForConsole, $returnDatabase] = $dbPool->getDBFromPool('console', $redis); - App::setResource('dbForConsole', fn() => $dbForConsole); + App::setResource('dbPool', fn() => $dbPool); + + $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */ Console::success('[Setup] - Server database init started...'); $collections = Config::getParam('collections', []); /** @var array $collections */ @@ -196,7 +198,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } - call_user_func($returnDatabase); + $dbPool->reset(); Console::success('[Setup] - Server database init completed...'); }); @@ -236,13 +238,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $dbPool = $register->get('dbPool'); App::setResource('dbPool', fn() => $dbPool); - [$dbForConsole, $returnConsoleDB] = $dbPool->getDBFromPool('console', $redis); - App::setResource('dbForConsole', fn() => $dbForConsole); - - $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', 'console')); - [$dbForProject, $returnProjectDB] = $dbPool->getDBFromPool($projectId, $redis); - App::setResource('dbForProject', fn() => $dbForProject); - try { Authorization::cleanRoles(); Authorization::setRole('role:all'); @@ -332,9 +327,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $swooleResponse->end(\json_encode($output)); } finally { - call_user_func($returnConsoleDB); - call_user_func($returnProjectDB); - + $dbPool->reset(); + /** @var RedisPool $redisPool */ $redisPool = $register->get('redisPool'); $redisPool->put($redis); diff --git a/app/init.php b/app/init.php index f4865df9fe..f0a4fdd6ba 100644 --- a/app/init.php +++ b/app/init.php @@ -861,25 +861,30 @@ App::setResource('console', function () { ]); }, []); -// App::setResource('dbForProject', function ($db, $cache, Document $project) { -// $cache = new Cache(new RedisCache($cache)); +App::setResource('dbForProject', function ($dbPool, $cache, Document $project) { + $database = $project->getAttribute('database', ''); + if (empty($database)) { + $database = $dbPool->getConsoleDB(); + } + $pdo = $dbPool->getDBFromPool($database); + $cache = new Cache(new RedisCache($cache)); + $database = new Database(new MariaDB($pdo), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace("_{$project->getInternalId()}"); -// $database = new Database(new MariaDB($db), $cache); -// $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); -// $database->setNamespace("_{$project->getInternalId()}"); + return $database; +}, ['dbPool', 'cache', 'project']); -// return $database; -// }, ['db', 'cache', 'project']); +App::setResource('dbForConsole', function ($dbPool, $cache) { + $database = $dbPool->getConsoleDB(); + $pdo = $dbPool->getDBFromPool($database); + $cache = new Cache(new RedisCache($cache)); + $database = new Database(new MariaDB($pdo), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace('_console'); -// App::setResource('dbForConsole', function ($dbPool, $cache) { -// $cache = new Cache(new RedisCache($cache)); - -// $database = new Database(new MariaDB($db), $cache); -// $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); -// $database->setNamespace('_console'); - -// return $database; -// }, ['dbPool', 'cache']); + return $database; +}, ['dbPool', 'cache']); App::setResource('deviceLocal', function () { return new Local(); diff --git a/app/realtime.php b/app/realtime.php index 745f41d6aa..ed49bb3feb 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -91,12 +91,12 @@ $server->error($logError); function getDatabase(Registry &$register, string $projectID) { $redis = $register->get('redisPool')->get(); - [$database, $returnDatabase] = $register->get('dbPool')->getDBFromPool($projectID, $redis); + $database = $register->get('dbPool')->getDBFromPool($projectID, $redis); return [ $database, - function () use ($register, $returnDatabase, $redis) { - call_user_func($returnDatabase); + function () use ($register, $redis) { + $register->get('dbPool')->reset(); $register->get('redisPool')->put($redis); } ]; @@ -345,7 +345,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, /** @var \Utopia\Database\Document $console */ $console = $app->getResource('console'); - [$dbForConsole, $returnConsoleDB] = $dbPool->getDBFromPool('console', $redis); + $dbForConsole = $dbPool->getDBFromPool('console', $redis); App::setResource('dbForConsole', fn() => $dbForConsole); /** @var \Utopia\Database\Document $project */ @@ -358,7 +358,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception('Missing or unknown project ID', 1008); } - [$dbForProject, $returnProjectDB] = $dbPool->getDBFromPool($project->getId(), $redis); + $dbForProject = $dbPool->getDBFromPool($project->getId(), $redis); App::setResource('dbForProject', fn() => $dbForProject); /** @var \Utopia\Database\Document $user */ @@ -448,8 +448,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, /** * Put used PDO and Redis Connections back into their pools. */ - call_user_func($returnConsoleDB); - call_user_func($returnProjectDB); + $dbPool->reset(); $register->get('redisPool')->put($redis); } }); @@ -463,7 +462,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $redis = $register->get('redisPool')->get(); $dbPool = $register->get('dbPool'); - [$dbForProject, $returnProjectDB] = $dbPool->getDBFromPool($projectId, $redis); + $dbForProject = $dbPool->getDBFromPool($projectId, $redis); /* * Abuse Check diff --git a/src/Appwrite/Database/DatabasePool.php b/src/Appwrite/Database/DatabasePool.php index 127aca3ad0..be7e44a682 100644 --- a/src/Appwrite/Database/DatabasePool.php +++ b/src/Appwrite/Database/DatabasePool.php @@ -7,10 +7,10 @@ use Utopia\App; use Appwrite\DSN\DSN; use Utopia\CLI\Console; use Utopia\Cache\Cache; -use Swoole\Database\PDOPool; use Swoole\Database\PDOProxy; use Utopia\Database\Database; use Appwrite\Extend\Exception; +use Appwrite\Database\PDOPool; use Swoole\Database\PDOConfig; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Validator\Authorization; @@ -62,24 +62,23 @@ class DatabasePool /** Create PDO pool instances for all the dsns */ foreach ($this->dsns as $name => $dsn) { $dsn = new DSN($dsn); - $pool = new PDOPool( - (new PDOConfig()) - ->withHost($dsn->getHost()) - ->withPort($dsn->getPort()) - ->withDbName($dsn->getDatabase()) - ->withCharset('utf8mb4') - ->withUsername($dsn->getUser()) - ->withPassword($dsn->getPassword()) - ->withOptions([ - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - ]), - 64 - ); + $pdoConfig = (new PDOConfig()) + ->withHost($dsn->getHost()) + ->withPort($dsn->getPort()) + ->withDbName($dsn->getDatabase()) + ->withCharset('utf8mb4') + ->withUsername($dsn->getUser()) + ->withPassword($dsn->getPassword()) + ->withOptions([ + PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + ]); + + $pool = new PDOPool($pdoConfig, 64); $this->pools[$name] = $pool; } @@ -191,41 +190,37 @@ class DatabasePool * * @return array */ - public function getDBFromPool(string $projectID, \Redis $redis): array + public function getDBFromPool(string $name): PDO|PDOProxy { /** Get DB name from the console database */ - [$name, $internalID] = $this->getName($projectID, $redis); + // [$name, $internalID] = $this->getName($projectID, $redis); $pool = $this->pools[$name] ?? throw new Exception("Database pool with name : $name not found. Check the value of _APP_DB_PROJECT in .env", 500); + $pdo = $pool->get(); - $namespace = "_$internalID"; - $attempts = 0; - do { - try { - $attempts++; - $pdo = $pool->get(); - $database = $this->getDatabase($pdo, $redis); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); + // $namespace = "_$internalID"; + // $attempts = 0; + // do { + // try { + // $attempts++; + // $pdo = $pool->get(); + // $database = $this->getDatabase($pdo, $redis); + // $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + // $database->setNamespace($namespace); - // if (!$database->exists($database->getDefaultDatabase(), 'metadata')) { - // throw new Exception('Collection not ready'); - // } - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + // // if (!$database->exists($database->getDefaultDatabase(), 'metadata')) { + // // throw new Exception('Collection not ready'); + // // } + // break; // leave loop if successful + // } catch (\Exception $e) { + // Console::warning("Database not ready. Retrying connection ({$attempts})..."); + // if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { + // throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + // } + // sleep(DATABASE_RECONNECT_SLEEP); + // } + // } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - return [ - $database, - function () use ($pdo, $name) { - $this->put($pdo, $name); - } - ]; + return $pdo; } /** @@ -261,13 +256,17 @@ class DatabasePool return [ $database, - function () use ($pdo, $name) { - $this->put($pdo, $name); - }, $name ]; } + public function reset(): void + { + foreach ($this->pools as $pool) { + $pool->reset(); + } + } + /** * Return a PDO instance back to its database pool * diff --git a/src/Appwrite/Database/PDOPool.php b/src/Appwrite/Database/PDOPool.php new file mode 100644 index 0000000000..5b1e938bde --- /dev/null +++ b/src/Appwrite/Database/PDOPool.php @@ -0,0 +1,46 @@ +pool = new SwoolePDOPool($pdoConfig, $size); + } + + public function getActiveConnections() + { + return $this->activeConnections; + } + + public function get(float $timeout = -1) + { + $connection = $this->pool->get($timeout); + $this->activeConnections[] = $connection; + return $connection; + } + + public function put($connection): void + { + $this->pool->put($connection); + unset($this->activeConnections[array_search($connection, $this->activeConnections)]); + } + + public function reset(): void + { + foreach($this->activeConnections as $connection) { + $this->pool->put($connection); + } + + $this->activeConnections = []; + } +} \ No newline at end of file