CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-10-01 01:37:56 +13:00
Code Issues Projects Releases Wiki Activity

feat: add db-pools

Browse source
This commit is contained in:
Christy Jacob 2022-06-23 10:50:00 +02:00
parent 039d9f0ead
commit 69f1798758
4 changed files with 178 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
app/controllers/api/projects.php
View file

@ -2,6 +2,7 @@
use Appwrite\Auth\Auth;
use Appwrite\Auth\Validator\Password;
use Appwrite\Database\DatabasePool;
use Appwrite\Event\Certificate;
use Appwrite\Event\Delete;
use Appwrite\Event\Validator\Event;
@ -10,6 +11,8 @@ use Appwrite\Network\Validator\Domain as DomainValidator;
use Appwrite\Network\Validator\Origin;
use Appwrite\Network\Validator\URL;
use Appwrite\Utopia\Database\Validator\CustomId;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Appwrite\Utopia\Response;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\App;
@ -23,6 +26,7 @@ use Utopia\Database\Validator\UID;
use Utopia\Domains\Domain;
use Utopia\Registry\Registry;
use Appwrite\Extend\Exception;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Boolean;
use Utopia\Validator\Hostname;
@ -61,8 +65,9 @@ App::post('/v1/projects')
->param('legalTaxId', '', new Text(256), 'Project legal Tax ID. Max length: 256 chars.', true)
->inject('response')
->inject('dbForConsole')
->inject('dbForProject')
->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Database $dbForProject) {
->inject('cache')
->inject('dbPool')
->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Redis $cache, DatabasePool $dbPool) {
$team = $dbForConsole->getDocument('teams', $teamId);
@ -80,6 +85,9 @@ App::post('/v1/projects')
if ($projectId === 'console') {
throw new Exception("'console' is a reserved project.", 400, Exception::PROJECT_RESERVED_PROJECT);
}
['name' => $dbName, 'db' => $db] = $dbPool->getAny();
$project = $dbForConsole->createDocument('projects', new Document([
'$id' => $projectId == 'unique()' ? $dbForConsole->getId() : $projectId,
'$read' => ['team:' . $teamId],
@ -104,12 +112,13 @@ App::post('/v1/projects')
'domains' => null,
'auths' => $auths,
'search' => implode(' ', [$projectId, $name]),
'database'
'database' => $dbName
]));
/** @var array $collections */
$collections = Config::getParam('collections', []);
$dbForProject->setNamespace("_{$project->getId()}");
$cache = new Cache(new RedisCache($cache));
$dbForProject = new Database(new MariaDB($db), $cache);
$dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$dbForProject->setNamespace("_{$projectId}");
$dbForProject->create('appwrite');
$audit = new Audit($dbForProject);
@ -118,6 +127,9 @@ App::post('/v1/projects')
$adapter = new TimeLimit('', 0, 1, $dbForProject);
$adapter->setup();
/** @var array $collections */
$collections = Config::getParam('collections', []);
foreach ($collections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
@ -152,6 +164,8 @@ App::post('/v1/projects')
$dbForProject->createCollection($key, $attributes, $indexes);
}
$dbPool->put($db, $dbName);
$response->setStatusCode(Response::STATUS_CODE_CREATED);
$response->dynamic($project, Response::MODEL_PROJECT);
});

37
app/http.php
View file

@ -17,6 +17,9 @@ use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Swoole\Files;
use Appwrite\Utopia\Request;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Logger\Log;
use Utopia\Logger\Log\User;
@ -66,7 +69,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
do {
try {
$attempts++;
$pool = $register->get('poolForConsole')->get();
$db = $register->get('dbPool')->getConsoleDB();
$redis = $register->get('redisPool')->get();
break; // leave the do-while if successful
} catch (\Exception $e) {
@ -239,12 +242,33 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$app = new App('UTC');
$db = $register->get('dbPool')->get();
$dbPool = $register->get('dbPool');
$db = $dbPool->getConsoleDB();
$redis = $register->get('redisPool')->get();
App::setResource('db', fn() => $db);
App::setResource('dbPool', fn() => $dbPool);
App::setResource('cache', fn() => $redis);
$projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', 'console'));
$projectDB = $db;
if ($projectId !== 'console') {
$dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */
$project = Authorization::skip(fn() => $dbForConsole->getDocument('projects', $projectId));
$dbName = $project->getAttribute('database', '');
if (!empty($dbName)) {
$projectDB = $register->get('dbPool')->get($dbName);
}
}
App::setResource('dbForProject', function ($cache) use ($projectDB, $projectId) {
$cache = new Cache(new RedisCache($cache));
$database = new Database(new MariaDB($projectDB), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace("_{$projectId}");
return $database;
}, ['cache']);
try {
Authorization::cleanRoles();
Authorization::setRole('role:all');
@ -334,9 +358,12 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$swooleResponse->end(\json_encode($output));
} finally {
/** @var PDOPool $dbPool */
$dbPool = $register->get('dbPool');
$dbPool->put($db);
/** @var PDOPool $consolePool */
$dbPool->putConsoleDb($db);
if (!empty($dbName) && !empty($projectDB)) {
$dbPool->put($projectDB, $dbName);
}
/** @var RedisPool $redisPool */
$redisPool = $register->get('redisPool');

94
app/init.php
View file

@ -445,80 +445,43 @@ $register->set('logger', function () {
return new Logger($adapter);
});
$register->set('poolForConsole', function () {
$dbs = App::getEnv('_APP_CONSOLE_DB', '');
$dbs = explode(',', $dbs);
$register->set('dbPool', function () {
/** Parse the console databases */
$consoleDb = App::getEnv('_APP_CONSOLE_DB', '');
$consoleDb = explode(',', $consoleDb)[0];
$consoleDb = explode('=', $consoleDb);
$name = $consoleDb[0];
$dsn = new DSN($consoleDb[1]);
$pools = new DatabasePool();
foreach ($dbs as $db) {
$db = explode('=', $db);
$name = $db[0];
$dsn = new DSN($db[1]);
/** Create a new Database Pool */
$pool = new DatabasePool();
// var_dump($dsn->getHost(), $dsn->getPort(), $dsn->getDatabase(), $dsn->getUser(), $dsn->getPassword());
$consolePool = new PDOPool(
(new PDOConfig())
->withHost($dsn->getHost())
->withPort($dsn->getPort())
->withDbName($dsn->getDatabase())
->withCharset('utf8mb4')
->withUsername($dsn->getUser())
->withPassword($dsn->getPassword())
->withOptions([
PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
]),
64
);
$pool = new PDOPool(
(new PDOConfig())
->withHost($dsn->getHost())
->withPort($dsn->getPort())
->withDbName($dsn->getDatabase())
->withCharset('utf8mb4')
->withUsername($dsn->getUser())
->withPassword($dsn->getPassword())
->withOptions([
PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
]),
64
);
$pools->add($name, $pool);
}
return $pools;
});
$register->set('poolForProject', function () {
// Register DB connection
// $dbHost = App::getEnv('_APP_DB_HOST', '');
// $dbPort = App::getEnv('_APP_DB_PORT', '');
// $dbUser = App::getEnv('_APP_DB_USER', '');
// $dbPass = App::getEnv('_APP_DB_PASS', '');
// $dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
// var_dump($dbHost, $dbPort, $dbScheme, $dbUser, $dbPass);
// $pool = new PDOPool(
// (new PDOConfig())
// ->withHost($dbHost)
// ->withPort($dbPort)
// ->withDbName($dbScheme)
// ->withCharset('utf8mb4')
// ->withUsername($dbUser)
// ->withPassword($dbPass)
// ->withOptions([
// PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
// ]),
// 64
// );
// return $pool;
// $dbForConsole = App::getEnv('_APP_CONSOLE_DB', '');
// $dbForConsole = explode('=', $dbForConsole);
// $name = $dbForConsole[0];
// $dsn = new DSN($dbForConsole[1]);
$pool->add($name, $consolePool);
$pool->setConsoleDB($name);
/** Parse the project databases */
$dbs = App::getEnv('_APP_PROJECT_DB', '');
$dbs = explode(',', $dbs);
$pools = new DatabasePool();
foreach ($dbs as $db) {
$db = explode('=', $db);
$name = $db[0];
$dsn = new DSN($db[1]);
// var_dump($dsn->getHost(), $dsn->getPort(), $dsn->getDatabase(), $dsn->getUser(), $dsn->getPassword());
$pool = new PDOPool(
$projectPool = new PDOPool(
(new PDOConfig())
->withHost($dsn->getHost())
->withPort($dsn->getPort())
@ -532,11 +495,12 @@ $register->set('poolForProject', function () {
64
);
$pools->add($name, $pool);
$pool->add($name, $projectPool);
}
return $pools;
return $pool;
});
$register->set('redisPool', function () {
$redisHost = App::getEnv('_APP_REDIS_HOST', '');
$redisPort = App::getEnv('_APP_REDIS_PORT', '');

103
src/Appwrite/Database/DatabasePool.php
View file

@ -2,36 +2,127 @@
namespace Appwrite\Database;
use PDO;
use Appwrite\Extend\Exception;
use Swoole\Database\PDOPool;
use Swoole\Database\PDOProxy;
class DatabasePool {
/**
* @var array
*/
protected array $pools = [];
/**
* @var string
*/
protected string $consoleDB = '';
/**
* Function to get the name of the console database.
*
* @return ?PDOProxy
*/
public function getConsoleDB(): ?PDOProxy
{
if (empty($this->consoleDB)) {
throw new Exception("Console DB not set", 500);
}
return $this->get($this->consoleDB);
}
/**
* Return a PDO instance back to the console database pool
*
* @param PDOProxy $db
*
* @return void
*/
public function putConsoleDb(PDOProxy $db): void
{
$this->put($db, $this->consoleDB);
}
/**
* Function to set the name of the console database
*
* @param string $consoleDB
*
* @return void
*/
public function setConsoleDB(string $consoleDB): void
{
if(!isset($this->pools[$consoleDB])) {
throw new Exception("Console DB with name : $consoleDB not found. Add it using ", 500);
}
$this->consoleDB = $consoleDB;
}
/**
* Add a new PDOPool into the list of available pools
*
* @param string $name
* @param PDOPool $dbPool
*
* @return void
*/
public function add(string $name, PDOPool $dbPool): void
{
$this->pools[$name] = $dbPool;
}
public function get(string $name = 'console'): ?PDOProxy
/**
* Get a PDO instance from the list of available database pools
*
* @param string $name
*
* @return ?PDOProxy
*/
public function get(string $name): ?PDOProxy
{
$pool = $this->pools[$name] ?? null;
if ($pool === null) {
throw new Exception("Database Pool with name : $name not found. Please check the value of _APP_PROJECT_DB in .env", 500);
throw new Exception("Database pool with name : $name not found. Check the value of _APP_PROJECT_DB in .env", 500);
}
return $pool->get();
}
public function put(PDOProxy $db, string $name = 'console'): void
/**
* Function to get a random PDO instance from the available database pools database
*
* @return array [PDO, string]
*/
public function getAny(): ?array
{
if (count($this->pools) === 0) {
throw new Exception("No database pools found. Add pools using DatabasePool::add() method", 500);
}
$key = array_rand($this->pools);
$pool = $this->pools[$key] ?? null;
return [
'name' => $key,
'db' => $pool ? $pool->get() : null
];
}
/**
* Return a PDO instance back to its database pool
*
* @param PDOProxy $db
* @param string $name
*
* @return void
*/
public function put(PDOProxy $db, string $name): void
{
$pool = $this->pools[$name] ?? null;
if ($pool === null) {
throw new Exception("Database Pool with name : $name not found. Cannot put", 500);
throw new Exception("Failed to put PDO into database pool. Database pool with name : $name not found", 500);
}
$pool->put($db);
}
}