Merge pull request #1278 from TorstenDittmann/feat-db-pools
feat(server): database connection pools
This commit is contained in:
commit
d469e3a7b6
16 changed files with 212 additions and 64 deletions
31
app/http.php
31
app/http.php
|
@ -12,16 +12,6 @@ use Swoole\Http\Request as SwooleRequest;
|
|||
use Swoole\Http\Response as SwooleResponse;
|
||||
use Utopia\App;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Domains\Domain;
|
||||
|
||||
// xdebug_start_trace('/tmp/trace');
|
||||
|
||||
ini_set('memory_limit','512M');
|
||||
ini_set('display_errors', 1);
|
||||
ini_set('display_startup_errors', 1);
|
||||
ini_set('default_socket_timeout', -1);
|
||||
error_reporting(E_ALL);
|
||||
|
||||
$http = new Server("0.0.0.0", App::getEnv('PORT', 80));
|
||||
|
||||
|
@ -68,7 +58,7 @@ Files::load(__DIR__ . '/../public');
|
|||
|
||||
include __DIR__ . '/controllers/general.php';
|
||||
|
||||
$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) {
|
||||
$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) {
|
||||
$request = new Request($swooleRequest);
|
||||
$response = new Response($swooleResponse);
|
||||
|
||||
|
@ -85,6 +75,17 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
|
|||
return;
|
||||
}
|
||||
|
||||
$db = $register->get('dbPool')->get();
|
||||
$redis = $register->get('redisPool')->get();
|
||||
|
||||
$register->set('db', function () use (&$db) {
|
||||
return $db;
|
||||
});
|
||||
|
||||
$register->set('cache', function () use (&$redis) {
|
||||
return $redis;
|
||||
});
|
||||
|
||||
$app = new App('UTC');
|
||||
|
||||
try {
|
||||
|
@ -104,6 +105,14 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
|
|||
else {
|
||||
$swooleResponse->end('500: Server Error');
|
||||
}
|
||||
} finally {
|
||||
/** @var PDOPool $dbPool */
|
||||
$dbPool = $register->get('dbPool');
|
||||
$dbPool->put($db);
|
||||
|
||||
/** @var RedisPool $redisPool */
|
||||
$redisPool = $register->get('redisPool');
|
||||
$redisPool->put($redis);
|
||||
}
|
||||
});
|
||||
|
||||
|
|
60
app/init.php
60
app/init.php
|
@ -11,6 +11,12 @@ if (\file_exists(__DIR__.'/../vendor/autoload.php')) {
|
|||
require_once __DIR__.'/../vendor/autoload.php';
|
||||
}
|
||||
|
||||
ini_set('memory_limit','512M');
|
||||
ini_set('display_errors', 1);
|
||||
ini_set('display_startup_errors', 1);
|
||||
ini_set('default_socket_timeout', -1);
|
||||
error_reporting(E_ALL);
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Ahc\Jwt\JWTException;
|
||||
use Appwrite\Auth\Auth;
|
||||
|
@ -18,9 +24,10 @@ use Appwrite\Database\Database;
|
|||
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
|
||||
use Appwrite\Database\Adapter\Redis as RedisAdapter;
|
||||
use Appwrite\Database\Document;
|
||||
use Appwrite\Database\Pool\PDOPool;
|
||||
use Appwrite\Database\Pool\RedisPool;
|
||||
use Appwrite\Database\Validator\Authorization;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\PDO;
|
||||
use Appwrite\OpenSSL\OpenSSL;
|
||||
use Utopia\App;
|
||||
use Utopia\View;
|
||||
|
@ -29,7 +36,6 @@ use Utopia\Locale\Locale;
|
|||
use Utopia\Registry\Registry;
|
||||
use MaxMind\Db\Reader;
|
||||
use PHPMailer\PHPMailer\PHPMailer;
|
||||
use PDO as PDONative;
|
||||
|
||||
const APP_NAME = 'Appwrite';
|
||||
const APP_DOMAIN = 'appwrite.io';
|
||||
|
@ -145,23 +151,32 @@ Database::addFilter('encrypt',
|
|||
/*
|
||||
* Registry
|
||||
*/
|
||||
$register->set('db', function () { // Register DB connection
|
||||
$register->set('dbPool', function () { // Register DB connection
|
||||
$dbHost = App::getEnv('_APP_DB_HOST', '');
|
||||
$dbUser = App::getEnv('_APP_DB_USER', '');
|
||||
$dbPass = App::getEnv('_APP_DB_PASS', '');
|
||||
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
|
||||
$pool = new PDOPool(10, $dbHost, $dbScheme, $dbUser, $dbPass);
|
||||
|
||||
$pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
|
||||
PDONative::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4',
|
||||
PDONative::ATTR_TIMEOUT => 3, // Seconds
|
||||
PDONative::ATTR_PERSISTENT => true
|
||||
));
|
||||
return $pool;
|
||||
});
|
||||
$register->set('redisPool', function () {
|
||||
$redisHost = App::getEnv('_APP_REDIS_HOST', '');
|
||||
$redisPort = App::getEnv('_APP_REDIS_PORT', '');
|
||||
$redisUser = App::getEnv('_APP_REDIS_USER', '');
|
||||
$redisPass = App::getEnv('_APP_REDIS_PASS', '');
|
||||
$redisAuth = [];
|
||||
|
||||
// Connection settings
|
||||
$pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC); // Return arrays
|
||||
$pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION); // Handle all errors with exceptions
|
||||
if ($redisUser) {
|
||||
$redisAuth[] = $redisUser;
|
||||
}
|
||||
if ($redisPass) {
|
||||
$redisAuth[] = $redisPass;
|
||||
}
|
||||
|
||||
return $pdo;
|
||||
$pool = new RedisPool(10, $redisHost, $redisPort, $redisAuth);
|
||||
|
||||
return $pool;
|
||||
});
|
||||
$register->set('influxdb', function () { // Register DB connection
|
||||
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
|
||||
|
@ -185,25 +200,6 @@ $register->set('statsd', function () { // Register DB connection
|
|||
|
||||
return $statsd;
|
||||
});
|
||||
$register->set('cache', function () { // Register cache connection
|
||||
$redis = new Redis();
|
||||
$redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
|
||||
$user = App::getEnv('_APP_REDIS_USER','');
|
||||
$pass = App::getEnv('_APP_REDIS_PASS','');
|
||||
$auth = [];
|
||||
if(!empty($user)) {
|
||||
$auth["user"] = $user;
|
||||
}
|
||||
if(!empty($pass)) {
|
||||
$auth["pass"] = $pass;
|
||||
}
|
||||
if(!empty($auth)) {
|
||||
$redis->auth($auth);
|
||||
}
|
||||
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
|
||||
|
||||
return $redis;
|
||||
});
|
||||
$register->set('smtp', function () {
|
||||
$mail = new PHPMailer(true);
|
||||
|
||||
|
@ -387,7 +383,7 @@ App::setResource('user', function($mode, $project, $console, $request, $response
|
|||
/** @var Appwrite\Database\Document $project */
|
||||
/** @var Appwrite\Database\Database $consoleDB */
|
||||
/** @var Appwrite\Database\Database $projectDB */
|
||||
/** @var bool $mode */
|
||||
/** @var string $mode */
|
||||
|
||||
Authorization::setDefaultStatus(true);
|
||||
|
||||
|
|
32
app/workers.php
Normal file
32
app/workers.php
Normal file
|
@ -0,0 +1,32 @@
|
|||
<?php
|
||||
|
||||
use Appwrite\Extend\PDO;
|
||||
use Utopia\App;
|
||||
|
||||
/** @var Utopia\Registry\Registry $register */
|
||||
|
||||
require_once __DIR__.'/init.php';
|
||||
|
||||
$register->set('db', function () {
|
||||
$dbHost = App::getEnv('_APP_DB_HOST', '');
|
||||
$dbUser = App::getEnv('_APP_DB_USER', '');
|
||||
$dbPass = App::getEnv('_APP_DB_PASS', '');
|
||||
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
|
||||
|
||||
$pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
|
||||
PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4',
|
||||
PDO::ATTR_TIMEOUT => 3, // Seconds
|
||||
PDO::ATTR_PERSISTENT => true,
|
||||
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
|
||||
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
|
||||
));
|
||||
|
||||
return $pdo;
|
||||
});
|
||||
$register->set('cache', function () { // Register cache connection
|
||||
$redis = new Redis();
|
||||
$redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
|
||||
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
|
||||
|
||||
return $redis;
|
||||
});
|
|
@ -5,7 +5,7 @@ use Utopia\Audit\Audit;
|
|||
use Utopia\Audit\Adapters\MySQL as AuditAdapter;
|
||||
use Utopia\CLI\Console;
|
||||
|
||||
require_once __DIR__.'/../init.php';
|
||||
require_once __DIR__.'/../workers.php';
|
||||
|
||||
Console::title('Audits V1 Worker');
|
||||
Console::success(APP_NAME.' audits worker v1 has started');
|
||||
|
|
|
@ -11,7 +11,7 @@ use Utopia\CLI\Console;
|
|||
use Utopia\Config\Config;
|
||||
use Utopia\Domains\Domain;
|
||||
|
||||
require_once __DIR__.'/../init.php';
|
||||
require_once __DIR__.'/../workers.php';
|
||||
|
||||
Console::title('Certificates V1 Worker');
|
||||
Console::success(APP_NAME.' certificates worker v1 has started');
|
||||
|
|
|
@ -14,7 +14,7 @@ use Utopia\Config\Config;
|
|||
use Utopia\Audit\Audit;
|
||||
use Utopia\Audit\Adapters\MySQL as AuditAdapter;
|
||||
|
||||
require_once __DIR__.'/../init.php';
|
||||
require_once __DIR__.'/../workers.php';
|
||||
|
||||
Console::title('Deletes V1 Worker');
|
||||
Console::success(APP_NAME.' deletes worker v1 has started'."\n");
|
||||
|
|
|
@ -13,7 +13,7 @@ use Utopia\App;
|
|||
use Utopia\CLI\Console;
|
||||
use Utopia\Config\Config;
|
||||
|
||||
require_once __DIR__.'/../init.php';
|
||||
require_once __DIR__.'/../workers.php';
|
||||
|
||||
Runtime::enableCoroutine(0);
|
||||
|
||||
|
|
|
@ -4,7 +4,7 @@ use Appwrite\Resque\Worker;
|
|||
use Utopia\App;
|
||||
use Utopia\CLI\Console;
|
||||
|
||||
require_once __DIR__.'/../init.php';
|
||||
require_once __DIR__.'/../workers.php';
|
||||
|
||||
Console::title('Mails V1 Worker');
|
||||
Console::success(APP_NAME.' mails worker v1 has started'."\n");
|
||||
|
|
|
@ -10,7 +10,7 @@ use Utopia\App;
|
|||
use Utopia\CLI\Console;
|
||||
use Utopia\Config\Config;
|
||||
|
||||
require_once __DIR__.'/../init.php';
|
||||
require_once __DIR__.'/../workers.php';
|
||||
|
||||
Console::title('Tasks V1 Worker');
|
||||
Console::success(APP_NAME.' tasks worker v1 has started');
|
||||
|
|
|
@ -4,7 +4,7 @@ use Appwrite\Resque\Worker;
|
|||
use Utopia\App;
|
||||
use Utopia\CLI\Console;
|
||||
|
||||
require_once __DIR__.'/../init.php';
|
||||
require_once __DIR__.'/../workers.php';
|
||||
|
||||
Console::title('Usage V1 Worker');
|
||||
Console::success(APP_NAME.' usage worker v1 has started');
|
||||
|
|
|
@ -4,7 +4,7 @@ use Appwrite\Resque\Worker;
|
|||
use Utopia\App;
|
||||
use Utopia\CLI\Console;
|
||||
|
||||
require_once __DIR__.'/../init.php';
|
||||
require_once __DIR__.'/../workers.php';
|
||||
|
||||
Console::title('Webhooks V1 Worker');
|
||||
Console::success(APP_NAME.' webhooks worker v1 has started');
|
||||
|
|
|
@ -46,7 +46,7 @@
|
|||
"utopia-php/cli": "0.11.*",
|
||||
"utopia-php/config": "0.2.*",
|
||||
"utopia-php/locale": "0.3.*",
|
||||
"utopia-php/registry": "0.4.*",
|
||||
"utopia-php/registry": "0.5.*",
|
||||
"utopia-php/preloader": "0.2.*",
|
||||
"utopia-php/domains": "1.1.*",
|
||||
"utopia-php/swoole": "0.2.*",
|
||||
|
|
24
composer.lock
generated
24
composer.lock
generated
|
@ -1899,16 +1899,16 @@
|
|||
},
|
||||
{
|
||||
"name": "utopia-php/registry",
|
||||
"version": "0.4.0",
|
||||
"version": "0.5.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/registry.git",
|
||||
"reference": "7aebbc6c5f3f04ff7a35ac3dad39fa91c9bd7c2d"
|
||||
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/registry/zipball/7aebbc6c5f3f04ff7a35ac3dad39fa91c9bd7c2d",
|
||||
"reference": "7aebbc6c5f3f04ff7a35ac3dad39fa91c9bd7c2d",
|
||||
"url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d",
|
||||
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
|
@ -1945,9 +1945,9 @@
|
|||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/registry/issues",
|
||||
"source": "https://github.com/utopia-php/registry/tree/0.4.0"
|
||||
"source": "https://github.com/utopia-php/registry/tree/0.5.0"
|
||||
},
|
||||
"time": "2021-03-10T06:50:09+00:00"
|
||||
"time": "2021-03-10T10:45:22+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/storage",
|
||||
|
@ -4472,16 +4472,16 @@
|
|||
},
|
||||
{
|
||||
"name": "sebastian/global-state",
|
||||
"version": "5.0.2",
|
||||
"version": "5.0.3",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/sebastianbergmann/global-state.git",
|
||||
"reference": "a90ccbddffa067b51f574dea6eb25d5680839455"
|
||||
"reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/a90ccbddffa067b51f574dea6eb25d5680839455",
|
||||
"reference": "a90ccbddffa067b51f574dea6eb25d5680839455",
|
||||
"url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/23bd5951f7ff26f12d4e3242864df3e08dec4e49",
|
||||
"reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
|
@ -4524,7 +4524,7 @@
|
|||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/sebastianbergmann/global-state/issues",
|
||||
"source": "https://github.com/sebastianbergmann/global-state/tree/5.0.2"
|
||||
"source": "https://github.com/sebastianbergmann/global-state/tree/5.0.3"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
|
@ -4532,7 +4532,7 @@
|
|||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2020-10-26T15:55:19+00:00"
|
||||
"time": "2021-06-11T13:31:12+00:00"
|
||||
},
|
||||
{
|
||||
"name": "sebastian/lines-of-code",
|
||||
|
|
20
src/Appwrite/Database/Pool.php
Normal file
20
src/Appwrite/Database/Pool.php
Normal file
|
@ -0,0 +1,20 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Database;
|
||||
|
||||
abstract class Pool
|
||||
{
|
||||
protected $available = true;
|
||||
protected $pool;
|
||||
protected $size = 5;
|
||||
|
||||
abstract public function get();
|
||||
|
||||
public function destruct()
|
||||
{
|
||||
$this->available = false;
|
||||
while (!$this->pool->isEmpty()) {
|
||||
$this->pool->pop();
|
||||
}
|
||||
}
|
||||
}
|
49
src/Appwrite/Database/Pool/PDOPool.php
Normal file
49
src/Appwrite/Database/Pool/PDOPool.php
Normal file
|
@ -0,0 +1,49 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Database\Pool;
|
||||
|
||||
use Appwrite\Database\Pool;
|
||||
use Appwrite\Extend\PDO;
|
||||
use SplQueue;
|
||||
|
||||
class PDOPool extends Pool
|
||||
{
|
||||
public function __construct(int $size, string $host = 'localhost', string $schema = 'appwrite', string $user = '', string $pass = '', string $charset = 'utf8mb4')
|
||||
{
|
||||
$this->pool = new SplQueue;
|
||||
$this->size = $size;
|
||||
for ($i = 0; $i < $this->size; $i++) {
|
||||
$pdo = new PDO(
|
||||
"mysql:" .
|
||||
"host={$host};" .
|
||||
"dbname={$schema};" .
|
||||
"charset={$charset}",
|
||||
$user,
|
||||
$pass,
|
||||
[
|
||||
PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4',
|
||||
PDO::ATTR_TIMEOUT => 3, // Seconds
|
||||
PDO::ATTR_PERSISTENT => true,
|
||||
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
|
||||
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
|
||||
PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true
|
||||
]
|
||||
);
|
||||
$this->pool->enqueue($pdo);
|
||||
}
|
||||
}
|
||||
|
||||
public function put(PDO $pdo)
|
||||
{
|
||||
$this->pool->enqueue($pdo);
|
||||
}
|
||||
|
||||
public function get(): PDO
|
||||
{
|
||||
if ($this->available && count($this->pool) > 0) {
|
||||
return $this->pool->dequeue();
|
||||
}
|
||||
sleep(0.01);
|
||||
return $this->get();
|
||||
}
|
||||
}
|
42
src/Appwrite/Database/Pool/RedisPool.php
Normal file
42
src/Appwrite/Database/Pool/RedisPool.php
Normal file
|
@ -0,0 +1,42 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Database\Pool;
|
||||
|
||||
use Appwrite\Database\Pool;
|
||||
use SplQueue;
|
||||
|
||||
use Redis;
|
||||
|
||||
class RedisPool extends Pool
|
||||
{
|
||||
public function __construct(int $size, string $host, int $port, array $auth = [])
|
||||
{
|
||||
$this->pool = new SplQueue;
|
||||
$this->size = $size;
|
||||
for ($i = 0; $i < $this->size; $i++) {
|
||||
$redis = new Redis();
|
||||
$redis->pconnect($host, $port);
|
||||
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
|
||||
|
||||
if ($auth) {
|
||||
$redis->auth($auth);
|
||||
}
|
||||
|
||||
$this->pool->enqueue($redis);
|
||||
}
|
||||
}
|
||||
|
||||
public function put(Redis $redis)
|
||||
{
|
||||
$this->pool->enqueue($redis);
|
||||
}
|
||||
|
||||
public function get(): Redis
|
||||
{
|
||||
if ($this->available && !$this->pool->isEmpty()) {
|
||||
return $this->pool->dequeue();
|
||||
}
|
||||
sleep(0.1);
|
||||
return $this->get();
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue