CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-24 06:29:47 +12:00
Code Issues Projects Releases Wiki Activity

feat(server): database connection pools

Browse source
This commit is contained in:
Torsten Dittmann 2021-06-11 18:09:46 +02:00
parent d96635ec23
commit 665d57106e
16 changed files with 213 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
app/http.php
View file

@ -12,16 +12,6 @@ use Swoole\Http\Request as SwooleRequest;
use Swoole\Http\Response as SwooleResponse;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Domains\Domain;
// xdebug_start_trace('/tmp/trace');
ini_set('memory_limit','512M');
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
ini_set('default_socket_timeout', -1);
error_reporting(E_ALL);
$http = new Server("0.0.0.0", App::getEnv('PORT', 80));
@ -68,7 +58,7 @@ Files::load(__DIR__ . '/../public');
include __DIR__ . '/controllers/general.php';
$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) {
$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) {
$request = new Request($swooleRequest);
$response = new Response($swooleResponse);
@ -85,6 +75,17 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
return;
}
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
$register->set('db', function () use (&$db) {
return $db;
});
$register->set('cache', function () use (&$redis) {
return $redis;
});
$app = new App('UTC');
try {
@ -104,6 +105,14 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
else {
$swooleResponse->end('500: Server Error');
}
} finally {
/** @var PDOPool $dbPool */
$dbPool = $register->get('dbPool');
$dbPool->put($db);
/** @var RedisPool $redisPool */
$redisPool = $register->get('redisPool');
$redisPool->put($redis);
}
});

58
app/init.php
View file

@ -11,6 +11,12 @@ if (\file_exists(__DIR__.'/../vendor/autoload.php')) {
require_once __DIR__.'/../vendor/autoload.php';
}
ini_set('memory_limit','512M');
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
ini_set('default_socket_timeout', -1);
error_reporting(E_ALL);
use Ahc\Jwt\JWT;
use Ahc\Jwt\JWTException;
use Appwrite\Auth\Auth;
@ -18,6 +24,8 @@ use Appwrite\Database\Database;
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
use Appwrite\Database\Adapter\Redis as RedisAdapter;
use Appwrite\Database\Document;
use Appwrite\Database\Pool\PDOPool;
use Appwrite\Database\Pool\RedisPool;
use Appwrite\Database\Validator\Authorization;
use Appwrite\Event\Event;
use Appwrite\Extend\PDO;
@ -145,23 +153,32 @@ Database::addFilter('encrypt',
/*
* Registry
*/
$register->set('db', function () { // Register DB connection
$register->set('dbPool', function () { // Register DB connection
$dbHost = App::getEnv('_APP_DB_HOST', '');
$dbUser = App::getEnv('_APP_DB_USER', '');
$dbPass = App::getEnv('_APP_DB_PASS', '');
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
$pool = new PDOPool(10, $dbHost, $dbScheme, $dbUser, $dbPass);
$pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
PDONative::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4',
PDONative::ATTR_TIMEOUT => 3, // Seconds
PDONative::ATTR_PERSISTENT => true
));
return $pool;
});
$register->set('redisPool', function () {
$redisHost = App::getEnv('_APP_REDIS_HOST', '');
$redisPort = App::getEnv('_APP_REDIS_PORT', '');
$redisUser = App::getEnv('_APP_REDIS_USER', '');
$redisPass = App::getEnv('_APP_REDIS_PASS', '');
$redisAuth = [];
// Connection settings
$pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC); // Return arrays
$pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION); // Handle all errors with exceptions
if ($redisUser) {
$redisAuth[] = $redisUser;
}
if ($redisPass) {
$redisAuth[] = $redisPass;
}
return $pdo;
$pool = new RedisPool(10, $redisHost, $redisPort, $redisAuth);
return $pool;
});
$register->set('influxdb', function () { // Register DB connection
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
@ -185,25 +202,6 @@ $register->set('statsd', function () { // Register DB connection
return $statsd;
});
$register->set('cache', function () { // Register cache connection
$redis = new Redis();
$redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
$user = App::getEnv('_APP_REDIS_USER','');
$pass = App::getEnv('_APP_REDIS_PASS','');
$auth = [];
if(!empty($user)) {
$auth["user"] = $user;
}
if(!empty($pass)) {
$auth["pass"] = $pass;
}
if(!empty($auth)) {
$redis->auth($auth);
}
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
return $redis;
});
$register->set('smtp', function () {
$mail = new PHPMailer(true);
@ -387,7 +385,7 @@ App::setResource('user', function($mode, $project, $console, $request, $response
/** @var Appwrite\Database\Document $project */
/** @var Appwrite\Database\Database $consoleDB */
/** @var Appwrite\Database\Database $projectDB */
/** @var bool $mode */
/** @var string $mode */
Authorization::setDefaultStatus(true);

32
app/workers.php Normal file
View file

@ -0,0 +1,32 @@
<?php
use Appwrite\Extend\PDO;
use Utopia\App;
/** @var Utopia\Registry\Registry $register */
require_once __DIR__.'/init.php';
$register->set('db', function () {
$dbHost = App::getEnv('_APP_DB_HOST', '');
$dbUser = App::getEnv('_APP_DB_USER', '');
$dbPass = App::getEnv('_APP_DB_PASS', '');
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
$pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4',
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
));
return $pdo;
});
$register->set('cache', function () { // Register cache connection
$redis = new Redis();
$redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
return $redis;
});

2
app/workers/audits.php
View file

@ -5,7 +5,7 @@ use Utopia\Audit\Audit;
use Utopia\Audit\Adapters\MySQL as AuditAdapter;
use Utopia\CLI\Console;
require_once __DIR__.'/../init.php';
require_once __DIR__.'/../workers.php';
Console::title('Audits V1 Worker');
Console::success(APP_NAME.' audits worker v1 has started');

2
app/workers/certificates.php
View file

@ -11,7 +11,7 @@ use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Domains\Domain;
require_once __DIR__.'/../init.php';
require_once __DIR__.'/../workers.php';
Console::title('Certificates V1 Worker');
Console::success(APP_NAME.' certificates worker v1 has started');

2
app/workers/deletes.php
View file

@ -14,7 +14,7 @@ use Utopia\Config\Config;
use Utopia\Audit\Audit;
use Utopia\Audit\Adapters\MySQL as AuditAdapter;
require_once __DIR__.'/../init.php';
require_once __DIR__.'/../workers.php';
Console::title('Deletes V1 Worker');
Console::success(APP_NAME.' deletes worker v1 has started'."\n");

2
app/workers/functions.php
View file

@ -13,7 +13,7 @@ use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
require_once __DIR__.'/../init.php';
require_once __DIR__.'/../workers.php';
Runtime::enableCoroutine(0);

2
app/workers/mails.php
View file

@ -4,7 +4,7 @@ use Appwrite\Resque\Worker;
use Utopia\App;
use Utopia\CLI\Console;
require_once __DIR__.'/../init.php';
require_once __DIR__.'/../workers.php';
Console::title('Mails V1 Worker');
Console::success(APP_NAME.' mails worker v1 has started'."\n");

2
app/workers/tasks.php
View file

@ -10,7 +10,7 @@ use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
require_once __DIR__.'/../init.php';
require_once __DIR__.'/../workers.php';
Console::title('Tasks V1 Worker');
Console::success(APP_NAME.' tasks worker v1 has started');

2
app/workers/usage.php
View file

@ -4,7 +4,7 @@ use Appwrite\Resque\Worker;
use Utopia\App;
use Utopia\CLI\Console;
require_once __DIR__.'/../init.php';
require_once __DIR__.'/../workers.php';
Console::title('Usage V1 Worker');
Console::success(APP_NAME.' usage worker v1 has started');

2
app/workers/webhooks.php
View file

@ -4,7 +4,7 @@ use Appwrite\Resque\Worker;
use Utopia\App;
use Utopia\CLI\Console;
require_once __DIR__.'/../init.php';
require_once __DIR__.'/../workers.php';
Console::title('Webhooks V1 Worker');
Console::success(APP_NAME.' webhooks worker v1 has started');

2
composer.json
View file

@ -46,7 +46,7 @@
"utopia-php/cli": "0.11.*",
"utopia-php/config": "0.2.*",
"utopia-php/locale": "0.3.*",
"utopia-php/registry": "0.4.*",
"utopia-php/registry": "0.5.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/domains": "1.1.*",
"utopia-php/swoole": "0.2.*",

26
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "6704f7df5ffe0baac3633dc8e683ed78",
"content-hash": "19426f6ef8c8da256a0ff0fba2c11051",
"packages": [
{
"name": "adhocore/jwt",
@ -1899,16 +1899,16 @@
},
{
"name": "utopia-php/registry",
"version": "0.4.0",
"version": "0.5.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/registry.git",
"reference": "7aebbc6c5f3f04ff7a35ac3dad39fa91c9bd7c2d"
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/registry/zipball/7aebbc6c5f3f04ff7a35ac3dad39fa91c9bd7c2d",
"reference": "7aebbc6c5f3f04ff7a35ac3dad39fa91c9bd7c2d",
"url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d",
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d",
"shasum": ""
},
"require": {
@ -1945,9 +1945,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/registry/issues",
"source": "https://github.com/utopia-php/registry/tree/0.4.0"
"source": "https://github.com/utopia-php/registry/tree/0.5.0"
},
"time": "2021-03-10T06:50:09+00:00"
"time": "2021-03-10T10:45:22+00:00"
},
{
"name": "utopia-php/storage",
@ -4472,16 +4472,16 @@
},
{
"name": "sebastian/global-state",
"version": "5.0.2",
"version": "5.0.3",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/global-state.git",
"reference": "a90ccbddffa067b51f574dea6eb25d5680839455"
"reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/a90ccbddffa067b51f574dea6eb25d5680839455",
"reference": "a90ccbddffa067b51f574dea6eb25d5680839455",
"url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/23bd5951f7ff26f12d4e3242864df3e08dec4e49",
"reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49",
"shasum": ""
},
"require": {
@ -4524,7 +4524,7 @@
],
"support": {
"issues": "https://github.com/sebastianbergmann/global-state/issues",
"source": "https://github.com/sebastianbergmann/global-state/tree/5.0.2"
"source": "https://github.com/sebastianbergmann/global-state/tree/5.0.3"
},
"funding": [
{
@ -4532,7 +4532,7 @@
"type": "github"
}
],
"time": "2020-10-26T15:55:19+00:00"
"time": "2021-06-11T13:31:12+00:00"
},
{
"name": "sebastian/lines-of-code",

20
src/Appwrite/Database/Pool.php Normal file
View file

@ -0,0 +1,20 @@
<?php
namespace Appwrite\Database;
abstract class Pool
{
protected $available = true;
protected $pool;
protected $size = 5;
abstract public function get();
public function destruct()
{
$this->available = false;
while (!$this->pool->isEmpty()) {
$this->pool->pop();
}
}
}

49
src/Appwrite/Database/Pool/PDOPool.php Normal file
View file

@ -0,0 +1,49 @@
<?php
namespace Appwrite\Database\Pool;
use Appwrite\Database\Pool;
use Appwrite\Extend\PDO;
use SplQueue;
class PDOPool extends Pool
{
public function __construct(int $size, string $host = 'localhost', string $schema = 'appwrite', string $user = '', string $pass = '', string $charset = 'utf8mb4')
{
$this->pool = new SplQueue;
$this->size = $size;
for ($i = 0; $i < $this->size; $i++) {
$pdo = new PDO(
"mysql:" .
"host={$host};" .
"dbname={$schema};" .
"charset={$charset}",
$user,
$pass,
[
PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4',
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true
]
);
$this->pool->enqueue($pdo);
}
}
public function put(PDO $pdo)
{
$this->pool->enqueue($pdo);
}
public function get(): PDO
{
if ($this->available && count($this->pool) > 0) {
return $this->pool->dequeue();
}
sleep(0.01);
return $this->get();
}
}

42
src/Appwrite/Database/Pool/RedisPool.php Normal file
View file

@ -0,0 +1,42 @@
<?php
namespace Appwrite\Database\Pool;
use Appwrite\Database\Pool;
use SplQueue;
use Redis;
class RedisPool extends Pool
{
public function __construct(int $size, string $host, int $port, array $auth = [])
{
$this->pool = new SplQueue;
$this->size = $size;
for ($i = 0; $i < $this->size; $i++) {
$redis = new Redis();
$redis->pconnect($host, $port);
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
if ($auth) {
$redis->auth($auth);
}
$this->pool->enqueue($redis);
}
}
public function put(Redis $redis)
{
$this->pool->enqueue($redis);
}
public function get(): Redis
{
if ($this->available && !$this->pool->isEmpty()) {
return $this->pool->dequeue();
}
sleep(0.1);
return $this->get();
}
}