CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-02 19:04:49 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1318 from appwrite/feat-pools-poc

Browse source 
POC
This commit is contained in:
Torsten Dittmann 2021-06-23 15:24:01 +02:00 committed by GitHub
parent 241255dabf 73489d6b73
commit 1e972db792
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 124 additions and 190 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -15,7 +15,7 @@ RUN composer update --ignore-platform-reqs --optimize-autoloader \
FROM php:8.0-cli-alpine as step1
ENV PHP_REDIS_VERSION=5.3.4 \
PHP_SWOOLE_VERSION=v4.6.6 \
PHP_SWOOLE_VERSION=v4.6.7 \
PHP_IMAGICK_VERSION=master \
PHP_YAML_VERSION=2.2.1 \
PHP_MAXMINDDB_VERSION=v1.10.1

70
app/controllers/shared/api.php
View file

@ -31,47 +31,47 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e
throw new Exception('Missing or unknown project ID', 400);
}
/*
* Abuse Check
*/
$timeLimit = new TimeLimit($route->getLabel('abuse-key', 'url:{url},ip:{ip}'), $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), function () use ($register) {
return $register->get('db');
});
$timeLimit->setNamespace('app_'.$project->getId());
$timeLimit
->setParam('{userId}', $user->getId())
->setParam('{userAgent}', $request->getUserAgent(''))
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getHostname().$route->getURL())
;
// /*
// * Abuse Check
// */
// $timeLimit = new TimeLimit($route->getLabel('abuse-key', 'url:{url},ip:{ip}'), $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), function () use ($register) {
// return $register->get('db');
// });
// $timeLimit->setNamespace('app_'.$project->getId());
// $timeLimit
// ->setParam('{userId}', $user->getId())
// ->setParam('{userAgent}', $request->getUserAgent(''))
// ->setParam('{ip}', $request->getIP())
// ->setParam('{url}', $request->getHostname().$route->getURL())
// ;
//TODO make sure we get array here
// //TODO make sure we get array here
foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys
if(!empty($value)) {
$timeLimit->setParam('{param-'.$key.'}', (\is_array($value)) ? \json_encode($value) : $value);
}
}
// foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys
// if(!empty($value)) {
// $timeLimit->setParam('{param-'.$key.'}', (\is_array($value)) ? \json_encode($value) : $value);
// }
// }
$abuse = new Abuse($timeLimit);
// $abuse = new Abuse($timeLimit);
if ($timeLimit->limit()) {
$response
->addHeader('X-RateLimit-Limit', $timeLimit->limit())
->addHeader('X-RateLimit-Remaining', $timeLimit->remaining())
->addHeader('X-RateLimit-Reset', $timeLimit->time() + $route->getLabel('abuse-time', 3600))
;
}
// if ($timeLimit->limit()) {
// $response
// ->addHeader('X-RateLimit-Limit', $timeLimit->limit())
// ->addHeader('X-RateLimit-Remaining', $timeLimit->remaining())
// ->addHeader('X-RateLimit-Reset', $timeLimit->time() + $route->getLabel('abuse-time', 3600))
// ;
// }
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles);
$isAppUser = Auth::isAppUser(Authorization::$roles);
// $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles);
// $isAppUser = Auth::isAppUser(Authorization::$roles);
if (($abuse->check() // Route is rate-limited
&& App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled') // Abuse is not diabled
&& (!$isAppUser && !$isPrivilegedUser)) // User is not an admin or API key
{
throw new Exception('Too many requests', 429);
}
// if (($abuse->check() // Route is rate-limited
// && App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled') // Abuse is not diabled
// && (!$isAppUser && !$isPrivilegedUser)) // User is not an admin or API key
// {
// throw new Exception('Too many requests', 429);
// }
/*
* Background Jobs

4
app/http.php
View file

@ -78,11 +78,11 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
$register->set('db', function () use (&$db) {
App::setResource('db', function () use (&$db) {
return $db;
});
$register->set('cache', function () use (&$redis) {
App::setResource('cache', function () use (&$redis) {
return $redis;
});

48
app/init.php
View file

@ -24,8 +24,6 @@ use Appwrite\Database\Database;
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
use Appwrite\Database\Adapter\Redis as RedisAdapter;
use Appwrite\Database\Document;
use Appwrite\Database\Pool\PDOPool;
use Appwrite\Database\Pool\RedisPool;
use Appwrite\Database\Validator\Authorization;
use Appwrite\Event\Event;
use Appwrite\Event\Realtime;
@ -37,6 +35,10 @@ use Utopia\Locale\Locale;
use Utopia\Registry\Registry;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOConfig;
use Swoole\Database\PDOPool;
use Swoole\Database\RedisConfig;
use Swoole\Database\RedisPool;
const APP_NAME = 'Appwrite';
const APP_DOMAIN = 'appwrite.io';
@ -154,10 +156,21 @@ Database::addFilter('encrypt',
*/
$register->set('dbPool', function () { // Register DB connection
$dbHost = App::getEnv('_APP_DB_HOST', '');
$dbPort = App::getEnv('_APP_DB_PORT', '');
$dbUser = App::getEnv('_APP_DB_USER', '');
$dbPass = App::getEnv('_APP_DB_PASS', '');
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
$pool = new PDOPool(10, $dbHost, $dbScheme, $dbUser, $dbPass);
$pool = new PDOPool((new PDOConfig())
->withHost($dbHost)
->withPort($dbPort)
// ->withUnixSocket('/tmp/mysql.sock')
->withDbName($dbScheme)
->withCharset('utf8mb4')
->withUsername($dbUser)
->withPassword($dbPass)
);
return $pool;
});
@ -166,16 +179,19 @@ $register->set('redisPool', function () {
$redisPort = App::getEnv('_APP_REDIS_PORT', '');
$redisUser = App::getEnv('_APP_REDIS_USER', '');
$redisPass = App::getEnv('_APP_REDIS_PASS', '');
$redisAuth = [];
$redisAuth = '';
if ($redisUser) {
$redisAuth[] = $redisUser;
}
if ($redisPass) {
$redisAuth[] = $redisPass;
if ($redisUser && $redisPass) {
$redisAuth = $redisUser.':'.$redisPass;
}
$pool = new RedisPool(10, $redisHost, $redisPort, $redisAuth);
$pool = new RedisPool((new RedisConfig)
->withHost($redisHost)
->withPort($redisPort)
->withAuth($redisAuth)
->withDbIndex(0)
->withTimeout(1)
);
return $pool;
});
@ -485,23 +501,23 @@ App::setResource('console', function($consoleDB) {
return $consoleDB->getDocument('console');
}, ['consoleDB']);
App::setResource('consoleDB', function($register) {
App::setResource('consoleDB', function($db, $cache) {
$consoleDB = new Database();
$consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
$consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
$consoleDB->setNamespace('app_console'); // Should be replaced with param if we want to have parent projects
$consoleDB->setMocks(Config::getParam('collections', []));
return $consoleDB;
}, ['register']);
}, ['db', 'cache']);
App::setResource('projectDB', function($register, $project) {
App::setResource('projectDB', function($db, $cache, $project) {
$projectDB = new Database();
$projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
$projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
$projectDB->setNamespace('app_'.$project->getId());
$projectDB->setMocks(Config::getParam('collections', []));
return $projectDB;
}, ['register', 'project']);
}, ['db', 'cache', 'project']);
App::setResource('mode', function($request) {
/** @var Utopia\Swoole\Request $request */

1
composer.lock generated
View file

@ -4819,6 +4819,7 @@
"type": "github"
}
],
"abandoned": true,
"time": "2020-09-28T06:45:17+00:00"
},
{

44
src/Appwrite/Database/Adapter/MySQL.php
View file

@ -2,13 +2,12 @@
namespace Appwrite\Database\Adapter;
use Utopia\Registry\Registry;
use Appwrite\Database\Adapter;
use Appwrite\Database\Exception\Duplicate;
use Appwrite\Database\Validator\Authorization;
use Exception;
use PDO;
use Redis as Client;
use Redis;
class MySQL extends Adapter
{
@ -23,11 +22,6 @@ class MySQL extends Adapter
const OPTIONS_LIMIT_ATTRIBUTES = 1000;
/**
* @var Registry
*/
protected $register;
/**
* Last modified.
*
@ -42,16 +36,28 @@ class MySQL extends Adapter
*/
protected $debug = [];
/**
* @var PDO
*/
protected $pdo;
/**
* @var Redis
*/
protected $redis;
/**
* Constructor.
*
* Set connection and settings
*
* @param Registry $register
* @param PDO $pdo
* @param Redis $redis
*/
public function __construct(Registry $register)
public function __construct($pdo, Redis $redis)
{
$this->register = $register;
$this->pdo = $pdo;
$this->redis = $redis;
}
/**
@ -87,8 +93,8 @@ class MySQL extends Adapter
ORDER BY `order`
');
$st->bindParam(':documentUid', $document['uid'], PDO::PARAM_STR);
$st->bindParam(':documentRevision', $document['revision'], PDO::PARAM_STR);
$st->bindParam(':documentUid', $document['uid'], PDO::PARAM_STR, 32);
$st->bindParam(':documentRevision', $document['revision'], PDO::PARAM_STR, 32);
$st->execute();
@ -116,8 +122,8 @@ class MySQL extends Adapter
ORDER BY `order`
');
$st->bindParam(':start', $document['uid'], PDO::PARAM_STR);
$st->bindParam(':revision', $document['revision'], PDO::PARAM_STR);
$st->bindParam(':start', $document['uid'], PDO::PARAM_STR, 32);
$st->bindParam(':revision', $document['revision'], PDO::PARAM_STR, 32);
$st->execute();
@ -933,18 +939,18 @@ class MySQL extends Adapter
*
* @throws Exception
*/
protected function getPDO(): PDO
protected function getPDO()
{
return $this->register->get('db');
return $this->pdo;
}
/**
* @throws Exception
*
* @return Client
* @return Redis
*/
protected function getRedis(): Client
protected function getRedis(): Redis
{
return $this->register->get('cache');
return $this->redis;
}
}

13
src/Appwrite/Database/Adapter/Redis.php
View file

@ -2,7 +2,6 @@
namespace Appwrite\Database\Adapter;
use Utopia\Registry\Registry;
use Appwrite\Database\Adapter;
use Exception;
use Redis as Client;
@ -10,9 +9,9 @@ use Redis as Client;
class Redis extends Adapter
{
/**
* @var Registry
* @var Client
*/
protected $register;
protected $redis;
/**
* @var Adapter
@ -23,11 +22,11 @@ class Redis extends Adapter
* Redis constructor.
*
* @param Adapter $adapter
* @param Registry $register
* @param Client $redis
*/
public function __construct(Adapter $adapter, Registry $register)
public function __construct(Adapter $adapter, Client $redis)
{
$this->register = $register;
$this->redis = $redis;
$this->adapter = $adapter;
}
@ -261,7 +260,7 @@ class Redis extends Adapter
*/
protected function getRedis(): Client
{
return $this->register->get('cache');
return $this->redis;
}
/**

48
src/Appwrite/Database/Pool/PDOPool.php
View file

@ -1,48 +0,0 @@
<?php
namespace Appwrite\Database\Pool;
use Appwrite\Database\Pool;
use Appwrite\Extend\PDO;
use Swoole\Coroutine\Channel;
class PDOPool extends Pool
{
public function __construct(int $size, string $host = 'localhost', string $schema = 'appwrite', string $user = '', string $pass = '', string $charset = 'utf8mb4')
{
$this->pool = new Channel($this->size = $size);
for ($i = 0; $i < $this->size; $i++) {
$pdo = new PDO(
"mysql:" .
"host={$host};" .
"dbname={$schema};" .
"charset={$charset}",
$user,
$pass,
[
PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4',
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true
]
);
$this->pool->push($pdo);
}
}
public function put(PDO $pdo)
{
$this->pool->push($pdo);
}
public function get(): PDO
{
if ($this->available && !$this->pool->isEmpty()) {
return $this->pool->pop();
}
sleep(0.01);
return $this->get();
}
}

40
src/Appwrite/Database/Pool/RedisPool.php
View file

@ -1,40 +0,0 @@
<?php
namespace Appwrite\Database\Pool;
use Appwrite\Database\Pool;
use Redis;
use Swoole\Coroutine\Channel;
class RedisPool extends Pool
{
public function __construct(int $size, string $host, int $port, array $auth = [])
{
$this->pool = new Channel($this->size = $size);
for ($i = 0; $i < $this->size; $i++) {
$redis = new Redis();
$redis->pconnect($host, $port);
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
if ($auth) {
$redis->auth($auth);
}
$this->pool->push($redis);
}
}
public function put(Redis $redis)
{
$this->pool->push($redis);
}
public function get(): Redis
{
if ($this->available && !$this->pool->isEmpty()) {
return $this->pool->pop();
}
sleep(0.1);
return $this->get();
}
}

40
src/Appwrite/Realtime/Server.php
View file

@ -176,16 +176,16 @@ class Server
$db = $this->register->get('dbPool')->get();
$redis = $this->register->get('redisPool')->get();
$this->register->set('db', function () use (&$db) {
Console::info("Connection open (user: {$connection}, worker: {$server->getWorkerId()})");
App::setResource('db', function () use (&$db) {
return $db;
});
$this->register->set('cache', function () use (&$redis) {
App::setResource('cache', function () use (&$redis) {
return $redis;
});
Console::info("Connection open (user: {$connection}, worker: {$server->getWorkerId()})");
App::setResource('request', function () use ($request) {
return $request;
});
@ -211,24 +211,24 @@ class Server
throw new Exception('Missing or unknown project ID', 1008);
}
/*
* Abuse Check
*
* Abuse limits are connecting 128 times per minute and ip address.
*/
$timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, function () use ($db) {
return $db;
});
$timeLimit
->setNamespace('app_' . $project->getId())
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getURI());
// /*
// * Abuse Check
// *
// * Abuse limits are connecting 128 times per minute and ip address.
// */
// $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, function () use ($db) {
// return $db;
// });
// $timeLimit
// ->setNamespace('app_' . $project->getId())
// ->setParam('{ip}', $request->getIP())
// ->setParam('{url}', $request->getURI());
$abuse = new Abuse($timeLimit);
// $abuse = new Abuse($timeLimit);
if ($abuse->check() && App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') {
throw new Exception('Too many requests', 1013);
}
// if ($abuse->check() && App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') {
// throw new Exception('Too many requests', 1013);
// }
/*
* Validate Client Domain - Check to avoid CSRF attack.

2
tests/benchmarks/http.js
View file

@ -26,7 +26,7 @@ export default function () {
'X-Appwrite-Project': '60479fe35d95d'
}}
const resDb = http.get('http://localhost:9501/v1/health/db', config);
const resDb = http.get('http://localhost:9501/', config);
check(resDb, {
'status is 200': (r) => r.status === 200,

2
tests/benchmarks/ws.js
View file

@ -21,7 +21,7 @@ export default function () {
// const url = new URL('wss://appwrite-realtime.monitor-api.com/v1/realtime');
// url.searchParams.append('project', '604249e6b1a9f');
const url = new URL('ws://localhost/v1/realtime');
url.searchParams.append('project', '60476312f335c');
url.searchParams.append('project', 'console');
url.searchParams.append('channels[]', 'files');
const res = ws.connect(url.toString(), function (socket) {