1
0
Fork 0
mirror of synced 2024-09-29 08:51:28 +13:00

feat: Moving to coroutine dedicated http server

This commit is contained in:
Binyamin Yawitz 2024-07-03 16:24:04 -04:00
parent a4f675c632
commit 92668fda0d
No known key found for this signature in database
4 changed files with 257 additions and 249 deletions

View file

@ -19,8 +19,10 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Http\Adapter\Swoole\Server;
use Utopia\DI\Container;
use Utopia\Http\Adapter\SwooleCoroutine\Server;
use Utopia\Http\Http;
use Utopia\Registry\Registry;
use Utopia\System\System;
global $global, $container;
@ -28,219 +30,236 @@ global $global, $container;
$payloadSize = 12 * (1024 * 1024); // 12MB - adding slight buffer for headers and other data that might be sent with the payload - update later with valid testing
$workerNumber = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6));
$server = new Server('0.0.0.0', '80', [
'open_http2_protocol' => true,
'http_compression' => true,
'http_compression_level' => 6,
'package_max_length' => $payloadSize,
'buffer_output_size' => $payloadSize,
// Server
// 'log_level' => 0,
'dispatch_mode' => 1,
'worker_num' => $workerNumber,
'reactor_num' => swoole_cpu_num() * 2,
'open_cpu_affinity' => true,
// Coroutine
'enable_coroutine' => true,
'max_coroutine' => 10000,
'send_yield' => true
]);
$pool = new Swoole\Process\Pool($workerNumber, 0, 0, true);
$http = new Http($server, $container, 'UTC');
function startCoroutineServer(float|int $payloadSize, float|int $workerNumber, Registry $global, Container $container, $workerId)
{
$server = new Server('0.0.0.0', '80', [
'open_http2_protocol' => true,
'http_compression' => true,
'http_compression_level' => 6,
'package_max_length' => $payloadSize,
'buffer_output_size' => $payloadSize,
// Server
// 'log_level' => 0,
'dispatch_mode' => 1,
'worker_num' => $workerNumber,
'reactor_num' => swoole_cpu_num() * 2,
'open_cpu_affinity' => true,
// Coroutine
'enable_coroutine' => true,
'max_coroutine' => 10000,
'send_yield' => true
]);
$http->loadFiles(__DIR__ . '/../console');
$http->setRequestClass(Request::class);
$http->setResponseClass(Response::class);
$http = new Http($server, $container, 'UTC');
global $global, $container;
http::onStart()
->inject('authorization')
->inject('cache')
->inject('pools')
->inject('connections')
->action(function (Authorization $authorization, Cache $cache, array $pools, Connections $connections) {
try {
// wait for database to be ready
$attempts = 0;
$max = 15;
$sleep = 2;
do {
try {
$attempts++;
$pool = $pools['pools-console-main']['pool'];
$dsn = $pools['pools-console-main']['dsn'];
$connection = $pool->get();
$connections->add($connection, $pool);
$adapter = match ($dsn->getScheme()) {
'mariadb' => new MariaDB($connection),
'mysql' => new MySQL($connection),
default => null
};
$adapter->setDatabase($dsn->getPath());
$dbForConsole = new Database($adapter, $cache);
$dbForConsole->setAuthorization($authorization);
$dbForConsole
->setNamespace('_console')
->setMetadata('host', \gethostname())
->setMetadata('project', 'console')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
$dbForConsole->ping();
break; // leave the do-while if successful
} catch (\Throwable $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
}
sleep($sleep);
}
} while ($attempts < $max);
Console::success('[Setup] - Server database init started...');
$http->loadFiles(__DIR__ . '/../console');
$http->setRequestClass(Request::class);
$http->setResponseClass(Response::class);
Http::onStart()
->inject('authorization')
->inject('cache')
->inject('pools')
->inject('connections')
->action(function (Authorization $authorization, Cache $cache, array $pools, Connections $connections) use ($workerId) {
if($workerId !== 0) {
return;
}
try {
Console::success('[Setup] - Creating database: appwrite...');
$dbForConsole->create();
// wait for database to be ready
$attempts = 0;
$max = 15;
$sleep = 2;
do {
try {
$attempts++;
$pool = $pools['pools-console-main']['pool'];
$dsn = $pools['pools-console-main']['dsn'];
$connection = $pool->get();
$connections->add($connection, $pool);
$adapter = match ($dsn->getScheme()) {
'mariadb' => new MariaDB($connection),
'mysql' => new MySQL($connection),
default => null
};
$adapter->setDatabase($dsn->getPath());
$dbForConsole = new Database($adapter, $cache);
$dbForConsole->setAuthorization($authorization);
$dbForConsole
->setNamespace('_console')
->setMetadata('host', \gethostname())
->setMetadata('project', 'console')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
$dbForConsole->ping();
break; // leave the do-while if successful
} catch (\Throwable $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
}
sleep($sleep);
}
} while ($attempts < $max);
Console::success('[Setup] - Server database init started...');
try {
Console::success('[Setup] - Creating database: appwrite...');
$dbForConsole->create();
} catch (\Throwable $e) {
Console::success('[Setup] - Skip: metadata table already exists');
return true;
}
if ($dbForConsole->getCollection(Audit::COLLECTION)->isEmpty()) {
$audit = new Audit($dbForConsole);
$audit->setup();
}
if ($dbForConsole->getCollection(TimeLimit::COLLECTION)->isEmpty()) {
$abuse = new TimeLimit("", 0, 1, $dbForConsole);
$abuse->setup();
}
/** @var array $collections */
$collections = Config::getParam('collections', []);
$consoleCollections = $collections['console'];
foreach ($consoleCollections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
if (!$dbForConsole->getCollection($key)->isEmpty()) {
continue;
}
Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...');
$attributes = [];
$indexes = [];
foreach ($collection['attributes'] as $attribute) {
$attributes[] = new Document([
'$id' => ID::custom($attribute['$id']),
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
'default' => $attribute['default'] ?? null,
'format' => $attribute['format'] ?? ''
]);
}
foreach ($collection['indexes'] as $index) {
$indexes[] = new Document([
'$id' => ID::custom($index['$id']),
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
$dbForConsole->createCollection($key, $attributes, $indexes);
}
if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists($dbForConsole->getDatabase(), 'bucket_1')) {
Console::success('[Setup] - Creating default bucket...');
$dbForConsole->createDocument('buckets', new Document([
'$id' => ID::custom('default'),
'$collection' => ID::custom('buckets'),
'name' => 'Default',
'maximumFileSize' => (int)System::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB
'allowedFileExtensions' => [],
'enabled' => true,
'compression' => 'gzip',
'encryption' => true,
'antivirus' => true,
'fileSecurity' => true,
'$permissions' => [
Permission::create(Role::any()),
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'search' => 'buckets Default',
]));
$bucket = $dbForConsole->getDocument('buckets', 'default');
Console::success('[Setup] - Creating files collection for default bucket...');
$files = $collections['buckets']['files'] ?? [];
if (empty($files)) {
throw new Exception('Files collection is not configured.');
}
$attributes = [];
$indexes = [];
foreach ($files['attributes'] as $attribute) {
$attributes[] = new Document([
'$id' => ID::custom($attribute['$id']),
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
'default' => $attribute['default'] ?? null,
'format' => $attribute['format'] ?? ''
]);
}
foreach ($files['indexes'] as $index) {
$indexes[] = new Document([
'$id' => ID::custom($index['$id']),
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
$dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
}
$connections->reclaim();
Console::success('[Setup] - Server database init completed...');
Console::success('Server started successfully');
} catch (\Throwable $e) {
Console::success('[Setup] - Skip: metadata table already exists');
return true;
Console::warning('Database not ready: ' . $e->getMessage());
exit(1);
}
});
if ($dbForConsole->getCollection(Audit::COLLECTION)->isEmpty()) {
$audit = new Audit($dbForConsole);
$audit->setup();
}
Http::init()
->inject('authorization')
->action(function (Authorization $authorization) {
$authorization->cleanRoles();
$authorization->addRole(Role::any()->toString());
});
if ($dbForConsole->getCollection(TimeLimit::COLLECTION)->isEmpty()) {
$abuse = new TimeLimit("", 0, 1, $dbForConsole);
$abuse->setup();
}
$http->start();
}
/** @var array $collections */
$collections = Config::getParam('collections', []);
$consoleCollections = $collections['console'];
foreach ($consoleCollections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
if (!$dbForConsole->getCollection($key)->isEmpty()) {
continue;
}
$pool->on("WorkerStart", function ($pool, $workerId) use ($payloadSize, $workerNumber, $global, $container) {
Console::success("Worker " . $workerId . " started");
startCoroutineServer($payloadSize, $workerNumber, $global, $container, $workerId);
});
Console::success('[Setup] - Creating collection: ' . $collection['$id'] . '...');
$pool->on("WorkerStop", function ($pool, $workerId) {
Console::success("Worker " . $workerId . " stopped");
});
$attributes = [];
$indexes = [];
foreach ($collection['attributes'] as $attribute) {
$attributes[] = new Document([
'$id' => ID::custom($attribute['$id']),
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
'default' => $attribute['default'] ?? null,
'format' => $attribute['format'] ?? ''
]);
}
foreach ($collection['indexes'] as $index) {
$indexes[] = new Document([
'$id' => ID::custom($index['$id']),
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
$dbForConsole->createCollection($key, $attributes, $indexes);
}
if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists($dbForConsole->getDatabase(), 'bucket_1')) {
Console::success('[Setup] - Creating default bucket...');
$dbForConsole->createDocument('buckets', new Document([
'$id' => ID::custom('default'),
'$collection' => ID::custom('buckets'),
'name' => 'Default',
'maximumFileSize' => (int) System::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB
'allowedFileExtensions' => [],
'enabled' => true,
'compression' => 'gzip',
'encryption' => true,
'antivirus' => true,
'fileSecurity' => true,
'$permissions' => [
Permission::create(Role::any()),
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'search' => 'buckets Default',
]));
$bucket = $dbForConsole->getDocument('buckets', 'default');
Console::success('[Setup] - Creating files collection for default bucket...');
$files = $collections['buckets']['files'] ?? [];
if (empty($files)) {
throw new Exception('Files collection is not configured.');
}
$attributes = [];
$indexes = [];
foreach ($files['attributes'] as $attribute) {
$attributes[] = new Document([
'$id' => ID::custom($attribute['$id']),
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
'default' => $attribute['default'] ?? null,
'format' => $attribute['format'] ?? ''
]);
}
foreach ($files['indexes'] as $index) {
$indexes[] = new Document([
'$id' => ID::custom($index['$id']),
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
$dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
}
$connections->reclaim();
Console::success('[Setup] - Server database init completed...');
Console::success('Server started successfully');
} catch (\Throwable $e) {
Console::warning('Database not ready: ' . $e->getMessage());
exit(1);
}
});
Http::init()
->inject('authorization')
->action(function (Authorization $authorization) {
$authorization->cleanRoles();
$authorization->addRole(Role::any()->toString());
});
$http->start();
$pool->start();

View file

@ -1153,13 +1153,12 @@ $schemaVariable
$schema
->setName('schema')
->inject('http')
->inject('context')
->inject('request')
->inject('response')
->inject('dbForProject')
->inject('authorization')
->inject('schemaVariable')
->setCallback(function (Http $http, Container $context, Request $request, Response $response, Database $dbForProject, Authorization $authorization, $schemaVariable) {
->setCallback(function (Http $http, Request $request, Response $response, Database $dbForProject, Authorization $authorization, $schemaVariable) {
$complexity = function (int $complexity, array $args) {
$queries = Query::parseQueries($args['queries'] ?? []);
$query = Query::getByType($queries, [Query::TYPE_LIMIT])[0] ?? null;
@ -1237,7 +1236,7 @@ $schema
$http,
$request,
$response,
$context,
$http->getContainer(),
$complexity,
$attributes,
$urls,

69
composer.lock generated
View file

@ -1572,16 +1572,16 @@
},
{
"name": "utopia-php/cache",
"version": "0.10.1",
"version": "0.10.2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/cache.git",
"reference": "87ee4fc91e50d4ddfef650aa999ea12be3a99583"
"reference": "b22c6eb6d308de246b023efd0fc9758aee8b8247"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/cache/zipball/87ee4fc91e50d4ddfef650aa999ea12be3a99583",
"reference": "87ee4fc91e50d4ddfef650aa999ea12be3a99583",
"url": "https://api.github.com/repos/utopia-php/cache/zipball/b22c6eb6d308de246b023efd0fc9758aee8b8247",
"reference": "b22c6eb6d308de246b023efd0fc9758aee8b8247",
"shasum": ""
},
"require": {
@ -1616,9 +1616,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/cache/issues",
"source": "https://github.com/utopia-php/cache/tree/0.10.1"
"source": "https://github.com/utopia-php/cache/tree/0.10.2"
},
"time": "2024-06-18T13:20:25+00:00"
"time": "2024-06-25T20:36:35+00:00"
},
{
"name": "utopia-php/cli",
@ -1783,7 +1783,7 @@
"version": "dev-feat-framework-v2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/di.git",
"url": "git@github.com:utopia-php/di.git",
"reference": "1fb7da5ead16de5d795ef10c1e22e39726eb6943"
},
"dist": {
@ -1833,10 +1833,6 @@
"php",
"upf"
],
"support": {
"source": "https://github.com/utopia-php/di/tree/feat-framework-v2",
"issues": "https://github.com/utopia-php/di/issues"
},
"time": "2024-06-11T16:03:00+00:00"
},
{
@ -1991,12 +1987,12 @@
"source": {
"type": "git",
"url": "https://github.com/utopia-php/http.git",
"reference": "bf2474554f78d870c74aaaa1dfb4f54795ae9497"
"reference": "f4ce8a42a5613f40992e6fe84ff2e23584465adb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/http/zipball/bf2474554f78d870c74aaaa1dfb4f54795ae9497",
"reference": "bf2474554f78d870c74aaaa1dfb4f54795ae9497",
"url": "https://api.github.com/repos/utopia-php/http/zipball/f4ce8a42a5613f40992e6fe84ff2e23584465adb",
"reference": "f4ce8a42a5613f40992e6fe84ff2e23584465adb",
"shasum": ""
},
"require": {
@ -2033,7 +2029,7 @@
"issues": "https://github.com/utopia-php/http/issues",
"source": "https://github.com/utopia-php/http/tree/feat-di-upgrade"
},
"time": "2024-06-11T16:38:45+00:00"
"time": "2024-07-03T20:21:21+00:00"
},
{
"name": "utopia-php/image",
@ -2670,7 +2666,7 @@
"version": "dev-dev",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/servers.git",
"url": "git@github.com:utopia-php/servers.git",
"reference": "4565c1c111f6da6b18bc0f00f350377a1e691e48"
},
"dist": {
@ -2730,10 +2726,6 @@
"upf",
"utopia"
],
"support": {
"source": "https://github.com/utopia-php/servers/tree/dev",
"issues": "https://github.com/utopia-php/servers/issues"
},
"time": "2024-06-07T18:49:59+00:00"
},
{
@ -3124,16 +3116,16 @@
"packages-dev": [
{
"name": "appwrite/sdk-generator",
"version": "0.38.7",
"version": "0.38.8",
"source": {
"type": "git",
"url": "https://github.com/appwrite/sdk-generator.git",
"reference": "0a66c1149ef05ed9f45ce1c897c4a0ce9ee5e95a"
"reference": "6367c57ddbcf7b88cacb900c4fe7ef3f28bf38ef"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0a66c1149ef05ed9f45ce1c897c4a0ce9ee5e95a",
"reference": "0a66c1149ef05ed9f45ce1c897c4a0ce9ee5e95a",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6367c57ddbcf7b88cacb900c4fe7ef3f28bf38ef",
"reference": "6367c57ddbcf7b88cacb900c4fe7ef3f28bf38ef",
"shasum": ""
},
"require": {
@ -3169,9 +3161,9 @@
"description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
"support": {
"issues": "https://github.com/appwrite/sdk-generator/issues",
"source": "https://github.com/appwrite/sdk-generator/tree/0.38.7"
"source": "https://github.com/appwrite/sdk-generator/tree/0.38.8"
},
"time": "2024-06-10T00:23:02+00:00"
"time": "2024-06-17T00:42:27+00:00"
},
{
"name": "doctrine/deprecations",
@ -3544,16 +3536,16 @@
},
{
"name": "nikic/php-parser",
"version": "dev-master",
"version": "v5.1.0",
"source": {
"type": "git",
"url": "https://github.com/nikic/PHP-Parser.git",
"reference": "3ef0811e45ba7e91fb0f066af5af7d52c3b24469"
"reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/3ef0811e45ba7e91fb0f066af5af7d52c3b24469",
"reference": "3ef0811e45ba7e91fb0f066af5af7d52c3b24469",
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/683130c2ff8c2739f4822ff7ac5c873ec529abd1",
"reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1",
"shasum": ""
},
"require": {
@ -3566,7 +3558,6 @@
"ircmaxell/php-yacc": "^0.0.7",
"phpunit/phpunit": "^9.0"
},
"default-branch": true,
"bin": [
"bin/php-parse"
],
@ -3597,9 +3588,9 @@
],
"support": {
"issues": "https://github.com/nikic/PHP-Parser/issues",
"source": "https://github.com/nikic/PHP-Parser/tree/master"
"source": "https://github.com/nikic/PHP-Parser/tree/v5.1.0"
},
"time": "2024-06-12T18:31:58+00:00"
"time": "2024-07-01T20:03:41+00:00"
},
{
"name": "phar-io/manifest",
@ -4079,12 +4070,12 @@
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/php-code-coverage.git",
"reference": "3352293d9e91513d5508c415835014881b420218"
"reference": "39d628812d8d83344a6c1b07799e3700d830d355"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/3352293d9e91513d5508c415835014881b420218",
"reference": "3352293d9e91513d5508c415835014881b420218",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/39d628812d8d83344a6c1b07799e3700d830d355",
"reference": "39d628812d8d83344a6c1b07799e3700d830d355",
"shasum": ""
},
"require": {
@ -4103,7 +4094,7 @@
"theseer/tokenizer": "^1.2.0"
},
"require-dev": {
"phpunit/phpunit": "^9.3"
"phpunit/phpunit": "^9.6"
},
"suggest": {
"ext-pcov": "PHP extension that provides line coverage",
@ -4112,7 +4103,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "9.2-dev"
"dev-main": "9.2-dev"
}
},
"autoload": {
@ -4149,7 +4140,7 @@
"type": "github"
}
],
"time": "2024-03-22T05:16:32+00:00"
"time": "2024-06-29T07:23:05+00:00"
},
{
"name": "phpunit/php-file-iterator",

View file

@ -41,8 +41,7 @@ class Specs extends Action
->param('version', 'latest', new Text(16), 'Spec version', true)
->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true)
->inject('register')
->inject('context')
->callback(fn (string $version, string $mode, Registry $register, Container $context) => $this->action($version, $mode, $register, $context));
->callback(fn (string $version, string $mode, Registry $register) => $this->action($version, $mode, $register));
}
public function action(string $version, string $mode, Registry $register, Container $container): void