CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-09-29 17:01:37 +13:00
Code Issues Projects Releases Wiki Activity

Merge branch 'feat-db-pools' into refactor-functions-worker

Browse source
This commit is contained in:
Matej Baco 2022-11-16 12:29:42 +01:00
commit 83639ea6f8
8 changed files with 51 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.env
View file

@ -23,7 +23,7 @@ _APP_DB_SCHEMA=appwrite
_APP_DB_USER=user
_APP_DB_PASS=password
_APP_DB_ROOT_PASS=rootsecretpassword
_APP_DB_MAX_CONNECTIONS=251
_APP_CONNECTIONS_MAX=251
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379

2
CHANGES.md
View file

@ -1,7 +1,7 @@
# TBD
- Replace Appwrite executor with OpenRuntimes Executor [#4650](https://github.com/appwrite/appwrite/pull/4650)
- Add `_APP_DB_MAX_CONNECTIONS` env var [#4673](https://github.com/appwrite/appwrite/pull/4673)
- Add `_APP_CONNECTIONS_MAX` env var [#4673](https://github.com/appwrite/appwrite/pull/4673)
- Update Traefik 2.7 -> 2.9 [#4673](https://github.com/appwrite/appwrite/pull/4673)
- Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673)

38
app/config/variables.php
View file

@ -307,7 +307,7 @@ return [
'filter' => 'password'
],
[
'name' => '_APP_DB_MAX_CONNECTIONS',
'name' => '_APP_CONNECTIONS_MAX',
'description' => 'MariaDB server maximum connections.',
'introduction' => 'TBD',
'default' => 251,
@ -315,24 +315,24 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_CONNECTIONS_DB_PROJECT',
'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
'introduction' => 'TBD',
'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
'required' => true,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_CONNECTIONS_DB_CONSOLE',
'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
'introduction' => 'TBD',
'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
'required' => true,
'question' => '',
'filter' => ''
]
// [
// 'name' => '_APP_CONNECTIONS_DB_PROJECT',
// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
// 'introduction' => 'TBD',
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
// 'required' => true,
// 'question' => '',
// 'filter' => ''
// ],
// [
// 'name' => '_APP_CONNECTIONS_DB_CONSOLE',
// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
// 'introduction' => 'TBD',
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
// 'required' => true,
// 'question' => '',
// 'filter' => ''
// ]
],
],
[

37
app/init.php
View file

@ -99,7 +99,6 @@ const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
const APP_CACHE_BUSTER = 501;
const APP_VERSION_STABLE = '1.0.3';
const APP_DEFAULT_POOL_SIZE = 64;
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@ -499,8 +498,7 @@ $register->set('logger', function () {
$adapter = new $classname($providerConfig);
return new Logger($adapter);
});
$register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
$register->set('pools', function () {
$group = new Group();
$fallbackForDB = AppwriteURL::unparse([
@ -551,6 +549,17 @@ $register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
],
];
$instances = 2; // REST, Realtime
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
$maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 251);
$instanceConnections = $maxConnections / $instances;
if ($workerCount > $instanceConnections) {
throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);
}
$poolSize = (int)($instanceConnections / $workerCount);
foreach ($connections as $key => $connection) {
$type = $connection['type'] ?? '';
$dsns = $connection['dsns'] ?? '';
@ -625,7 +634,7 @@ $register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
break;
}
$pool = new Pool($name, $size, function () use ($type, $resource, $dsn) {
$pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) {
// Get Adapter
$adapter = null;
@ -1132,26 +1141,6 @@ function getDevice($root): Device
}
}
/**
* Get database connection pool size for worker processes.
*
* @return int
* @throws \Exception
*/
function getWorkerPoolSize(): int
{
$reservedConnections = APP_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
$maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 251);
$workerConnections = $maxConnections - $reservedConnections;
if ($workerCount > $workerConnections) {
throw new \Exception('Worker pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);
}
return (int)($workerConnections / $workerCount);
}
App::setResource('mode', function ($request) {
/** @var Appwrite\Utopia\Request $request */

4
app/realtime.php
View file

@ -37,7 +37,7 @@ function getConsoleDB(): Database
global $register;
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools', args: [getWorkerPoolSize()]);
$pools = $register->get('pools');
$dbAdapter = $pools
->get('console')
@ -57,7 +57,7 @@ function getProjectDB(Document $project): Database
global $register;
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools', args: [getWorkerPoolSize()]);
$pools = $register->get('pools');
if ($project->isEmpty() || $project->getId() === 'console') {
return getConsoleDB();

6
app/views/install/compose.phtml
View file

@ -102,7 +102,7 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_DB_MAX_CONNECTIONS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -197,7 +197,7 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_DB_MAX_CONNECTIONS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -622,7 +622,7 @@ services:
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
- MYSQL_USER=${_APP_DB_USER}
- MYSQL_PASSWORD=${_APP_DB_PASS}
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}'
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}'
redis:
image: redis:7.0.4-alpine

88
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "f2faa670abaa356f9b14e4f1c3542b33",
"content-hash": "bf3e2ed6ee8e49ab74af97b368b89a63",
"packages": [
{
"name": "adhocore/jwt",
@ -2357,79 +2357,18 @@
},
"time": "2020-10-24T07:04:59+00:00"
},
{
"name": "utopia-php/queue",
"version": "0.4.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/queue.git",
"reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e",
"reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e",
"shasum": ""
},
"require": {
"php": ">=8.0",
"utopia-php/cli": "0.14.*",
"utopia-php/framework": "0.*.*"
},
"require-dev": {
"laravel/pint": "^0.2.3",
"phpstan/phpstan": "^1.8",
"phpunit/phpunit": "^9.5.5",
"swoole/ide-helper": "4.8.8",
"workerman/workerman": "^4.0"
},
"suggest": {
"ext-swoole": "Needed to support Swoole.",
"workerman/workerman": "Needed to support Workerman."
},
"type": "library",
"autoload": {
"psr-4": {
"Utopia\\Queue\\": "src/Queue"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Torsten Dittmann",
"email": "torsten@appwrite.io"
}
],
"description": "A powerful task queue.",
"keywords": [
"Tasks",
"framework",
"php",
"queue",
"upf",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/queue/issues",
"source": "https://github.com/utopia-php/queue/tree/0.4.1"
},
"time": "2022-11-15T16:56:37+00:00"
},
{
"name": "utopia-php/registry",
"version": "dev-feat-allow-params",
"version": "0.5.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/registry.git",
"reference": "6c571f8f4127094b3af8909d1b595fd6b937255d"
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/registry/zipball/6c571f8f4127094b3af8909d1b595fd6b937255d",
"reference": "6c571f8f4127094b3af8909d1b595fd6b937255d",
"url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d",
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d",
"shasum": ""
},
"require": {
@ -2466,9 +2405,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/registry/issues",
"source": "https://github.com/utopia-php/registry/tree/feat-allow-params"
"source": "https://github.com/utopia-php/registry/tree/0.5.0"
},
"time": "2022-11-09T02:23:35+00:00"
"time": "2021-03-10T10:45:22+00:00"
},
{
"name": "utopia-php/storage",
@ -5276,18 +5215,9 @@
"time": "2022-09-28T08:42:51+00:00"
}
],
"aliases": [
{
"package": "utopia-php/registry",
"version": "dev-feat-allow-params",
"alias": "0.5.0",
"alias_normalized": "0.5.0.0"
}
],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": {
"utopia-php/registry": 20
},
"stability-flags": [],
"prefer-stable": false,
"prefer-lowest": false,
"platform": {

6
docker-compose.yml
View file

@ -123,7 +123,7 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_DB_MAX_CONNECTIONS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -229,7 +229,7 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_DB_MAX_CONNECTIONS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -753,7 +753,7 @@ services:
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
- MYSQL_USER=${_APP_DB_USER}
- MYSQL_PASSWORD=${_APP_DB_PASS}
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}'
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}'
# smtp:
# image: appwrite/smtp:1.2.0