CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-09-28 07:21:35 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'feat-db-pools' into refactor-scheduler

Browse source
This commit is contained in:
Matej Baco 2022-11-16 14:13:05 +01:00
commit 8b000aff40
9 changed files with 52 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.env
View file

@ -23,7 +23,7 @@ _APP_DB_SCHEMA=appwrite
_APP_DB_USER=user
_APP_DB_PASS=password
_APP_DB_ROOT_PASS=rootsecretpassword
_APP_DB_MAX_CONNECTIONS=251
_APP_CONNECTIONS_MAX=251
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379

2
CHANGES.md
View file

@ -1,7 +1,7 @@
# TBD
- Replace Appwrite executor with OpenRuntimes Executor [#4650](https://github.com/appwrite/appwrite/pull/4650)
- Add `_APP_DB_MAX_CONNECTIONS` env var [#4673](https://github.com/appwrite/appwrite/pull/4673)
- Add `_APP_CONNECTIONS_MAX` env var [#4673](https://github.com/appwrite/appwrite/pull/4673)
- Update Traefik 2.7 -> 2.9 [#4673](https://github.com/appwrite/appwrite/pull/4673)
- Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673)

38
app/config/variables.php
View file

@ -307,7 +307,7 @@ return [
'filter' => 'password'
],
[
'name' => '_APP_DB_MAX_CONNECTIONS',
'name' => '_APP_CONNECTIONS_MAX',
'description' => 'MariaDB server maximum connections.',
'introduction' => 'TBD',
'default' => 251,
@ -315,24 +315,24 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_CONNECTIONS_DB_PROJECT',
'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
'introduction' => 'TBD',
'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
'required' => true,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_CONNECTIONS_DB_CONSOLE',
'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
'introduction' => 'TBD',
'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
'required' => true,
'question' => '',
'filter' => ''
]
// [
// 'name' => '_APP_CONNECTIONS_DB_PROJECT',
// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
// 'introduction' => 'TBD',
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
// 'required' => true,
// 'question' => '',
// 'filter' => ''
// ],
// [
// 'name' => '_APP_CONNECTIONS_DB_CONSOLE',
// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
// 'introduction' => 'TBD',
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
// 'required' => true,
// 'question' => '',
// 'filter' => ''
// ]
],
],
[

36
app/init.php
View file

@ -97,7 +97,6 @@ const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
const APP_CACHE_BUSTER = 501;
const APP_VERSION_STABLE = '1.0.3';
const APP_DEFAULT_POOL_SIZE = 64;
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@ -497,7 +496,7 @@ $register->set('logger', function () {
$adapter = new $classname($providerConfig);
return new Logger($adapter);
});
$register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
$register->set('pools', function () {
$group = new Group();
$fallbackForDB = AppwriteURL::unparse([
@ -548,6 +547,17 @@ $register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
],
];
$instances = 2; // REST, Realtime
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
$maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 251);
$instanceConnections = $maxConnections / $instances;
if ($workerCount > $instanceConnections) {
throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);
}
$poolSize = (int)($instanceConnections / $workerCount);
foreach ($connections as $key => $connection) {
$type = $connection['type'] ?? '';
$dsns = $connection['dsns'] ?? '';
@ -621,7 +631,7 @@ $register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
break;
}
$pool = new Pool($name, $size, function () use ($type, $resource, $dsn) {
$pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) {
// Get Adapter
$adapter = null;
@ -1123,26 +1133,6 @@ function getDevice($root): Device
}
}
/**
* Get database connection pool size for worker processes.
*
* @return int
* @throws \Exception
*/
function getWorkerPoolSize(): int
{
$reservedConnections = APP_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
$maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 251);
$workerConnections = $maxConnections - $reservedConnections;
if ($workerCount > $workerConnections) {
throw new \Exception('Worker pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);
}
return (int)($workerConnections / $workerCount);
}
App::setResource('mode', function ($request) {
/** @var Appwrite\Utopia\Request $request */

4
app/realtime.php
View file

@ -37,7 +37,7 @@ function getConsoleDB(): Database
global $register;
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools', args: [getWorkerPoolSize()]);
$pools = $register->get('pools');
$dbAdapter = $pools
->get('console')
@ -57,7 +57,7 @@ function getProjectDB(Document $project): Database
global $register;
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools', args: [getWorkerPoolSize()]);
$pools = $register->get('pools');
if ($project->isEmpty() || $project->getId() === 'console') {
return getConsoleDB();

6
app/views/install/compose.phtml
View file

@ -102,7 +102,7 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_DB_MAX_CONNECTIONS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -197,7 +197,7 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_DB_MAX_CONNECTIONS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -622,7 +622,7 @@ services:
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
- MYSQL_USER=${_APP_DB_USER}
- MYSQL_PASSWORD=${_APP_DB_PASS}
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}'
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}'
redis:
image: redis:7.0.4-alpine

2
composer.json
View file

@ -59,7 +59,7 @@
"utopia-php/platform": "0.3.*",
"utopia-php/pools": "0.4.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/registry": "dev-feat-allow-params as 0.5.0",
"utopia-php/registry": "0.5.0",
"utopia-php/storage": "0.11.*",
"utopia-php/swoole": "0.5.*",
"utopia-php/websocket": "0.1.0",

27
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "b125039c64ae4cbe0d2a1b57322d0ebe",
"content-hash": "bf3e2ed6ee8e49ab74af97b368b89a63",
"packages": [
{
"name": "adhocore/jwt",
@ -2359,16 +2359,16 @@
},
{
"name": "utopia-php/registry",
"version": "dev-feat-allow-params",
"version": "0.5.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/registry.git",
"reference": "6c571f8f4127094b3af8909d1b595fd6b937255d"
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/registry/zipball/6c571f8f4127094b3af8909d1b595fd6b937255d",
"reference": "6c571f8f4127094b3af8909d1b595fd6b937255d",
"url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d",
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d",
"shasum": ""
},
"require": {
@ -2405,9 +2405,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/registry/issues",
"source": "https://github.com/utopia-php/registry/tree/feat-allow-params"
"source": "https://github.com/utopia-php/registry/tree/0.5.0"
},
"time": "2022-11-09T02:23:35+00:00"
"time": "2021-03-10T10:45:22+00:00"
},
{
"name": "utopia-php/storage",
@ -5215,18 +5215,9 @@
"time": "2022-09-28T08:42:51+00:00"
}
],
"aliases": [
{
"package": "utopia-php/registry",
"version": "dev-feat-allow-params",
"alias": "0.5.0",
"alias_normalized": "0.5.0.0"
}
],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": {
"utopia-php/registry": 20
},
"stability-flags": [],
"prefer-stable": false,
"prefer-lowest": false,
"platform": {

6
docker-compose.yml
View file

@ -123,7 +123,7 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_DB_MAX_CONNECTIONS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -229,7 +229,7 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_DB_MAX_CONNECTIONS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@ -752,7 +752,7 @@ services:
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
- MYSQL_USER=${_APP_DB_USER}
- MYSQL_PASSWORD=${_APP_DB_PASS}
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}'
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}'
# smtp:
# image: appwrite/smtp:1.2.0