Merge branch 'feat-db-pools' into refactor-scheduler
This commit is contained in:
commit
8b000aff40
9 changed files with 52 additions and 71 deletions
2
.env
2
.env
|
@ -23,7 +23,7 @@ _APP_DB_SCHEMA=appwrite
|
||||||
_APP_DB_USER=user
|
_APP_DB_USER=user
|
||||||
_APP_DB_PASS=password
|
_APP_DB_PASS=password
|
||||||
_APP_DB_ROOT_PASS=rootsecretpassword
|
_APP_DB_ROOT_PASS=rootsecretpassword
|
||||||
_APP_DB_MAX_CONNECTIONS=251
|
_APP_CONNECTIONS_MAX=251
|
||||||
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
|
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
|
||||||
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
|
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
|
||||||
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
|
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# TBD
|
# TBD
|
||||||
|
|
||||||
- Replace Appwrite executor with OpenRuntimes Executor [#4650](https://github.com/appwrite/appwrite/pull/4650)
|
- Replace Appwrite executor with OpenRuntimes Executor [#4650](https://github.com/appwrite/appwrite/pull/4650)
|
||||||
- Add `_APP_DB_MAX_CONNECTIONS` env var [#4673](https://github.com/appwrite/appwrite/pull/4673)
|
- Add `_APP_CONNECTIONS_MAX` env var [#4673](https://github.com/appwrite/appwrite/pull/4673)
|
||||||
- Update Traefik 2.7 -> 2.9 [#4673](https://github.com/appwrite/appwrite/pull/4673)
|
- Update Traefik 2.7 -> 2.9 [#4673](https://github.com/appwrite/appwrite/pull/4673)
|
||||||
- Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673)
|
- Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673)
|
||||||
|
|
||||||
|
|
|
@ -307,7 +307,7 @@ return [
|
||||||
'filter' => 'password'
|
'filter' => 'password'
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
'name' => '_APP_DB_MAX_CONNECTIONS',
|
'name' => '_APP_CONNECTIONS_MAX',
|
||||||
'description' => 'MariaDB server maximum connections.',
|
'description' => 'MariaDB server maximum connections.',
|
||||||
'introduction' => 'TBD',
|
'introduction' => 'TBD',
|
||||||
'default' => 251,
|
'default' => 251,
|
||||||
|
@ -315,24 +315,24 @@ return [
|
||||||
'question' => '',
|
'question' => '',
|
||||||
'filter' => ''
|
'filter' => ''
|
||||||
],
|
],
|
||||||
[
|
// [
|
||||||
'name' => '_APP_CONNECTIONS_DB_PROJECT',
|
// 'name' => '_APP_CONNECTIONS_DB_PROJECT',
|
||||||
'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
|
// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
|
||||||
'introduction' => 'TBD',
|
// 'introduction' => 'TBD',
|
||||||
'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
|
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
|
||||||
'required' => true,
|
// 'required' => true,
|
||||||
'question' => '',
|
// 'question' => '',
|
||||||
'filter' => ''
|
// 'filter' => ''
|
||||||
],
|
// ],
|
||||||
[
|
// [
|
||||||
'name' => '_APP_CONNECTIONS_DB_CONSOLE',
|
// 'name' => '_APP_CONNECTIONS_DB_CONSOLE',
|
||||||
'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
|
// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
|
||||||
'introduction' => 'TBD',
|
// 'introduction' => 'TBD',
|
||||||
'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
|
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
|
||||||
'required' => true,
|
// 'required' => true,
|
||||||
'question' => '',
|
// 'question' => '',
|
||||||
'filter' => ''
|
// 'filter' => ''
|
||||||
]
|
// ]
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
|
|
36
app/init.php
36
app/init.php
|
@ -97,7 +97,6 @@ const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
|
||||||
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
|
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
|
||||||
const APP_CACHE_BUSTER = 501;
|
const APP_CACHE_BUSTER = 501;
|
||||||
const APP_VERSION_STABLE = '1.0.3';
|
const APP_VERSION_STABLE = '1.0.3';
|
||||||
const APP_DEFAULT_POOL_SIZE = 64;
|
|
||||||
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
|
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
|
||||||
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
|
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
|
||||||
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
|
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
|
||||||
|
@ -497,7 +496,7 @@ $register->set('logger', function () {
|
||||||
$adapter = new $classname($providerConfig);
|
$adapter = new $classname($providerConfig);
|
||||||
return new Logger($adapter);
|
return new Logger($adapter);
|
||||||
});
|
});
|
||||||
$register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
|
$register->set('pools', function () {
|
||||||
$group = new Group();
|
$group = new Group();
|
||||||
|
|
||||||
$fallbackForDB = AppwriteURL::unparse([
|
$fallbackForDB = AppwriteURL::unparse([
|
||||||
|
@ -548,6 +547,17 @@ $register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
|
||||||
],
|
],
|
||||||
];
|
];
|
||||||
|
|
||||||
|
$instances = 2; // REST, Realtime
|
||||||
|
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
|
||||||
|
$maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 251);
|
||||||
|
$instanceConnections = $maxConnections / $instances;
|
||||||
|
|
||||||
|
if ($workerCount > $instanceConnections) {
|
||||||
|
throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);
|
||||||
|
}
|
||||||
|
|
||||||
|
$poolSize = (int)($instanceConnections / $workerCount);
|
||||||
|
|
||||||
foreach ($connections as $key => $connection) {
|
foreach ($connections as $key => $connection) {
|
||||||
$type = $connection['type'] ?? '';
|
$type = $connection['type'] ?? '';
|
||||||
$dsns = $connection['dsns'] ?? '';
|
$dsns = $connection['dsns'] ?? '';
|
||||||
|
@ -621,7 +631,7 @@ $register->set('pools', function ($size = APP_DEFAULT_POOL_SIZE) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
$pool = new Pool($name, $size, function () use ($type, $resource, $dsn) {
|
$pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) {
|
||||||
// Get Adapter
|
// Get Adapter
|
||||||
$adapter = null;
|
$adapter = null;
|
||||||
|
|
||||||
|
@ -1123,26 +1133,6 @@ function getDevice($root): Device
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get database connection pool size for worker processes.
|
|
||||||
*
|
|
||||||
* @return int
|
|
||||||
* @throws \Exception
|
|
||||||
*/
|
|
||||||
function getWorkerPoolSize(): int
|
|
||||||
{
|
|
||||||
$reservedConnections = APP_DEFAULT_POOL_SIZE; // Pool of default size is reserved for the HTTP API
|
|
||||||
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
|
|
||||||
$maxConnections = App::getenv('_APP_DB_MAX_CONNECTIONS', 251);
|
|
||||||
$workerConnections = $maxConnections - $reservedConnections;
|
|
||||||
|
|
||||||
if ($workerCount > $workerConnections) {
|
|
||||||
throw new \Exception('Worker pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (int)($workerConnections / $workerCount);
|
|
||||||
}
|
|
||||||
|
|
||||||
App::setResource('mode', function ($request) {
|
App::setResource('mode', function ($request) {
|
||||||
/** @var Appwrite\Utopia\Request $request */
|
/** @var Appwrite\Utopia\Request $request */
|
||||||
|
|
||||||
|
|
|
@ -37,7 +37,7 @@ function getConsoleDB(): Database
|
||||||
global $register;
|
global $register;
|
||||||
|
|
||||||
/** @var \Utopia\Pools\Group $pools */
|
/** @var \Utopia\Pools\Group $pools */
|
||||||
$pools = $register->get('pools', args: [getWorkerPoolSize()]);
|
$pools = $register->get('pools');
|
||||||
|
|
||||||
$dbAdapter = $pools
|
$dbAdapter = $pools
|
||||||
->get('console')
|
->get('console')
|
||||||
|
@ -57,7 +57,7 @@ function getProjectDB(Document $project): Database
|
||||||
global $register;
|
global $register;
|
||||||
|
|
||||||
/** @var \Utopia\Pools\Group $pools */
|
/** @var \Utopia\Pools\Group $pools */
|
||||||
$pools = $register->get('pools', args: [getWorkerPoolSize()]);
|
$pools = $register->get('pools');
|
||||||
|
|
||||||
if ($project->isEmpty() || $project->getId() === 'console') {
|
if ($project->isEmpty() || $project->getId() === 'console') {
|
||||||
return getConsoleDB();
|
return getConsoleDB();
|
||||||
|
|
|
@ -102,7 +102,7 @@ services:
|
||||||
- _APP_DB_SCHEMA
|
- _APP_DB_SCHEMA
|
||||||
- _APP_DB_USER
|
- _APP_DB_USER
|
||||||
- _APP_DB_PASS
|
- _APP_DB_PASS
|
||||||
- _APP_DB_MAX_CONNECTIONS
|
- _APP_CONNECTIONS_MAX
|
||||||
- _APP_REDIS_HOST
|
- _APP_REDIS_HOST
|
||||||
- _APP_REDIS_PORT
|
- _APP_REDIS_PORT
|
||||||
- _APP_REDIS_USER
|
- _APP_REDIS_USER
|
||||||
|
@ -197,7 +197,7 @@ services:
|
||||||
- _APP_DB_SCHEMA
|
- _APP_DB_SCHEMA
|
||||||
- _APP_DB_USER
|
- _APP_DB_USER
|
||||||
- _APP_DB_PASS
|
- _APP_DB_PASS
|
||||||
- _APP_DB_MAX_CONNECTIONS
|
- _APP_CONNECTIONS_MAX
|
||||||
- _APP_REDIS_HOST
|
- _APP_REDIS_HOST
|
||||||
- _APP_REDIS_PORT
|
- _APP_REDIS_PORT
|
||||||
- _APP_REDIS_USER
|
- _APP_REDIS_USER
|
||||||
|
@ -622,7 +622,7 @@ services:
|
||||||
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
|
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
|
||||||
- MYSQL_USER=${_APP_DB_USER}
|
- MYSQL_USER=${_APP_DB_USER}
|
||||||
- MYSQL_PASSWORD=${_APP_DB_PASS}
|
- MYSQL_PASSWORD=${_APP_DB_PASS}
|
||||||
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}'
|
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}'
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:7.0.4-alpine
|
image: redis:7.0.4-alpine
|
||||||
|
|
|
@ -59,7 +59,7 @@
|
||||||
"utopia-php/platform": "0.3.*",
|
"utopia-php/platform": "0.3.*",
|
||||||
"utopia-php/pools": "0.4.*",
|
"utopia-php/pools": "0.4.*",
|
||||||
"utopia-php/preloader": "0.2.*",
|
"utopia-php/preloader": "0.2.*",
|
||||||
"utopia-php/registry": "dev-feat-allow-params as 0.5.0",
|
"utopia-php/registry": "0.5.0",
|
||||||
"utopia-php/storage": "0.11.*",
|
"utopia-php/storage": "0.11.*",
|
||||||
"utopia-php/swoole": "0.5.*",
|
"utopia-php/swoole": "0.5.*",
|
||||||
"utopia-php/websocket": "0.1.0",
|
"utopia-php/websocket": "0.1.0",
|
||||||
|
|
27
composer.lock
generated
27
composer.lock
generated
|
@ -4,7 +4,7 @@
|
||||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||||
"This file is @generated automatically"
|
"This file is @generated automatically"
|
||||||
],
|
],
|
||||||
"content-hash": "b125039c64ae4cbe0d2a1b57322d0ebe",
|
"content-hash": "bf3e2ed6ee8e49ab74af97b368b89a63",
|
||||||
"packages": [
|
"packages": [
|
||||||
{
|
{
|
||||||
"name": "adhocore/jwt",
|
"name": "adhocore/jwt",
|
||||||
|
@ -2359,16 +2359,16 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "utopia-php/registry",
|
"name": "utopia-php/registry",
|
||||||
"version": "dev-feat-allow-params",
|
"version": "0.5.0",
|
||||||
"source": {
|
"source": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://github.com/utopia-php/registry.git",
|
"url": "https://github.com/utopia-php/registry.git",
|
||||||
"reference": "6c571f8f4127094b3af8909d1b595fd6b937255d"
|
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d"
|
||||||
},
|
},
|
||||||
"dist": {
|
"dist": {
|
||||||
"type": "zip",
|
"type": "zip",
|
||||||
"url": "https://api.github.com/repos/utopia-php/registry/zipball/6c571f8f4127094b3af8909d1b595fd6b937255d",
|
"url": "https://api.github.com/repos/utopia-php/registry/zipball/bedc4ed54527b2803e6dfdccc39449f98522b70d",
|
||||||
"reference": "6c571f8f4127094b3af8909d1b595fd6b937255d",
|
"reference": "bedc4ed54527b2803e6dfdccc39449f98522b70d",
|
||||||
"shasum": ""
|
"shasum": ""
|
||||||
},
|
},
|
||||||
"require": {
|
"require": {
|
||||||
|
@ -2405,9 +2405,9 @@
|
||||||
],
|
],
|
||||||
"support": {
|
"support": {
|
||||||
"issues": "https://github.com/utopia-php/registry/issues",
|
"issues": "https://github.com/utopia-php/registry/issues",
|
||||||
"source": "https://github.com/utopia-php/registry/tree/feat-allow-params"
|
"source": "https://github.com/utopia-php/registry/tree/0.5.0"
|
||||||
},
|
},
|
||||||
"time": "2022-11-09T02:23:35+00:00"
|
"time": "2021-03-10T10:45:22+00:00"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "utopia-php/storage",
|
"name": "utopia-php/storage",
|
||||||
|
@ -5215,18 +5215,9 @@
|
||||||
"time": "2022-09-28T08:42:51+00:00"
|
"time": "2022-09-28T08:42:51+00:00"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"aliases": [
|
"aliases": [],
|
||||||
{
|
|
||||||
"package": "utopia-php/registry",
|
|
||||||
"version": "dev-feat-allow-params",
|
|
||||||
"alias": "0.5.0",
|
|
||||||
"alias_normalized": "0.5.0.0"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"minimum-stability": "stable",
|
"minimum-stability": "stable",
|
||||||
"stability-flags": {
|
"stability-flags": [],
|
||||||
"utopia-php/registry": 20
|
|
||||||
},
|
|
||||||
"prefer-stable": false,
|
"prefer-stable": false,
|
||||||
"prefer-lowest": false,
|
"prefer-lowest": false,
|
||||||
"platform": {
|
"platform": {
|
||||||
|
|
|
@ -123,7 +123,7 @@ services:
|
||||||
- _APP_DB_SCHEMA
|
- _APP_DB_SCHEMA
|
||||||
- _APP_DB_USER
|
- _APP_DB_USER
|
||||||
- _APP_DB_PASS
|
- _APP_DB_PASS
|
||||||
- _APP_DB_MAX_CONNECTIONS
|
- _APP_CONNECTIONS_MAX
|
||||||
- _APP_REDIS_HOST
|
- _APP_REDIS_HOST
|
||||||
- _APP_REDIS_PORT
|
- _APP_REDIS_PORT
|
||||||
- _APP_REDIS_USER
|
- _APP_REDIS_USER
|
||||||
|
@ -229,7 +229,7 @@ services:
|
||||||
- _APP_DB_SCHEMA
|
- _APP_DB_SCHEMA
|
||||||
- _APP_DB_USER
|
- _APP_DB_USER
|
||||||
- _APP_DB_PASS
|
- _APP_DB_PASS
|
||||||
- _APP_DB_MAX_CONNECTIONS
|
- _APP_CONNECTIONS_MAX
|
||||||
- _APP_REDIS_HOST
|
- _APP_REDIS_HOST
|
||||||
- _APP_REDIS_PORT
|
- _APP_REDIS_PORT
|
||||||
- _APP_REDIS_USER
|
- _APP_REDIS_USER
|
||||||
|
@ -752,7 +752,7 @@ services:
|
||||||
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
|
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
|
||||||
- MYSQL_USER=${_APP_DB_USER}
|
- MYSQL_USER=${_APP_DB_USER}
|
||||||
- MYSQL_PASSWORD=${_APP_DB_PASS}
|
- MYSQL_PASSWORD=${_APP_DB_PASS}
|
||||||
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_DB_MAX_CONNECTIONS}'
|
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}'
|
||||||
|
|
||||||
# smtp:
|
# smtp:
|
||||||
# image: appwrite/smtp:1.2.0
|
# image: appwrite/smtp:1.2.0
|
||||||
|
|
Loading…
Reference in a new issue