CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-06 23:21:05 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'feat-db-pools' of github.com:appwrite/appwrite into feat-db-pools

Browse source
This commit is contained in:
Christy Jacob 2022-11-23 03:03:19 +05:30
parent 5730c3deb6 e7a51fac74
commit 2fcaa02226
15 changed files with 173 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.env
View file

@ -1,6 +1,6 @@
_APP_ENV=development _APP_ENV=development
_APP_LOCALE=en _APP_LOCALE=en
_APP_WORKER_PER_CORE=6 _APP_WORKER_PER_CORE=2
_APP_CONSOLE_WHITELIST_ROOT=disabled _APP_CONSOLE_WHITELIST_ROOT=disabled
_APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_EMAILS=
_APP_CONSOLE_WHITELIST_IPS= _APP_CONSOLE_WHITELIST_IPS=
@ -25,6 +25,7 @@ _APP_DB_USER=user
_APP_DB_PASS=password _APP_DB_PASS=password
_APP_DB_ROOT_PASS=rootsecretpassword _APP_DB_ROOT_PASS=rootsecretpassword
_APP_CONNECTIONS_MAX=251 _APP_CONNECTIONS_MAX=251
_APP_POOL_CLIENTS=14
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
@ -53,6 +54,7 @@ _APP_FUNCTIONS_BUILD_TIMEOUT=900
_APP_FUNCTIONS_CPUS=1 _APP_FUNCTIONS_CPUS=1
_APP_FUNCTIONS_MEMORY=512 _APP_FUNCTIONS_MEMORY=512
_APP_FUNCTIONS_INACTIVE_THRESHOLD=600 _APP_FUNCTIONS_INACTIVE_THRESHOLD=600
_APP_FUNCTIONS_MAINTENANCE_INTERVAL=600
_APP_FUNCTIONS_RUNTIMES_NETWORK=runtimes _APP_FUNCTIONS_RUNTIMES_NETWORK=runtimes
_APP_EXECUTOR_SECRET=your-secret-key _APP_EXECUTOR_SECRET=your-secret-key
_APP_EXECUTOR_HOST=http://exc1/v1 _APP_EXECUTOR_HOST=http://exc1/v1
@ -63,9 +65,9 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_MAINTENANCE_RETENTION_AUDIT=1209600
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400
_APP_USAGE_TIMESERIES_INTERVAL=20
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
_APP_USAGE_STATS=enabled _APP_USAGE_STATS=enabled
_APP_USAGE_AGGREGATION_INTERVAL=30
_APP_LOGGING_PROVIDER= _APP_LOGGING_PROVIDER=
_APP_LOGGING_CONFIG= _APP_LOGGING_CONFIG=
_APP_REGION=default _APP_REGION=default

1
.github/workflows/tests.yml vendored
View file

@ -33,6 +33,7 @@ jobs:
docker compose build appwrite docker compose build appwrite
docker compose up -d docker compose up -d
sleep 30 sleep 30
- name: Doctor - name: Doctor
run: docker compose exec -T appwrite doctor run: docker compose exec -T appwrite doctor

89
app/cli.php
View file

@ -43,17 +43,44 @@ CLI::setResource('pools', function (Registry $register) {
}, ['register']); }, ['register']);
CLI::setResource('dbForConsole', function ($pools, $cache) { CLI::setResource('dbForConsole', function ($pools, $cache) {
$dbAdapter = $pools $sleep = 3;
->get('console') $maxAttempts = 5;
->pop() $attempts = 0;
->getResource() $ready = false;
;
$database = new Database($dbAdapter, $cache); do {
$attempts++;
try {
// Prepare database connection
$dbAdapter = $pools
->get('console')
->pop()
->getResource();
$database->setNamespace('console'); $dbForConsole = new Database($dbAdapter, $cache);
$dbForConsole->setNamespace('console');
return $database; // Ensure tables exist
$collections = Config::getParam('collections', []);
$last = \array_key_last($collections);
if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { /** TODO cache ready variable using registry */
throw new Exception('Tables not ready yet.');
}
$ready = true;
} catch (\Exception $err) {
Console::warning($err->getMessage());
$pools->get('console')->reclaim();
sleep($sleep);
}
} while ($attempts < $maxAttempts);
if (!$ready) {
throw new Exception("Console is not ready yet. Please try again later.");
}
return $dbForConsole;
}, ['pools', 'cache']); }, ['pools', 'cache']);
CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
@ -165,50 +192,4 @@ $cli
Console::error($error->getMessage()); Console::error($error->getMessage());
}); });
$cli
->init()
->inject('pools')
->inject('cache')
->action(function (Group $pools, Cache $cache) {
$maxAttempts = 5;
$sleep = 3;
$attempts = 0;
$ready = false;
do {
$attempts++;
try {
$pools->get('console')->reclaim();
// Prepare database connection
$dbAdapter = $pools
->get('console')
->pop()
->getResource();
$dbForConsole = new Database($dbAdapter, $cache);
$dbForConsole->setNamespace('console');
// Ensure tables exist
$collections = Config::getParam('collections', []);
$last = \array_key_last($collections);
if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) {
throw new Exception('Tables not ready yet.');
}
$ready = true;
} catch (\Exception $err) {
Console::warning($err->getMessage());
sleep($sleep);
}
} while ($attempts < $maxAttempts);
if (!$ready) {
throw new Exception("Console is not ready yet. Please try again later.");
}
});
$cli->run(); $cli->run();

20
app/config/variables.php
View file

@ -877,7 +877,16 @@ return [
'required' => false, 'required' => false,
'question' => '', 'question' => '',
'filter' => '' 'filter' => ''
] ],
[
'name' => '_APP_FUNCTIONS_MAINTENANCE_INTERVAL',
'description' => 'Interval how often executor checks for inactive runimes. The default value is 60 seconds.',
'introduction' => '1.2.0',
'default' => '60',
'required' => false,
'question' => '',
'filter' => ''
],
], ],
], ],
[ [
@ -929,6 +938,15 @@ return [
'question' => '', 'question' => '',
'filter' => '' 'filter' => ''
], ],
[
'name' => '_APP_MAINTENANCE_RETENTION_USAGE_HOURLY',
'description' => 'The maximum duration (in seconds) upto which to retain hourly usage metrics. The default value is 8640000 seconds (100 days).',
'introduction' => '',
'default' => '8640000',
'required' => false,
'question' => '',
'filter' => ''
],
[ [
'name' => '_APP_MAINTENANCE_RETENTION_SCHEDULES', 'name' => '_APP_MAINTENANCE_RETENTION_SCHEDULES',
'description' => 'Schedules deletion interval ( in seconds ) ', 'description' => 'Schedules deletion interval ( in seconds ) ',

2
app/console

@ -1 +1 @@
Subproject commit 6aa50b8b96b7941a710fe097f3f4d3a74f1eb823 Subproject commit 4e2cecefb571104f0dbbe5a578729f0e17a10242

14
app/init.php
View file

@ -552,10 +552,16 @@ $register->set('pools', function () {
], ],
]; ];
$instances = 3; // REST, Realtime, CLI $maxConnections = App::getEnv('_APP_CONNECTIONS_MAX', 151);
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); $instanceConnections = $maxConnections / App::getEnv('_APP_POOL_CLIENTS', 14);
$maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 251);
$instanceConnections = $maxConnections / $instances; $multiprocessing = App::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled';
if ($multiprocessing) {
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
} else {
$workerCount = 1;
}
if ($workerCount > $instanceConnections) { if ($workerCount > $instanceConnections) {
throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);

35
app/views/install/compose.phtml
View file

@ -139,6 +139,7 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_RETENTION_SCHEDULES
- _APP_SMS_PROVIDER - _APP_SMS_PROVIDER
- _APP_SMS_FROM - _APP_SMS_FROM
@ -483,44 +484,13 @@ services:
- _APP_DB_SCHEMA - _APP_DB_SCHEMA
- _APP_DB_USER - _APP_DB_USER
- _APP_DB_PASS - _APP_DB_PASS
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST - _APP_REDIS_HOST
- _APP_REDIS_PORT - _APP_REDIS_PORT
- _APP_REDIS_USER - _APP_REDIS_USER
- _APP_REDIS_PASS - _APP_REDIS_PASS
- _APP_INFLUXDB_HOST - _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT - _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-usage-database:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint:
- usage
- --type=database
container_name: appwrite-usage-database
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_LOGGING_PROVIDER - _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG - _APP_LOGGING_CONFIG
@ -558,6 +528,7 @@ services:
environment: environment:
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
- OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
- OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL
- OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
- OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME
- OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD

1
app/workers/builds.php
View file

@ -254,6 +254,7 @@ class BuildsV1 extends Worker
$statsd = $register->get('statsd'); $statsd = $register->get('statsd');
$usage = new Stats($statsd); $usage = new Stats($statsd);
$usage $usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId()) ->setParam('projectId', $project->getId())
->setParam('functionId', $function->getId()) ->setParam('functionId', $function->getId())
->setParam('builds.{scope}.compute', 1) ->setParam('builds.{scope}.compute', 1)

1
app/workers/deletes.php
View file

@ -260,6 +260,7 @@ class DeletesV1 extends Worker
{ {
$this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) { $this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) {
$dbForProject = $this->getProjectDB($project); $dbForProject = $this->getProjectDB($project);
// Delete Usage stats
$this->deleteByGroup('stats', [ $this->deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']), Query::equal('period', ['1h']),

6
app/workers/functions.php
View file

@ -152,6 +152,10 @@ Server::setResource('execute', function () {
->setAttribute('status', 'failed') ->setAttribute('status', 'failed')
->setAttribute('statusCode', $th->getCode()) ->setAttribute('statusCode', $th->getCode())
->setAttribute('stderr', $th->getMessage()); ->setAttribute('stderr', $th->getMessage());
Console::error($th->getTraceAsString());
Console::error($th->getFile());
Console::error($th->getLine());
Console::error($th->getMessage()); Console::error($th->getMessage());
} }
@ -205,7 +209,7 @@ Server::setResource('execute', function () {
$usage $usage
->setParam('projectId', $project->getId()) ->setParam('projectId', $project->getId())
->setParam('projectInternalId', $project->getInternalId()) ->setParam('projectInternalId', $project->getInternalId())
->setParam('functionId', $function->getId()) ->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats
->setParam('executions.{scope}.compute', 1) ->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', '')) ->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration')) ->setParam('executionTime', $execution->getAttribute('duration'))

16
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "4893e1c13630239fe6a20d1c652eb484", "content-hash": "7a4830071d4d0c427adc32da23ed1856",
"packages": [ "packages": [
{ {
"name": "adhocore/jwt", "name": "adhocore/jwt",
@ -2356,16 +2356,16 @@
}, },
{ {
"name": "utopia-php/pools", "name": "utopia-php/pools",
"version": "0.4.1", "version": "0.4.2",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/utopia-php/pools.git", "url": "https://github.com/utopia-php/pools.git",
"reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979" "reference": "d2870ab74b31b7f4027799f082e85122154f8bed"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/utopia-php/pools/zipball/c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", "url": "https://api.github.com/repos/utopia-php/pools/zipball/d2870ab74b31b7f4027799f082e85122154f8bed",
"reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", "reference": "d2870ab74b31b7f4027799f082e85122154f8bed",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -2401,9 +2401,9 @@
], ],
"support": { "support": {
"issues": "https://github.com/utopia-php/pools/issues", "issues": "https://github.com/utopia-php/pools/issues",
"source": "https://github.com/utopia-php/pools/tree/0.4.1" "source": "https://github.com/utopia-php/pools/tree/0.4.2"
}, },
"time": "2022-11-15T08:55:16+00:00" "time": "2022-11-22T07:55:45+00:00"
}, },
{ {
"name": "utopia-php/preloader", "name": "utopia-php/preloader",
@ -5403,5 +5403,5 @@
"platform-overrides": { "platform-overrides": {
"php": "8.0" "php": "8.0"
}, },
"plugin-api-version": "2.1.0" "plugin-api-version": "2.3.0"
} }

46
docker-compose.yml
View file

@ -93,6 +93,7 @@ services:
- ./public:/usr/src/code/public - ./public:/usr/src/code/public
- ./src:/usr/src/code/src - ./src:/usr/src/code/src
- ./dev:/usr/local/dev - ./dev:/usr/local/dev
depends_on: depends_on:
- mariadb - mariadb
- redis - redis
@ -105,6 +106,7 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_SERVER_MULTIPROCESS=enabled
- _APP_LOCALE - _APP_LOCALE
- _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS - _APP_CONSOLE_WHITELIST_EMAILS
@ -125,6 +127,7 @@ services:
- _APP_DB_USER - _APP_DB_USER
- _APP_DB_PASS - _APP_DB_PASS
- _APP_CONNECTIONS_MAX - _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST - _APP_REDIS_HOST
- _APP_REDIS_PORT - _APP_REDIS_PORT
- _APP_REDIS_USER - _APP_REDIS_USER
@ -204,6 +207,7 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_SERVER_MULTIPROCESS=enabled
- _APP_OPTIONS_ABUSE - _APP_OPTIONS_ABUSE
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
- _APP_DB_HOST - _APP_DB_HOST
@ -212,6 +216,7 @@ services:
- _APP_DB_USER - _APP_DB_USER
- _APP_DB_PASS - _APP_DB_PASS
- _APP_CONNECTIONS_MAX - _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST - _APP_REDIS_HOST
- _APP_REDIS_PORT - _APP_REDIS_PORT
- _APP_REDIS_USER - _APP_REDIS_USER
@ -275,6 +280,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
- _APP_REDIS_HOST - _APP_REDIS_HOST
@ -306,6 +313,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
- _APP_DB_HOST - _APP_DB_HOST
- _APP_DB_PORT - _APP_DB_PORT
@ -342,6 +351,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
- _APP_DB_HOST - _APP_DB_HOST
- _APP_DB_PORT - _APP_DB_PORT
@ -375,6 +386,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
- _APP_EXECUTOR_SECRET - _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST - _APP_EXECUTOR_HOST
@ -413,6 +426,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
- _APP_DOMAIN - _APP_DOMAIN
- _APP_DOMAIN_TARGET - _APP_DOMAIN_TARGET
@ -450,6 +465,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
- _APP_DB_HOST - _APP_DB_HOST
- _APP_DB_PORT - _APP_DB_PORT
@ -488,6 +505,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
- _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_NAME
- _APP_SYSTEM_EMAIL_ADDRESS - _APP_SYSTEM_EMAIL_ADDRESS
@ -519,6 +538,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST - _APP_REDIS_HOST
- _APP_REDIS_PORT - _APP_REDIS_PORT
- _APP_REDIS_USER - _APP_REDIS_USER
@ -544,6 +565,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_DOMAIN - _APP_DOMAIN
- _APP_DOMAIN_TARGET - _APP_DOMAIN_TARGET
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
@ -564,6 +587,7 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-usage: appwrite-usage:
@ -583,6 +607,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1 - _APP_OPENSSL_KEY_V1
- _APP_DB_HOST - _APP_DB_HOST
- _APP_DB_PORT - _APP_DB_PORT
@ -591,6 +617,7 @@ services:
- _APP_DB_PASS - _APP_DB_PASS
- _APP_INFLUXDB_HOST - _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT - _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST - _APP_REDIS_HOST
- _APP_REDIS_PORT - _APP_REDIS_PORT
- _APP_REDIS_USER - _APP_REDIS_USER
@ -598,7 +625,6 @@ services:
- _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT - _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE - _APP_CONNECTIONS_CACHE
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_LOGGING_PROVIDER - _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG - _APP_LOGGING_CONFIG
@ -618,6 +644,8 @@ services:
environment: environment:
- _APP_ENV - _APP_ENV
- _APP_WORKER_PER_CORE - _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST - _APP_REDIS_HOST
- _APP_REDIS_PORT - _APP_REDIS_PORT
- _APP_REDIS_USER - _APP_REDIS_USER
@ -638,7 +666,7 @@ services:
hostname: exc1 hostname: exc1
<<: *x-logging <<: *x-logging
stop_signal: SIGINT stop_signal: SIGINT
image: openruntimes/executor:0.1.4 image: openruntimes/executor:0.1.6
networks: networks:
- appwrite - appwrite
- runtimes - runtimes
@ -650,6 +678,7 @@ services:
environment: environment:
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
- OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
- OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL
- OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
- OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME
- OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD
@ -769,6 +798,19 @@ services:
- 9506:8080 - 9506:8080
networks: networks:
- appwrite - appwrite
# appwrite-volume-sync:
# entrypoint: volume-sync
# <<: *x-logging
# container_name: appwrite-volume-sync
# image: appwrite-dev
# command:
# - --source=/data/src/ --destination=/data/dest/ --interval=10
# networks:
# - appwrite
# # volumes: # Mount the rsync source and destination directories
# # - /nfs/config:/data/src
# # - /storage/config:/data/dest
# redis-commander: # redis-commander:
# image: rediscommander/redis-commander:latest # image: rediscommander/redis-commander:latest

2
src/Appwrite/Platform/Tasks/Maintenance.php
View file

@ -136,7 +136,7 @@ class Maintenance extends Action
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetentionHourly, $cacheRetention, $schedulesDeletionRetention, $dbForConsole) { Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole) {
$time = DateTime::now(); $time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");

36
src/Appwrite/Platform/Tasks/Usage.php
View file

@ -7,9 +7,9 @@ use InfluxDB\Database as InfluxDatabase;
use Utopia\App; use Utopia\App;
use Utopia\CLI\Console; use Utopia\CLI\Console;
use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Validator\WhiteList;
use Throwable; use Throwable;
use Utopia\Platform\Action; use Utopia\Platform\Action;
use Utopia\Registry\Registry;
class Usage extends Action class Usage extends Action
{ {
@ -22,18 +22,28 @@ class Usage extends Action
{ {
$this $this
->desc('Schedules syncing data from influxdb to Appwrite console db') ->desc('Schedules syncing data from influxdb to Appwrite console db')
->param('type', 'timeseries', new WhiteList(['timeseries', 'database']))
->inject('dbForConsole') ->inject('dbForConsole')
->inject('influxdb') ->inject('influxdb')
->inject('register')
->inject('getProjectDB')
->inject('logError') ->inject('logError')
->callback(fn ($type, $dbForConsole, $influxDB, $logError) => $this->action($type, $dbForConsole, $influxDB, $logError)); ->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError));
} }
protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
{ {
$interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) }
public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError)
{
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
$region = App::getEnv('region', 'default'); $region = App::getEnv('region', 'default');
$usage = new TimeSeries($region, $database, $influxDB, $logError); $usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger);
Console::loop(function () use ($interval, $usage) { Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time()); $now = date('d-m-Y H:i:s', time());
@ -47,20 +57,4 @@ class Usage extends Action
Console::info("[{$now}] Aggregation took {$loopTook} seconds"); Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval); }, $interval);
} }
public function action(string $type, UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, callable $logError)
{
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
switch ($type) {
case 'timeseries':
$this->aggregateTimeseries($dbForConsole, $influxDB, $errorLogger);
break;
default:
Console::error("Unsupported usage aggregation type");
}
}
} }

30
src/Appwrite/Usage/Calculators/TimeSeries.php
View file

@ -8,6 +8,7 @@ use Utopia\Database\Database;
use Utopia\Database\Document; use Utopia\Database\Document;
use InfluxDB\Database as InfluxDatabase; use InfluxDB\Database as InfluxDatabase;
use DateTime; use DateTime;
use Utopia\Registry\Registry;
class TimeSeries extends Calculator class TimeSeries extends Calculator
{ {
@ -32,6 +33,20 @@ class TimeSeries extends Calculator
*/ */
protected $errorHandler; protected $errorHandler;
/**
* Callback to get project DB
*
* @var callable
*/
protected mixed $getProjectDB;
/**
* Registry
*
* @var Registry
*/
protected Registry $register;
/** /**
* Latest times for metric that was synced to the database * Latest times for metric that was synced to the database
* *
@ -382,11 +397,13 @@ class TimeSeries extends Calculator
] ]
]; ];
public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null)
{ {
parent::__construct($region); parent::__construct($region);
$this->database = $database; $this->database = $database;
$this->influxDB = $influxDB; $this->influxDB = $influxDB;
$this->getProjectDB = $getProjectDB;
$this->register = $register;
$this->errorHandler = $errorHandler; $this->errorHandler = $errorHandler;
} }
@ -406,12 +423,13 @@ class TimeSeries extends Calculator
private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
{ {
$id = \md5("{$time}_{$period}_{$metric}"); $id = \md5("{$time}_{$period}_{$metric}");
$this->database->setNamespace('_' . $projectId); $project = $this->database->getDocument('projects', $projectId);
$database = call_user_func($this->getProjectDB, $project);
try { try {
$document = $this->database->getDocument('stats', $id); $document = $database->getDocument('stats', $id);
if ($document->isEmpty()) { if ($document->isEmpty()) {
$this->database->createDocument('stats', new Document([ $database->createDocument('stats', new Document([
'$id' => $id, '$id' => $id,
'period' => $period, 'period' => $period,
'time' => $time, 'time' => $time,
@ -421,7 +439,7 @@ class TimeSeries extends Calculator
'region' => $this->region, 'region' => $this->region,
])); ]));
} else { } else {
$this->database->updateDocument( $database->updateDocument(
'stats', 'stats',
$document->getId(), $document->getId(),
$document->setAttribute('value', $value) $document->setAttribute('value', $value)
@ -434,6 +452,8 @@ class TimeSeries extends Calculator
throw $e; throw $e;
} }
} }
$this->register->get('pools')->reclaim();
} }
/** /**