diff --git a/.env b/.env index 15a25aa933..f042b1686d 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ _APP_ENV=development _APP_LOCALE=en -_APP_WORKER_PER_CORE=6 +_APP_WORKER_PER_CORE=2 _APP_CONSOLE_WHITELIST_ROOT=disabled _APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_IPS= @@ -25,6 +25,7 @@ _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword _APP_CONNECTIONS_MAX=251 +_APP_POOL_CLIENTS=14 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 @@ -53,6 +54,7 @@ _APP_FUNCTIONS_BUILD_TIMEOUT=900 _APP_FUNCTIONS_CPUS=1 _APP_FUNCTIONS_MEMORY=512 _APP_FUNCTIONS_INACTIVE_THRESHOLD=600 +_APP_FUNCTIONS_MAINTENANCE_INTERVAL=600 _APP_FUNCTIONS_RUNTIMES_NETWORK=runtimes _APP_EXECUTOR_SECRET=your-secret-key _APP_EXECUTOR_HOST=http://exc1/v1 @@ -63,9 +65,9 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 -_APP_USAGE_TIMESERIES_INTERVAL=20 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled +_APP_USAGE_AGGREGATION_INTERVAL=30 _APP_LOGGING_PROVIDER= _APP_LOGGING_CONFIG= _APP_REGION=default diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e185944b6..cff75b73b4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,6 +33,7 @@ jobs: docker compose build appwrite docker compose up -d sleep 30 + - name: Doctor run: docker compose exec -T appwrite doctor diff --git a/app/cli.php b/app/cli.php index 9857261b47..bfe7bfcefb 100644 --- a/app/cli.php +++ b/app/cli.php @@ -43,17 +43,44 @@ CLI::setResource('pools', function (Registry $register) { }, ['register']); CLI::setResource('dbForConsole', function ($pools, $cache) { - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource() - ; + $sleep = 3; + $maxAttempts = 5; + $attempts = 0; + $ready = false; - $database = new Database($dbAdapter, $cache); + do { + $attempts++; + try { + // Prepare database connection + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource(); - $database->setNamespace('console'); + $dbForConsole = new Database($dbAdapter, $cache); + $dbForConsole->setNamespace('console'); - return $database; + // Ensure tables exist + $collections = Config::getParam('collections', []); + $last = \array_key_last($collections); + + if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { /** TODO cache ready variable using registry */ + throw new Exception('Tables not ready yet.'); + } + + $ready = true; + } catch (\Exception $err) { + Console::warning($err->getMessage()); + $pools->get('console')->reclaim(); + sleep($sleep); + } + } while ($attempts < $maxAttempts); + + if (!$ready) { + throw new Exception("Console is not ready yet. Please try again later."); + } + + return $dbForConsole; }, ['pools', 'cache']); CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { @@ -165,50 +192,4 @@ $cli Console::error($error->getMessage()); }); -$cli - ->init() - ->inject('pools') - ->inject('cache') - ->action(function (Group $pools, Cache $cache) { - $maxAttempts = 5; - $sleep = 3; - - $attempts = 0; - $ready = false; - - do { - $attempts++; - - try { - $pools->get('console')->reclaim(); - - // Prepare database connection - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource(); - - $dbForConsole = new Database($dbAdapter, $cache); - $dbForConsole->setNamespace('console'); - - // Ensure tables exist - $collections = Config::getParam('collections', []); - $last = \array_key_last($collections); - - if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { - throw new Exception('Tables not ready yet.'); - } - - $ready = true; - } catch (\Exception $err) { - Console::warning($err->getMessage()); - sleep($sleep); - } - } while ($attempts < $maxAttempts); - - if (!$ready) { - throw new Exception("Console is not ready yet. Please try again later."); - } - }); - $cli->run(); diff --git a/app/config/variables.php b/app/config/variables.php index 1eb2688c62..85c7a04f25 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -877,7 +877,16 @@ return [ 'required' => false, 'question' => '', 'filter' => '' - ] + ], + [ + 'name' => '_APP_FUNCTIONS_MAINTENANCE_INTERVAL', + 'description' => 'Interval how often executor checks for inactive runimes. The default value is 60 seconds.', + 'introduction' => '1.2.0', + 'default' => '60', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], [ @@ -929,6 +938,15 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', + 'description' => 'The maximum duration (in seconds) upto which to retain hourly usage metrics. The default value is 8640000 seconds (100 days).', + 'introduction' => '', + 'default' => '8640000', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_MAINTENANCE_RETENTION_SCHEDULES', 'description' => 'Schedules deletion interval ( in seconds ) ', diff --git a/app/console b/app/console index 6aa50b8b96..4e2cecefb5 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 6aa50b8b96b7941a710fe097f3f4d3a74f1eb823 +Subproject commit 4e2cecefb571104f0dbbe5a578729f0e17a10242 diff --git a/app/init.php b/app/init.php index 5016a4246c..6af338838d 100644 --- a/app/init.php +++ b/app/init.php @@ -552,10 +552,16 @@ $register->set('pools', function () { ], ]; - $instances = 3; // REST, Realtime, CLI - $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); - $maxConnections = App::getenv('_APP_CONNECTIONS_MAX', 251); - $instanceConnections = $maxConnections / $instances; + $maxConnections = App::getEnv('_APP_CONNECTIONS_MAX', 151); + $instanceConnections = $maxConnections / App::getEnv('_APP_POOL_CLIENTS', 14); + + $multiprocessing = App::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; + + if ($multiprocessing) { + $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); + } else { + $workerCount = 1; + } if ($workerCount > $instanceConnections) { throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index dc7549aa3f..cc53fb8854 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -139,6 +139,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER - _APP_SMS_FROM @@ -483,44 +484,13 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_LOGGING_PROVIDER - - _APP_LOGGING_CONFIG - - appwrite-usage-database: - image: /: - entrypoint: - - usage - - --type=database - container_name: appwrite-usage-database - <<: *x-logging - restart: unless-stopped - networks: - - appwrite - depends_on: - - influxdb - - mariadb - environment: - - _APP_ENV - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -558,6 +528,7 @@ services: environment: - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD + - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD diff --git a/app/workers/builds.php b/app/workers/builds.php index 6b2cfcf66b..d26f07ab75 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -254,6 +254,7 @@ class BuildsV1 extends Worker $statsd = $register->get('statsd'); $usage = new Stats($statsd); $usage + ->setParam('projectInternalId', $project->getInternalId()) ->setParam('projectId', $project->getId()) ->setParam('functionId', $function->getId()) ->setParam('builds.{scope}.compute', 1) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index cd33ed0b79..399fe79673 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -260,6 +260,7 @@ class DeletesV1 extends Worker { $this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) { $dbForProject = $this->getProjectDB($project); + // Delete Usage stats $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::equal('period', ['1h']), diff --git a/app/workers/functions.php b/app/workers/functions.php index 3047bf027e..2333cb315e 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -152,6 +152,10 @@ Server::setResource('execute', function () { ->setAttribute('status', 'failed') ->setAttribute('statusCode', $th->getCode()) ->setAttribute('stderr', $th->getMessage()); + + Console::error($th->getTraceAsString()); + Console::error($th->getFile()); + Console::error($th->getLine()); Console::error($th->getMessage()); } @@ -205,7 +209,7 @@ Server::setResource('execute', function () { $usage ->setParam('projectId', $project->getId()) ->setParam('projectInternalId', $project->getInternalId()) - ->setParam('functionId', $function->getId()) + ->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats ->setParam('executions.{scope}.compute', 1) ->setParam('executionStatus', $execution->getAttribute('status', '')) ->setParam('executionTime', $execution->getAttribute('duration')) diff --git a/composer.lock b/composer.lock index f47d853d6b..7e3705f123 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4893e1c13630239fe6a20d1c652eb484", + "content-hash": "7a4830071d4d0c427adc32da23ed1856", "packages": [ { "name": "adhocore/jwt", @@ -2356,16 +2356,16 @@ }, { "name": "utopia-php/pools", - "version": "0.4.1", + "version": "0.4.2", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979" + "reference": "d2870ab74b31b7f4027799f082e85122154f8bed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", - "reference": "c8f96a33e7fbf58c1145eb6cf0f2c00cbe319979", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/d2870ab74b31b7f4027799f082e85122154f8bed", + "reference": "d2870ab74b31b7f4027799f082e85122154f8bed", "shasum": "" }, "require": { @@ -2401,9 +2401,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.4.1" + "source": "https://github.com/utopia-php/pools/tree/0.4.2" }, - "time": "2022-11-15T08:55:16+00:00" + "time": "2022-11-22T07:55:45+00:00" }, { "name": "utopia-php/preloader", @@ -5403,5 +5403,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.3.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 4ca8876f9b..6c040df4cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,6 +93,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/local/dev + depends_on: - mariadb - redis @@ -105,6 +106,7 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_SERVER_MULTIPROCESS=enabled - _APP_LOCALE - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS @@ -125,6 +127,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -204,6 +207,7 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_SERVER_MULTIPROCESS=enabled - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST @@ -212,6 +216,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -275,6 +280,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -306,6 +313,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -342,6 +351,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -375,6 +386,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -413,6 +426,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET @@ -450,6 +465,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -488,6 +505,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS @@ -519,6 +538,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -544,6 +565,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_OPENSSL_KEY_V1 @@ -564,6 +587,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES appwrite-usage: @@ -583,6 +607,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -591,6 +617,7 @@ services: - _APP_DB_PASS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -598,7 +625,6 @@ services: - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT - _APP_CONNECTIONS_CACHE - - _APP_USAGE_TIMESERIES_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -618,6 +644,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -638,7 +666,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.1.4 + image: openruntimes/executor:0.1.6 networks: - appwrite - runtimes @@ -650,6 +678,7 @@ services: environment: - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD + - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD @@ -769,6 +798,19 @@ services: - 9506:8080 networks: - appwrite + + # appwrite-volume-sync: + # entrypoint: volume-sync + # <<: *x-logging + # container_name: appwrite-volume-sync + # image: appwrite-dev + # command: + # - --source=/data/src/ --destination=/data/dest/ --interval=10 + # networks: + # - appwrite + # # volumes: # Mount the rsync source and destination directories + # # - /nfs/config:/data/src + # # - /storage/config:/data/dest # redis-commander: # image: rediscommander/redis-commander:latest diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 357f0ad0b6..667288330a 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -136,7 +136,7 @@ class Maintenance extends Action $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetentionHourly, $cacheRetention, $schedulesDeletionRetention, $dbForConsole) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole) { $time = DateTime::now(); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); diff --git a/src/Appwrite/Platform/Tasks/Usage.php b/src/Appwrite/Platform/Tasks/Usage.php index 402211c8f2..fa677ea142 100644 --- a/src/Appwrite/Platform/Tasks/Usage.php +++ b/src/Appwrite/Platform/Tasks/Usage.php @@ -7,9 +7,9 @@ use InfluxDB\Database as InfluxDatabase; use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database as UtopiaDatabase; -use Utopia\Validator\WhiteList; use Throwable; use Utopia\Platform\Action; +use Utopia\Registry\Registry; class Usage extends Action { @@ -22,18 +22,28 @@ class Usage extends Action { $this ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->inject('dbForConsole') ->inject('influxdb') + ->inject('register') + ->inject('getProjectDB') ->inject('logError') - ->callback(fn ($type, $dbForConsole, $influxDB, $logError) => $this->action($type, $dbForConsole, $influxDB, $logError)); + ->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError)); } protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void { - $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) + } + + public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError) + { + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); + + $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action); + + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) $region = App::getEnv('region', 'default'); - $usage = new TimeSeries($region, $database, $influxDB, $logError); + $usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger); Console::loop(function () use ($interval, $usage) { $now = date('d-m-Y H:i:s', time()); @@ -47,20 +57,4 @@ class Usage extends Action Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); } - - public function action(string $type, UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, callable $logError) - { - Console::title('Usage Aggregation V1'); - Console::success(APP_NAME . ' usage aggregation process v1 has started'); - - $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action); - - switch ($type) { - case 'timeseries': - $this->aggregateTimeseries($dbForConsole, $influxDB, $errorLogger); - break; - default: - Console::error("Unsupported usage aggregation type"); - } - } } diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 6f4fb94a7d..ce92b6462f 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -8,6 +8,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use InfluxDB\Database as InfluxDatabase; use DateTime; +use Utopia\Registry\Registry; class TimeSeries extends Calculator { @@ -32,6 +33,20 @@ class TimeSeries extends Calculator */ protected $errorHandler; + /** + * Callback to get project DB + * + * @var callable + */ + protected mixed $getProjectDB; + + /** + * Registry + * + * @var Registry + */ + protected Registry $register; + /** * Latest times for metric that was synced to the database * @@ -382,11 +397,13 @@ class TimeSeries extends Calculator ] ]; - public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) + public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null) { parent::__construct($region); $this->database = $database; $this->influxDB = $influxDB; + $this->getProjectDB = $getProjectDB; + $this->register = $register; $this->errorHandler = $errorHandler; } @@ -406,12 +423,13 @@ class TimeSeries extends Calculator private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_' . $projectId); + $project = $this->database->getDocument('projects', $projectId); + $database = call_user_func($this->getProjectDB, $project); try { - $document = $this->database->getDocument('stats', $id); + $document = $database->getDocument('stats', $id); if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ + $database->createDocument('stats', new Document([ '$id' => $id, 'period' => $period, 'time' => $time, @@ -421,7 +439,7 @@ class TimeSeries extends Calculator 'region' => $this->region, ])); } else { - $this->database->updateDocument( + $database->updateDocument( 'stats', $document->getId(), $document->setAttribute('value', $value) @@ -434,6 +452,8 @@ class TimeSeries extends Calculator throw $e; } } + + $this->register->get('pools')->reclaim(); } /**