diff --git a/.env b/.env index c8837dacfb..096bde5c52 100644 --- a/.env +++ b/.env @@ -46,6 +46,7 @@ _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= _APP_HAMSTER_RECIPIENTS= +_APP_HAMSTER_INTERVAL=86400 _APP_SMS_PROVIDER=sms://username:password@mock _APP_SMS_FROM=+123456789 _APP_STORAGE_LIMIT=30000000 diff --git a/app/console b/app/console index aea8c5f2bb..43891a526e 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit aea8c5f2bbe0836fc9fb4720b21d44fe01ac93d1 +Subproject commit 43891a526e061454617cbb13def3c4901d99a7f1 diff --git a/docker-compose.yml b/docker-compose.yml index 45863846cd..fd8c425e48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -544,6 +544,39 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + appwrite-hamster: + entrypoint: hamster + <<: *x-logging + container_name: appwrite-hamster + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_HAMSTER_RECIPIENTS + - _APP_HAMSTER_INTERVAL + appwrite-maintenance: entrypoint: maintenance <<: *x-logging diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 801fff4537..5e300e686c 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -145,79 +145,94 @@ class Hamster extends Action public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void { - Console::info('Getting stats...'); - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); + Console::title('Cloud Hamster V1'); + Console::success(APP_NAME . ' cloud hamster process v1 has started'); - /** CSV stuff */ - $this->date = date('Y-m-d'); - $this->path = "{$this->directory}/stats_{$this->date}.csv"; - $csv = Writer::createFromPath($this->path, 'w'); - $csv->insertOne($this->columns); + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) - /** Database connections */ - $totalProjects = $dbForConsole->count('projects') + 1; - Console::success("Found a total of: {$totalProjects} projects"); + Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $interval) { - $projects = [$console]; - $count = 0; - $limit = 30; - $sum = 30; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Getting Cloud Usage Stats every {$interval} seconds"); + $loopStart = microtime(true); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects') + 1; + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project); + $csv->insertOne(array_values($statsPerProject)); + } catch (\Throwable $th) { + throw $th; + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } } - Console::info("Getting stats for {$project->getId()}"); + $sum = \count($projects); - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); + $offset = $offset + $limit; + $count = $count + $sum; - $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project); - $csv->insertOne(array_values($statsPerProject)); - } catch (\Throwable $th) { - throw $th; - Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } + Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); } - $sum = \count($projects); + $this->sendEmail($register); - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); + $pools + ->get('console') + ->reclaim(); - $offset = $offset + $limit; - $count = $count + $sum; - - Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); - } - - $this->sendEmail($register); - - $pools - ->get('console') - ->reclaim(); + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Cloud Stats took {$loopTook} seconds"); + }, $interval); } private function sendEmail(Registry $register)