From 1225c8b7c3fd5095323f59d3fed6576f473621a9 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 30 May 2023 18:06:51 +0300 Subject: [PATCH 1/9] calc tier stats --- Dockerfile | 2 +- app/console | 2 +- bin/calc-tier-stats | 3 + src/Appwrite/Platform/Services/Tasks.php | 5 +- src/Appwrite/Platform/Tasks/CalcTierStats.php | 292 ++++++++++++++++++ 5 files changed, 301 insertions(+), 3 deletions(-) create mode 100644 bin/calc-tier-stats create mode 100644 src/Appwrite/Platform/Tasks/CalcTierStats.php diff --git a/Dockerfile b/Dockerfile index ec5cad2ea..baf6f4d19 100755 --- a/Dockerfile +++ b/Dockerfile @@ -120,7 +120,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/calc-tier-stats && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ diff --git a/app/console b/app/console index 3a0c8f033..2fac7c1f3 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 3a0c8f0334d402a6e27c8b5c0512f3d60080ddbd +Subproject commit 2fac7c1f390637f7dfc11882dcad6fb6508c83f1 diff --git a/bin/calc-tier-stats b/bin/calc-tier-stats new file mode 100644 index 000000000..c7fb71e6f --- /dev/null +++ b/bin/calc-tier-stats @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php calc-tier-stats $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 1073dfb0f..e05baee68 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -20,6 +20,7 @@ use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; use Appwrite\Platform\Tasks\CalcUsersStats; +use Appwrite\Platform\Tasks\CalcTierStats; class Tasks extends Service { @@ -43,6 +44,8 @@ class Tasks extends Service ->addAction(SDKs::getName(), new SDKs()) ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()) - ->addAction(CalcUsersStats::getName(), new CalcUsersStats()); + ->addAction(CalcUsersStats::getName(), new CalcUsersStats()) + ->addAction(CalctierStats::getName(), new CalcTierStats()) + ; } } diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php new file mode 100644 index 000000000..82d3e0200 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -0,0 +1,292 @@ + 'Requests', + 'project.$all.network.bandwidth' => 'Bandwidth', + + ]; + + public static function getName(): string + { + return 'calc-tier-stats'; + } + + public function __construct() + { + + $this + ->desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('register') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($pools, $cache, $dbForConsole, $register); + }); + } + + public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + { + //docker compose exec -t appwrite calc-tier-stats + + Console::title('Cloud free tier stats calculation V1'); + Console::success(APP_NAME . ' cloud free tier stats calculation has started'); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/tier_stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects'); + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + /** Get Project ID */ + $stats['Project ID'] = $project->getId(); + + ///** Get Project Name */ + //$stats['Project Name'] = $project->getAttribute('name'); + + + /** Get Organization Name and Id */ + //$teamId = $project->getAttribute('teamId', null); + //$teamName = null; + //if ($teamId) { + //$team = $dbForConsole->getDocument('teams', $teamId); + //$teamName = $team->getAttribute('name'); + // } + + $stats['Organization ID'] = $project->getAttribute('teamInternalId', null); + + /** Get Total Members */ + $teamInternalId = $project->getAttribute('teamInternalId', null); + if ($teamInternalId) { + $stats['Users'] = $dbForConsole->count('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]) + ]); + } else { + $stats['Users'] = 0; + } + + /** Get Total internal Teams */ + $stats['Teams'] = $dbForProject->count('teams', []); + + /** Get Usage stats */ + $range = '90d'; + $periods = [ + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $tmp = []; + $metrics = $this->usageStats; + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) { + foreach ($metrics as $metric => $name) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $tmp[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $tmp[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $tmp[$metric] = array_reverse($tmp[$metric]); + $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value')); + } + }); + + foreach ($tmp as $key => $value) { + $stats[$metrics[$key]] = $value; + } + + /** Get Domains */ + $stats['Domains'] = $dbForConsole->count('domains', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + ]); + + + /** Get Webhooks */ + $stats['Webhooks'] = $dbForConsole->count('webhooks', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + ]); + + /** Get Platforms */ + $stats['Platforms'] = $dbForConsole->count('platforms', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + ]); + + /** Get Files & Buckets */ + $filesCount = 0; + $filesSum = 0; + $maxFileSize = 0; + $buckets = $dbForProject->find('buckets', []); + $counter = 0; + foreach ($buckets as $bucket) { + $filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', [], 0); + $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []); + $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]); + if ($file->getAttribute('sizeOriginal') > $maxFileSize) { + $maxFileSize = $file->getAttribute('sizeOriginal'); + } + $counter++; + } + $stats['Buckets'] = $counter; + $stats['Files'] = $filesCount; + $stats['Storage'] = $filesSum; + $stats['Max file size'] = $maxFileSize; + + /** Get Total Functions */ + $stats['Databases'] = $dbForProject->count('databases', []); + + /** Get Total Functions */ + $stats['Functions'] = $dbForProject->count('functions', []); + + /** Get Total Deployments */ + $stats['Deployments'] = $dbForProject->count('deployments', []); + + /** Get Total Executions */ + $stats['Executions'] = $dbForProject->count('executions', []); + + $csv->insertOne(array_values($stats)); + } catch (\Throwable $th) { + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + } + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); + $pools + ->get('console') + ->reclaim(); +// +// /** @var PHPMailer $mail */ +// $mail = $register->get('smtp'); +// +// $mail->clearAddresses(); +// $mail->clearAllRecipients(); +// $mail->clearReplyTos(); +// $mail->clearAttachments(); +// $mail->clearBCCs(); +// $mail->clearCCs(); +// +// try { +// /** Addresses */ +// $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); +// $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); +// +// foreach ($recipients as $recipient) { +// $mail->addAddress($recipient); +// } +// +// /** Attachments */ +// $mail->addAttachment($this->path); +// +// /** Content */ +// $mail->Subject = "Cloud Report for {$this->date}"; +// $mail->Body = "Please find the daily cloud report atttached"; +// $mail->send(); + // Console::success('Email has been sent!'); + // } catch (Exception $e) { + // Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + // } + } +} From da75a2e409c1bb86d62a305757bf0392a76e3d03 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 30 May 2023 18:13:30 +0300 Subject: [PATCH 2/9] calc tier stats --- src/Appwrite/Platform/Tasks/CalcTierStats.php | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 82d3e0200..1e6ec0718 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -257,36 +257,36 @@ class CalcTierStats extends Action $pools ->get('console') ->reclaim(); -// -// /** @var PHPMailer $mail */ -// $mail = $register->get('smtp'); -// -// $mail->clearAddresses(); -// $mail->clearAllRecipients(); -// $mail->clearReplyTos(); -// $mail->clearAttachments(); -// $mail->clearBCCs(); -// $mail->clearCCs(); -// -// try { -// /** Addresses */ -// $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); -// $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); -// -// foreach ($recipients as $recipient) { -// $mail->addAddress($recipient); -// } -// -// /** Attachments */ -// $mail->addAttachment($this->path); -// -// /** Content */ -// $mail->Subject = "Cloud Report for {$this->date}"; -// $mail->Body = "Please find the daily cloud report atttached"; -// $mail->send(); - // Console::success('Email has been sent!'); - // } catch (Exception $e) { - // Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); - // } + + /** @var PHPMailer $mail */ + $mail = $register->get('smtp'); + + $mail->clearAddresses(); + $mail->clearAllRecipients(); + $mail->clearReplyTos(); + $mail->clearAttachments(); + $mail->clearBCCs(); + $mail->clearCCs(); + + try { + /** Addresses */ + $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); + $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); + + foreach ($recipients as $recipient) { + $mail->addAddress($recipient); + } + + /** Attachments */ + $mail->addAttachment($this->path); + + /** Content */ + $mail->Subject = "Cloud Report for {$this->date}"; + $mail->Body = "Please find the daily cloud report atttached"; + $mail->send(); + Console::success('Email has been sent!'); + } catch (Exception $e) { + Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + } } } From 3b6128c560be4830d677228b18eddd52abacccb0 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 30 May 2023 18:17:47 +0300 Subject: [PATCH 3/9] calc tier stats --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index baf6f4d19..b0ae0248e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -121,6 +121,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/calc-users-stats && \ chmod +x /usr/local/bin/calc-tier-stats && \ + chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ From 8ce6d38922a9eae9b4f5cffd2ae659faee6e2c4c Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 30 May 2023 18:30:15 +0300 Subject: [PATCH 4/9] minor fix --- src/Appwrite/Platform/Tasks/CalcTierStats.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 1e6ec0718..dd75edd6c 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -121,7 +121,6 @@ class CalcTierStats extends Action ///** Get Project Name */ //$stats['Project Name'] = $project->getAttribute('name'); - /** Get Organization Name and Id */ //$teamId = $project->getAttribute('teamId', null); //$teamName = null; From 9ac4a2bfd5210a33987edce595e16a270597aa7d Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 30 May 2023 18:46:36 +0300 Subject: [PATCH 5/9] minor fix --- src/Appwrite/Platform/Tasks/CalcTierStats.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index dd75edd6c..28688b815 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -252,7 +252,9 @@ class CalcTierStats extends Action $offset = $offset + $limit; $count = $count + $sum; } + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); + $pools ->get('console') ->reclaim(); From 5e2e28995d8b505c748d7b6460013a921faf9702 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 30 May 2023 18:47:35 +0300 Subject: [PATCH 6/9] minor fix --- app/console | 1 - 1 file changed, 1 deletion(-) delete mode 160000 app/console diff --git a/app/console b/app/console deleted file mode 160000 index 2fac7c1f3..000000000 --- a/app/console +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2fac7c1f390637f7dfc11882dcad6fb6508c83f1 From 876b0da2e25bf1f7060c1701571ec7046a857fcb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 30 May 2023 16:27:39 +0000 Subject: [PATCH 7/9] feat: update console --- app/console | 1 + 1 file changed, 1 insertion(+) create mode 160000 app/console diff --git a/app/console b/app/console new file mode 160000 index 000000000..3a0c8f033 --- /dev/null +++ b/app/console @@ -0,0 +1 @@ +Subproject commit 3a0c8f0334d402a6e27c8b5c0512f3d60080ddbd From 0fd7db3c36fb0ac6521759d0bdf11c8f4acbd6a3 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 30 May 2023 20:11:25 +0300 Subject: [PATCH 8/9] addressing comments --- app/console | 2 +- composer.lock | 2 +- src/Appwrite/Platform/Services/Tasks.php | 2 +- src/Appwrite/Platform/Tasks/CalcTierStats.php | 25 +++++++++++++------ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/app/console b/app/console index 3a0c8f033..9174d8f8c 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 3a0c8f0334d402a6e27c8b5c0512f3d60080ddbd +Subproject commit 9174d8f8cb584744dd7a53f69d324f490ee82ee3 diff --git a/composer.lock b/composer.lock index 3d9a6c8bd..4057dc70b 100644 --- a/composer.lock +++ b/composer.lock @@ -5542,5 +5542,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index e05baee68..7e8d9a133 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -45,7 +45,7 @@ class Tasks extends Service ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()) ->addAction(CalcUsersStats::getName(), new CalcUsersStats()) - ->addAction(CalctierStats::getName(), new CalcTierStats()) + ->addAction(CalcTierStats::getName(), new CalcTierStats()) ; } } diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 28688b815..bdc46e978 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Tasks; use Exception; +use League\Csv\CannotInsertRecord; use Utopia\App; use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; @@ -20,17 +21,18 @@ class CalcTierStats extends Action private array $columns = [ 'Project ID', 'Organization ID', - 'Users', + 'Organization Members', 'Teams', 'Requests', 'Bandwidth', 'Domains', + 'Api keys', 'Webhooks', 'Platforms', 'Buckets', 'Files', - 'Storage', - 'Max file size', + 'Storage (bytes)', + 'Max File Size (bytes)', 'Databases', 'Functions', 'Deployments', @@ -66,6 +68,10 @@ class CalcTierStats extends Action }); } + /** + * @throws \Utopia\Exception + * @throws CannotInsertRecord + */ public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void { //docker compose exec -t appwrite calc-tier-stats @@ -129,12 +135,12 @@ class CalcTierStats extends Action //$teamName = $team->getAttribute('name'); // } - $stats['Organization ID'] = $project->getAttribute('teamInternalId', null); + $stats['Organization ID'] = $project->getAttribute('teamId', null); /** Get Total Members */ $teamInternalId = $project->getAttribute('teamInternalId', null); if ($teamInternalId) { - $stats['Users'] = $dbForConsole->count('memberships', [ + $stats['Organization Members'] = $dbForConsole->count('memberships', [ Query::equal('teamInternalId', [$teamInternalId]) ]); } else { @@ -190,6 +196,11 @@ class CalcTierStats extends Action ]); + /** Get Api keys */ + $stats['Api keys'] = $dbForConsole->count('keys', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + ]); + /** Get Webhooks */ $stats['Webhooks'] = $dbForConsole->count('webhooks', [ Query::equal('projectInternalId', [$project->getInternalId()]), @@ -217,8 +228,8 @@ class CalcTierStats extends Action } $stats['Buckets'] = $counter; $stats['Files'] = $filesCount; - $stats['Storage'] = $filesSum; - $stats['Max file size'] = $maxFileSize; + $stats['Storage (bytes)'] = $filesSum; + $stats['Max File Size (bytes)'] = $maxFileSize; /** Get Total Functions */ $stats['Databases'] = $dbForProject->count('databases', []); From 072232e1afe5be0c6f2a2618d518271034bf1ac1 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 30 May 2023 18:17:48 +0000 Subject: [PATCH 9/9] fix: console --- app/console | 2 +- composer.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/console b/app/console index 9174d8f8c..3a0c8f033 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 9174d8f8cb584744dd7a53f69d324f490ee82ee3 +Subproject commit 3a0c8f0334d402a6e27c8b5c0512f3d60080ddbd diff --git a/composer.lock b/composer.lock index 4057dc70b..3d9a6c8bd 100644 --- a/composer.lock +++ b/composer.lock @@ -5542,5 +5542,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" }