Merge pull request #7298 from appwrite/refactor-hamster-usage

combining network inbound

src/Appwrite/Platform/Tasks/CalcTierStats.php

@@ -5,6 +5,7 @@ namespace Appwrite\Platform\Tasks;
 use Exception;
 use League\Csv\CannotInsertRecord;
 use Utopia\App;
+use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Platform\Action;
 use Utopia\Cache\Cache;
@@ -15,6 +16,7 @@ use League\Csv\Writer;
 use PHPMailer\PHPMailer\PHPMailer;
 use Utopia\Pools\Group;
 use Utopia\Registry\Registry;
+use Utopia\Validator\Text;
 
 class CalcTierStats extends Action
 {
@@ -24,6 +26,7 @@ class CalcTierStats extends Action
     private array $columns = [
         'Project ID',
         'Organization ID',
+        'Organization Email',
         'Organization Members',
         'Teams',
         'Users',
@@ -49,8 +52,9 @@ class CalcTierStats extends Action
     protected string $date;
 
     private array $usageStats = [
-        'project.$all.network.requests'  => 'Requests',
-        'project.$all.network.bandwidth' => 'Bandwidth',
+        'network.requests'  => 'Requests',
+        'network.inbound' => 'Inbound',
+        'network.outbound' => 'Outbound',
 
     ];
 
@@ -64,78 +68,172 @@ class CalcTierStats extends Action
 
         $this
             ->desc('Get stats for projects')
+            ->param('after', '', new Text(36), 'After cursor', true)
+            ->param('projectId', '', new Text(36), 'Select project to validate', true)
             ->inject('pools')
             ->inject('cache')
             ->inject('dbForConsole')
+            ->inject('getProjectDB')
             ->inject('register')
-            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
-                $this->action($pools, $cache, $dbForConsole, $register);
+            ->callback(function ($after, $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) {
+                $this->action($after, $projectId, $pools, $cache, $dbForConsole, $getProjectDB, $register);
             });
     }
 
-    /**
-     * @throws \Utopia\Exception
-     * @throws CannotInsertRecord
-     */
-    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+
+    public function action(string $after, string $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register): void
     {
         //docker compose exec -t appwrite calc-tier-stats
 
         Console::title('Cloud free tier  stats calculation V1');
         Console::success(APP_NAME . ' cloud free tier  stats calculation has started');
 
-        /* Initialise new Utopia app */
-        $app = new App('UTC');
-        $console = $app->getResource('console');
-
         /** CSV stuff */
         $this->date = date('Y-m-d');
         $this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
         $csv = Writer::createFromPath($this->path, 'w');
         $csv->insertOne($this->columns);
 
-        /** Database connections */
-        $totalProjects = $dbForConsole->count('projects');
-        Console::success("Found a total of: {$totalProjects} projects");
+        if (!empty($projectId)) {
+            try {
+                console::log("Project " . $projectId);
+                $project = $dbForConsole->getDocument('projects', $projectId);
+                $dbForProject = call_user_func($getProjectDB, $project);
+                $data = $this->getData($project, $dbForConsole, $dbForProject);
+                $csv->insertOne($data);
+                $this->sendMail($register);
 
-        $projects = [$console];
-        $count = 0;
-        $limit = 100;
-        $sum = 100;
-        $offset = 0;
-        while (!empty($projects)) {
-            foreach ($projects as $project) {
+                return;
+            } catch (\Throwable $th) {
+                Console::error("Unexpected error occured with Project ID {$projectId}");
+                Console::error('[Error] Type: ' . get_class($th));
+                Console::error('[Error] Message: ' . $th->getMessage());
+                Console::error('[Error] File: ' . $th->getFile());
+                Console::error('[Error] Line: ' . $th->getLine());
+            }
+        }
+            $queries = [];
+        if (!empty($after)) {
+            Console::info("Iterating remaining projects after project with ID {$after}");
+            $project = $dbForConsole->getDocument('projects', $after);
+            $queries = [Query::cursorAfter($project)];
+        } else {
+            Console::info("Iterating all projects");
+        }
+            $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole, $csv) {
+                $projectId = $project->getId();
+                console::log("Project " . $projectId);
+                try {
+                    $dbForProject = call_user_func($getProjectDB, $project);
+                    $data = $this->getData($project, $dbForConsole, $dbForProject);
+                    $csv->insertOne($data);
+                } catch (\Throwable $th) {
+                    Console::error("Unexpected error occured with Project ID {$projectId}");
+                    Console::error('[Error] Type: ' . get_class($th));
+                    Console::error('[Error] Message: ' . $th->getMessage());
+                    Console::error('[Error] File: ' . $th->getFile());
+                    Console::error('[Error] Line: ' . $th->getLine());
+                }
+            });
 
-                /**
-                 * Skip user projects with id 'console'
-                 */
-                if ($project->getId() === 'console') {
-                    continue;
+        $this->sendMail($register);
     }
 
-                Console::info("Getting stats for {$project->getId()}");
+    private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
+    {
+        $limit = 1000;
+        $results = [];
+        $sum = $limit;
+        $latestDocument = null;
+
+        while ($sum === $limit) {
+            $newQueries = $queries;
+
+            if ($latestDocument != null) {
+                array_unshift($newQueries, Query::cursorAfter($latestDocument));
+            }
+            $newQueries[] = Query::limit($limit);
+            $results = $database->find('projects', $newQueries);
+
+            if (empty($results)) {
+                return;
+            }
+
+            $sum = count($results);
+
+            foreach ($results as $document) {
+                if (is_callable($callback)) {
+                    $callback($document);
+                }
+            }
+            $latestDocument = $results[array_key_last($results)];
+        }
+    }
+
+    private function sendMail(Registry $register): void
+    {
+        /** @var PHPMailer $mail */
+        $mail = $register->get('smtp');
+        $mail->clearAddresses();
+        $mail->clearAllRecipients();
+        $mail->clearReplyTos();
+        $mail->clearAttachments();
+        $mail->clearBCCs();
+        $mail->clearCCs();
 
         try {
-                    $db = $project->getAttribute('database');
-                    $adapter = $pools
-                        ->get($db)
-                        ->pop()
-                        ->getResource();
+            /** Addresses */
+            $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
+            $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
+            foreach ($recipients as $recipient) {
+                $mail->addAddress($recipient);
+            }
 
-                    $dbForProject = new Database($adapter, $cache);
-                    $dbForProject->setDefaultDatabase('appwrite');
-                    $dbForProject->setNamespace('_' . $project->getInternalId());
+            /** Attachments */
+            $mail->addAttachment($this->path);
+
+            /** Content */
+            $mail->Subject = "Cloud Report for {$this->date}";
+            $mail->Body = "Please find the daily cloud report atttached";
+            $mail->send();
+            Console::success('Email has been sent!');
+        } catch (Exception $e) {
+            Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
+        }
+    }
+
+
+    private function getData(Document $project, Database $dbForConsole, Database $dbForProject): array
+    {
 
-                    /** Get Project ID */
             $stats['Project ID'] = $project->getId();
-
             $stats['Organization ID']   = $project->getAttribute('teamId', null);
 
+            $teamInternalId = $project->getAttribute('teamInternalId', 0);
+
+        if ($teamInternalId) {
+            $membership = $dbForConsole->findOne('memberships', [
+                Query::equal('teamInternalId', [$teamInternalId]),
+            ]);
+
+            if (!$membership || $membership->isEmpty()) {
+                Console::error('Membership not found. Skipping project : ' . $project->getId());
+            }
+
+            $userId = $membership->getAttribute('userId', null);
+            if ($userId) {
+                $user = $dbForConsole->getDocument('users', $userId);
+                $stats['Organization Email'] = $user->getAttribute('email', null);
+            }
+        } else {
+            Console::error("Email was not found for this Organization ID :{$teamInternalId}");
+        }
+
             /** Get Total Members */
             $teamInternalId = $project->getAttribute('teamInternalId', null);
         if ($teamInternalId) {
             $stats['Organization Members'] = $dbForConsole->count('memberships', [
-                            Query::equal('teamInternalId', [$teamInternalId])
+                Query::equal('$internalId', [(string)$teamInternalId])
             ]);
         } else {
             $stats['Organization Members'] = 0;
@@ -199,6 +297,13 @@ class CalcTierStats extends Action
             $stats[$metrics[$key]]  = $value;
         }
 
+            /**
+             * Workaround to combine network.inbound+network.outbound as network.
+             */
+            $stats['Network'] = ($stats['Inbound'] ?? 0) + ($stats['Outbound'] ?? 0);
+            unset($stats['Inbound']);
+            unset($stats['Outbound']);
+
         try {
             /** Get Domains */
             $stats['Domains'] = $dbForConsole->count('rules', [
@@ -254,8 +359,8 @@ class CalcTierStats extends Action
                 }
                 $counter++;
             }
-                    } catch (\Throwable) {
-                        ;
+        } catch (\Throwable $t) {
+            Console::error("Error while counting buckets: {$project->getId()}");
         }
             $stats['Buckets'] = $counter;
             $stats['Files'] = $filesCount;
@@ -298,62 +403,6 @@ class CalcTierStats extends Action
             $stats['Migrations'] = 0;
         }
 
-                    $csv->insertOne(array_values($stats));
-                } catch (\Throwable $th) {
-                    Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . '  line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
-                } finally {
-                    $pools
-                        ->get($db)
-                        ->reclaim();
-                }
-            }
-
-            $sum = \count($projects);
-
-            $projects = $dbForConsole->find('projects', [
-                Query::limit($limit),
-                Query::offset($offset),
-            ]);
-
-            $offset = $offset + $limit;
-            $count = $count + $sum;
-        }
-
-        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
-
-        $pools
-            ->get('console')
-            ->reclaim();
-
-        /** @var PHPMailer $mail */
-        $mail = $register->get('smtp');
-
-        $mail->clearAddresses();
-        $mail->clearAllRecipients();
-        $mail->clearReplyTos();
-        $mail->clearAttachments();
-        $mail->clearBCCs();
-        $mail->clearCCs();
-
-        try {
-            /** Addresses */
-            $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
-            $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
-
-            foreach ($recipients as $recipient) {
-                $mail->addAddress($recipient);
-            }
-
-            /** Attachments */
-            $mail->addAttachment($this->path);
-
-            /** Content */
-            $mail->Subject = "Cloud Report for {$this->date}";
-            $mail->Body = "Please find the daily cloud report atttached";
-            $mail->send();
-            Console::success('Email has been sent!');
-        } catch (Exception $e) {
-            Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
-        }
+        return array_values($stats);
     }
 }

src/Appwrite/Platform/Tasks/Hamster.php

@@ -20,17 +20,18 @@ use Utopia\Pools\Group;
 class Hamster extends Action
 {
     private array $metrics = [
-        'usage_files' => 'files.$all.count.total',
-        'usage_buckets' => 'buckets.$all.count.total',
-        'usage_databases' => 'databases.$all.count.total',
-        'usage_documents' => 'documents.$all.count.total',
-        'usage_collections' => 'collections.$all.count.total',
-        'usage_storage' => 'project.$all.storage.size',
-        'usage_requests' => 'project.$all.network.requests',
-        'usage_bandwidth' => 'project.$all.network.bandwidth',
-        'usage_users' => 'users.$all.count.total',
-        'usage_sessions' => 'sessions.email.requests.create',
-        'usage_executions' => 'executions.$all.compute.total',
+        'usage_files' => 'files',
+        'usage_buckets' => 'buckets',
+        'usage_databases' => 'databases',
+        'usage_documents' => 'documents',
+        'usage_collections' => 'collections',
+        'usage_storage' => 'files.storage',
+        'usage_requests' => 'network.requests',
+        'usage_inbound' => 'network.inbound',
+        'usage_outbound' => 'network.outbound',
+        'usage_users' => 'users',
+        'usage_sessions' => 'sessions',
+        'usage_executions' => 'executions',
     ];
 
     protected string $directory = '/usr/local';
@@ -261,6 +262,16 @@ class Hamster extends Action
                     }
                 });
 
+                /**
+                 * Workaround to combine network.inbound+network.outbound as network.
+                 */
+                $statsPerProject["usage_network_infinity"]  = $statsPerProject["usage_inbound_infinity"] + $statsPerProject["usage_outbound_infinity"];
+                $statsPerProject["usage_network_24h"]  = $statsPerProject["usage_inbound_24h"] + $statsPerProject["usage_outbound_24h"];
+                unset($statsPerProject["usage_outbound_24h"]);
+                unset($statsPerProject["usage_inbound_24h"]);
+                unset($statsPerProject["usage_outbound_infinity"]);
+                unset($statsPerProject["usage_inbound_infinity"]);
+
                 if (isset($statsPerProject['email'])) {
                     /** Send data to mixpanel */
                     $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
@@ -300,7 +311,6 @@ class Hamster extends Action
         Console::success(APP_NAME . ' cloud hamster process has started');
 
         $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
-
         $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
         $now = new \DateTime();
         $now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
@@ -315,7 +325,6 @@ class Hamster extends Action
             $next->add(\DateInterval::createFromDateString('1 days'));
             $delay = $next->getTimestamp() - $now->getTimestamp();
         }
-
         Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
 
         Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) {