feat: add new stats

.env

@@ -46,8 +46,8 @@ _APP_SMTP_SECURE=
 _APP_SMTP_USERNAME=
 _APP_SMTP_PASSWORD=
 _APP_HAMSTER_INTERVAL=86400
-_APP_HAMSTER_TIME=21:00
+_APP_HAMSTER_TIME=12:31
-_APP_MIXPANEL_TOKEN=
+_APP_MIXPANEL_TOKEN=bce512333a58ec62f44541328607f53c
 _APP_SMS_PROVIDER=sms://username:password@mock
 _APP_SMS_FROM=+123456789
 _APP_STORAGE_LIMIT=30000000

composer.json

@@ -44,7 +44,7 @@
         "appwrite/php-clamav": "1.1.*",
         "appwrite/php-runtimes": "0.11.*",
         "utopia-php/abuse": "0.16.*",
-        "utopia-php/analytics": "0.10.1",
+        "utopia-php/analytics": "0.10.2",
         "utopia-php/audit": "0.17.*",
         "utopia-php/cache": "0.8.*",
         "utopia-php/cli": "0.15.*",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "f409ee69f8040b1928cbc618ff3e8a43",
+    "content-hash": "ca2a083ff1c0d0c77942674400137793",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -1583,16 +1583,16 @@
         },
         {
             "name": "utopia-php/analytics",
-            "version": "0.10.1",
+            "version": "0.10.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/analytics.git",
-                "reference": "70ada5e6b192ae27e6d5467899a4cdd886003835"
+                "reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/analytics/zipball/70ada5e6b192ae27e6d5467899a4cdd886003835",
+                "url": "https://api.github.com/repos/utopia-php/analytics/zipball/14c805114736f44c26d6d24b176a2f8b93d86a1f",
-                "reference": "70ada5e6b192ae27e6d5467899a4cdd886003835",
+                "reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f",
                 "shasum": ""
             },
             "require": {
@@ -1623,9 +1623,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/analytics/issues",
-                "source": "https://github.com/utopia-php/analytics/tree/0.10.1"
+                "source": "https://github.com/utopia-php/analytics/tree/0.10.2"
             },
-            "time": "2023-03-17T14:42:35+00:00"
+            "time": "2023-03-22T12:01:09+00:00"
         },
         {
             "name": "utopia-php/audit",

src/Appwrite/Platform/Tasks/Hamster.php

@@ -3,6 +3,7 @@
 namespace Appwrite\Platform\Tasks;
 
 use Exception;
+use finfo;
 use Utopia\App;
 use Utopia\Platform\Action;
 use Utopia\Cache\Cache;
@@ -14,22 +15,23 @@ use Utopia\Analytics\Adapter\Mixpanel;
 use Utopia\Analytics\Event;
 use Utopia\Database\Document;
 use Utopia\Pools\Group;
+use Utopia\Pools\Pool;
 use Utopia\Registry\Registry;
 
 class Hamster extends Action
 {
-    private array $usageStats = [
+    private array $metrics = [
-        'files' => 'files.$all.count.total',
+        'usage_files' => 'files.$all.count.total',
-        'buckets' => 'buckets.$all.count.total',
+        'usage_buckets' => 'buckets.$all.count.total',
-        'databases' => 'databases.$all.count.total',
+        'usage_databases' => 'databases.$all.count.total',
-        'documents' => 'documents.$all.count.total',
+        'usage_documents' => 'documents.$all.count.total',
-        'collections' => 'collections.$all.count.total',
+        'usage_collections' => 'collections.$all.count.total',
-        'storage' => 'project.$all.storage.size',
+        'usage_storage' => 'project.$all.storage.size',
-        'requests' => 'project.$all.network.requests',
+        'usage_requests' => 'project.$all.network.requests',
-        'bandwidth' => 'project.$all.network.bandwidth',
+        'usage_bandwidth' => 'project.$all.network.bandwidth',
-        'users' => 'users.$all.count.total',
+        'usage_users' => 'users.$all.count.total',
-        'sessions' => 'sessions.email.requests.create',
+        'usage_sessions' => 'sessions.email.requests.create',
-        'executions' => 'executions.$all.compute.total',
+        'usage_executions' => 'executions.$all.compute.total',
     ];
 
     protected string $directory = '/usr/local';
@@ -51,41 +53,68 @@ class Hamster extends Action
 
         $this
             ->desc('Get stats for projects')
-            ->inject('register')
             ->inject('pools')
             ->inject('cache')
             ->inject('dbForConsole')
-            ->callback(function (Registry $register, Group $pools, Cache $cache, Database $dbForConsole) {
+            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole) {
-                $this->action($register, $pools, $cache, $dbForConsole);
+                $this->action($pools, $cache, $dbForConsole);
             });
     }
 
-    private function getStats(Database $dbForConsole, Database $dbForProject, Document $project): array
+    private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole)
     {
-        $stats = [];
+        $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache){
+            /**
+             * Skip user projects with id 'console'
+             */
+            if ($project->getId() === 'console') {
+                Console::info("Skipping project console");
+                return;
+            }
 
-        $stats['time'] = microtime(true);
+            Console::log("Getting stats for {$project->getId()}");
+
+            try {
+                $db = $project->getAttribute('database');
+                $adapter = $pools
+                    ->get($db)
+                    ->pop()
+                    ->getResource();
+
+                $dbForProject = new Database($adapter, $cache);
+                $dbForProject->setDefaultDatabase('appwrite');
+                $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                $statsPerProject = [];
+
+                $statsPerProject['time'] = microtime(true);
 
                 /** Get Project ID */
-        $stats['projectId'] = $project->getId();
+                $statsPerProject['project_id'] = $project->getId();
+
+                /** Get project created time */
+                $statsPerProject['project_created'] = $project->getAttribute('$createdAt');
 
                 /** Get Project Name */
-        $stats['projectName'] = $project->getAttribute('name');
+                $statsPerProject['project_name'] = $project->getAttribute('name');
 
                 /** Get Total Functions */
-        $stats['functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
+                $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
 
                 /** Get Total Deployments */
-        $stats['deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
+                $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
+
+                /** Get Total Teams */
+                $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
 
                 /** Get Total Members */
                 $teamInternalId = $project->getAttribute('teamInternalId', null);
                 if ($teamInternalId) {
-            $stats['members'] = $dbForConsole->count('memberships', [
+                    $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [
                         Query::equal('teamInternalId', [$teamInternalId])
                     ], APP_LIMIT_COUNT);
                 } else {
-            $stats['members'] = 0;
+                    $statsPerProject['custom_organization_members'] = 0;
                 }
 
                 /** Get Email and Name of the project owner */
@@ -104,13 +133,13 @@ class Hamster extends Action
                             Query::equal('_id', [$userInternalId]),
                         ]);
 
-                $stats['email'] = $user->getAttribute('email', null);
+                        $statsPerProject['email'] = $user->getAttribute('email', null);
-                $stats['name'] = $user->getAttribute('name', null);
+                        $statsPerProject['name'] = $user->getAttribute('name', null);
                     }
                 }
 
                 /** Get Domains */
-        $stats['domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT);
+                $statsPerProject['custom_domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT);
 
                 /** Get Platforms */
                 $platforms = $dbForConsole->find('platforms', [
@@ -118,36 +147,39 @@ class Hamster extends Action
                     Query::limit(APP_LIMIT_COUNT)
                 ]);
 
-        $stats['platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
+                $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
                     return $platform['type'] === 'web';
                 }));
 
-        $stats['platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
+                $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
                     return $platform['type'] === 'android';
                 }));
 
-        $stats['platforms_iOS'] = sizeof(array_filter($platforms, function ($platform) {
+                $statsPerProject['custom_platforms_iOS'] = sizeof(array_filter($platforms, function ($platform) {
                     return str_contains($platform['type'], 'apple');
                 }));
 
-        $stats['platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
+                $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
                     return str_contains($platform['type'], 'flutter');
                 }));
 
-        /** Get Usage stats */
+                /** Get Usage $statsPerProject */
-        $range = '90d';
                 $periods = [
-            '90d' => [
+                    'infinity' => [
                         'period' => '1d',
                         'limit' => 90,
                     ],
+                    '24h' => [
+                        'period' => '1h',
+                        'limit' => 24,
+                    ],
                 ];
 
-        $metrics = array_values($this->usageStats);
+                Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) {
-        Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
+                    foreach ($this->metrics as $key => $metric) {
-            foreach ($metrics as $metric) {
+                        foreach ($periods as $periodKey => $periodValue) {
-                $limit = $periods[$range]['limit'];
+                            $limit = $periodValue['limit'];
-                $period = $periods[$range]['period'];
+                            $period = $periodValue['period'];
 
                             $requestDocs = $dbForProject->find('stats', [
                                 Query::equal('period', [$period]),
@@ -156,24 +188,53 @@ class Hamster extends Action
                                 Query::orderDesc('time'),
                             ]);
 
-                $stats[$metric] = [];
+                            $statsPerProject[$key . '_' . $periodKey] = [];
                             foreach ($requestDocs as $requestDoc) {
-                    $stats[$metric][] = [
+                                $statsPerProject[$key . '_' . $periodKey][] = [
                                     'value' => $requestDoc->getAttribute('value'),
                                     'date' => $requestDoc->getAttribute('time'),
                                 ];
                             }
 
-                $stats[$metric] = array_reverse($stats[$metric]);
+                            $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]);
                             // Calculate aggregate of each metric
-                $stats[$metric] = array_sum(array_column($stats[$metric], 'value'));
+                            $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value'));
+                        }
                     }
                 });
 
-        return $stats;
+                if (isset($statsPerProject['email'])) {
+                    /** Send data to mixpanel */
+                    $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
+                        'name' => $statsPerProject['name'],
+                        'email' => $statsPerProject['email']
+                    ]);
+
+                    if (!$res) {
+                        Console::error('Failed to create user profile for project: ' . $project->getId());
                     }
 
-    public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void
+                    $event = new Event();
+                    $event
+                        ->setName('Project Daily Usage')
+                        ->setProps($statsPerProject);
+                    $res = $this->mixpanel->createEvent($event);
+                    if (!$res) {
+                        Console::error('Failed to create event for project: ' . $project->getId());
+                    }
+                }
+            } catch (Exception $e) {
+                Console::error('Failed to send stats for project: ' . $project->getId());
+                Console::error($e->getMessage());
+            } finally {
+                $pools
+                    ->get($db)
+                    ->reclaim();
+            }
+        });
+    }
+
+    public function action(Group $pools, Cache $cache, Database $dbForConsole): void
     {
 
         Console::title('Cloud Hamster V1');
@@ -181,7 +242,7 @@ class Hamster extends Action
 
         $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
 
-        $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00');// (hour:minutes)
+        $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
         $now = new \DateTime();
         $now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
         $next = new \DateTime($now->format("Y-m-d $jobInitTime"));
@@ -198,90 +259,25 @@ class Hamster extends Action
 
         Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
 
-        Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $sleep) {
+        Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) {
             $now = date('d-m-Y H:i:s', time());
             Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds");
             $loopStart = microtime(true);
 
             /* Initialise new Utopia app */
             $app = new App('UTC');
-            $console = $app->getResource('console');
 
-            /** Database connections */
+            Console::info('Getting stats for all projects');
-            $totalProjects = $dbForConsole->count('projects') + 1;
+            $this->getStatsPerProject($pools, $cache, $dbForConsole);
-            Console::success("Found a total of: {$totalProjects} projects");
+            Console::success('Completed getting stats for all projects');
 
-            $projects = [$console];
+            Console::info('Getting stats for all organizations');
-            $count = 0;
+            $this->getStatsPerOrganization($dbForConsole);
-            $limit = 30;
+            Console::success('Completed getting stats for all organizations');
-            $sum = 30;
-            $offset = 0;
-            while (!empty($projects)) {
-                foreach ($projects as $project) {
-                    /**
-                     * Skip user projects with id 'console'
-                     */
-                    if ($project->getId() === 'console') {
-                        continue;
-                    }
-
-                    Console::info("Getting stats for {$project->getId()}");
-
-                    try {
-                        $db = $project->getAttribute('database');
-                        $adapter = $pools
-                            ->get($db)
-                            ->pop()
-                            ->getResource();
-
-                        $dbForProject = new Database($adapter, $cache);
-                        $dbForProject->setDefaultDatabase('appwrite');
-                        $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                        $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project);
-
-                        if (isset($statsPerProject['email'])) {
-                            /** Send data to mixpanel */
-                            $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
-                                'name' => $statsPerProject['name'],
-                                'email' => $statsPerProject['email']
-                            ]);
-
-                            if (!$res) {
-                                Console::error('Failed to create user profile for project: ' . $project->getId());
-                            }
-
-                            $event = new Event();
-                            $event
-                                ->setName('Appwrite Cloud Project Stats')
-                                ->setProps($statsPerProject);
-                            $res = $this->mixpanel->createEvent($event);
-                            if (!$res) {
-                                Console::error('Failed to create event for project: ' . $project->getId());
-                            }
-                        }
-                    } catch (\Throwable $th) {
-                        Console::error('Failed to get stats for project ("' . $project->getId() . '") with error: ' . $th->getMessage());
-                    } finally {
-                        $pools
-                            ->get($db)
-                            ->reclaim();
-                    }
-                }
-
-                $sum = \count($projects);
-
-                $projects = $dbForConsole->find('projects', [
-                    Query::limit($limit),
-                    Query::offset($offset),
-                ]);
-
-                $offset = $offset + $limit;
-                $count = $count + $sum;
-
-                Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...');
-            }
 
+            Console::info('Getting stats for all users');
+            $this->getStatsPerUser($dbForConsole);
+            Console::success('Completed getting stats for all users');
 
             $pools
                 ->get('console')
@@ -292,4 +288,125 @@ class Hamster extends Action
             Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
         }, $sleep, $delay);
     }
+
+    protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback)
+    {
+        $count = 0;
+        $chunk = 0;
+        $limit = 50;
+        $results = [];
+        $sum = $limit;
+
+        $executionStart = \microtime(true);
+
+        while ($sum === $limit) {
+            $chunk++;
+
+            $results = $dbForConsole->find($collection, \array_merge([Query::limit($limit)]));
+
+            $sum = count($results);
+
+            Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents');
+
+            foreach ($results as $document) {
+                call_user_func($callback, $dbForConsole, $document);
+                $count++;
+            }
+        }
+
+        $executionEnd = \microtime(true);
+
+        Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
+    }
+
+    protected function getStatsPerOrganization(Database $dbForConsole)
+    {
+
+        $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) {
+            $statsPerOrganization = [];
+
+            /** Organization name */
+            $statsPerOrganization['name'] = $document->getAttribute('name');
+
+            /** Get Email and of the organization owner */
+            $membership = $dbForConsole->findOne('memberships', [
+                Query::equal('teamInternalId', [$document->getInternalId()]),
+            ]);
+
+            if (!$membership || $membership->isEmpty()) {
+                throw new Exception('Membership not found. Skipping organization : ' . $document->getId());
+            }
+
+            $userInternalId = $membership->getAttribute('userInternalId', null);
+            if ($userInternalId) {
+                $user = $dbForConsole->findOne('users', [
+                    Query::equal('_id', [$userInternalId]),
+                ]);
+
+                $statsPerOrganization['email'] = $user->getAttribute('email', null);
+            }
+
+            /** Organization Creation Date */
+            $statsPerOrganization['created'] = $document->getAttribute('$createdAt');
+
+            /** Number of team members */
+            $statsPerOrganization['members'] = $document->getAttribute('total');
+
+            /** Number of projects in this organization */
+            $statsPerOrganization['projects'] = $dbForConsole->count('projects', [
+                Query::equal('teamId', [$document->getId()]),
+                Query::limit(APP_LIMIT_COUNT)
+            ]);
+
+            if (!isset($statsPerOrganization['email'])) {
+                throw new Exception('Email not found. Skipping organization : ' . $document->getId());
+            }
+
+            $event = new Event();
+            $event
+                ->setName('Organization Daily Usage')
+                ->setProps($statsPerOrganization);
+            $res = $this->mixpanel->createEvent($event);
+            if (!$res) {
+                throw new Exception('Failed to create event for organization : ' . $document->getId());
+            }
+        });
+    }
+
+    protected function getStatsPerUser(Database $dbForConsole)
+    {
+        $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) {
+            $statsPerUser = [];
+
+            /** Organization name */
+            $statsPerUser['name'] = $document->getAttribute('name');
+
+            /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */
+            $statsPerUser['email'] = $document->getAttribute('email');
+
+            /** Organization Creation Date */
+            $statsPerUser['created'] = $document->getAttribute('$createdAt');
+
+            /** Number of teams this user is a part of */
+            $statsPerUser['memberships'] = $dbForConsole->count('memberships', [
+                Query::equal('userInternalId', [$document->getInternalId()]),
+                Query::limit(APP_LIMIT_COUNT)
+            ]);
+
+            if (!isset($statsPerUser['email'])) {
+                throw new Exception('User has no email: ' . $document->getId());
+            }
+
+            /** Send data to mixpanel */
+            $event = new Event();
+            $event
+                ->setName('User Daily Usage')
+                ->setProps($statsPerUser);
+            $res = $this->mixpanel->createEvent($event);
+
+            if (!$res) {
+                throw new Exception('Failed to create user profile for user: ' . $document->getId());
+            }
+        });
+    }
 }