Merge pull request #5264 from appwrite/upgrade-hamster-script

feat: add new stats

composer.json

@@ -44,7 +44,7 @@
         "appwrite/php-clamav": "1.1.*",
         "appwrite/php-runtimes": "0.11.*",
         "utopia-php/abuse": "0.16.*",
-        "utopia-php/analytics": "0.10.1",
+        "utopia-php/analytics": "0.10.2",
         "utopia-php/audit": "0.17.*",
         "utopia-php/cache": "0.8.*",
         "utopia-php/cli": "0.15.*",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "f409ee69f8040b1928cbc618ff3e8a43",
+    "content-hash": "ca2a083ff1c0d0c77942674400137793",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -1583,16 +1583,16 @@
         },
         {
             "name": "utopia-php/analytics",
-            "version": "0.10.1",
+            "version": "0.10.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/analytics.git",
-                "reference": "70ada5e6b192ae27e6d5467899a4cdd886003835"
+                "reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/analytics/zipball/70ada5e6b192ae27e6d5467899a4cdd886003835",
-                "reference": "70ada5e6b192ae27e6d5467899a4cdd886003835",
+                "url": "https://api.github.com/repos/utopia-php/analytics/zipball/14c805114736f44c26d6d24b176a2f8b93d86a1f",
+                "reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f",
                 "shasum": ""
             },
             "require": {
@@ -1623,9 +1623,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/analytics/issues",
-                "source": "https://github.com/utopia-php/analytics/tree/0.10.1"
+                "source": "https://github.com/utopia-php/analytics/tree/0.10.2"
             },
-            "time": "2023-03-17T14:42:35+00:00"
+            "time": "2023-03-22T12:01:09+00:00"
         },
         {
             "name": "utopia-php/audit",

src/Appwrite/Platform/Tasks/Hamster.php

@@ -14,22 +14,23 @@ use Utopia\Analytics\Adapter\Mixpanel;
 use Utopia\Analytics\Event;
 use Utopia\Database\Document;
 use Utopia\Pools\Group;
+use Utopia\Pools\Pool;
 use Utopia\Registry\Registry;
 
 class Hamster extends Action
 {
-    private array $usageStats = [
-        'files' => 'files.$all.count.total',
-        'buckets' => 'buckets.$all.count.total',
-        'databases' => 'databases.$all.count.total',
-        'documents' => 'documents.$all.count.total',
-        'collections' => 'collections.$all.count.total',
-        'storage' => 'project.$all.storage.size',
-        'requests' => 'project.$all.network.requests',
-        'bandwidth' => 'project.$all.network.bandwidth',
-        'users' => 'users.$all.count.total',
-        'sessions' => 'sessions.email.requests.create',
-        'executions' => 'executions.$all.compute.total',
+    private array $metrics = [
+        'usage_files' => 'files.$all.count.total',
+        'usage_buckets' => 'buckets.$all.count.total',
+        'usage_databases' => 'databases.$all.count.total',
+        'usage_documents' => 'documents.$all.count.total',
+        'usage_collections' => 'collections.$all.count.total',
+        'usage_storage' => 'project.$all.storage.size',
+        'usage_requests' => 'project.$all.network.requests',
+        'usage_bandwidth' => 'project.$all.network.bandwidth',
+        'usage_users' => 'users.$all.count.total',
+        'usage_sessions' => 'sessions.email.requests.create',
+        'usage_executions' => 'executions.$all.compute.total',
     ];
 
     protected string $directory = '/usr/local';
@@ -51,129 +52,188 @@ class Hamster extends Action
 
         $this
             ->desc('Get stats for projects')
-            ->inject('register')
             ->inject('pools')
             ->inject('cache')
             ->inject('dbForConsole')
-            ->callback(function (Registry $register, Group $pools, Cache $cache, Database $dbForConsole) {
-                $this->action($register, $pools, $cache, $dbForConsole);
+            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole) {
+                $this->action($pools, $cache, $dbForConsole);
             });
     }
 
-    private function getStats(Database $dbForConsole, Database $dbForProject, Document $project): array
+    private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole)
     {
-        $stats = [];
-
-        $stats['time'] = microtime(true);
-
-        /** Get Project ID */
-        $stats['projectId'] = $project->getId();
-
-        /** Get Project Name */
-        $stats['projectName'] = $project->getAttribute('name');
-
-        /** Get Total Functions */
-        $stats['functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
-
-        /** Get Total Deployments */
-        $stats['deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
-
-        /** Get Total Members */
-        $teamInternalId = $project->getAttribute('teamInternalId', null);
-        if ($teamInternalId) {
-            $stats['members'] = $dbForConsole->count('memberships', [
-                Query::equal('teamInternalId', [$teamInternalId])
-            ], APP_LIMIT_COUNT);
-        } else {
-            $stats['members'] = 0;
-        }
-
-        /** Get Email and Name of the project owner */
-        if ($teamInternalId) {
-            $membership = $dbForConsole->findOne('memberships', [
-                Query::equal('teamInternalId', [$teamInternalId]),
-            ]);
-
-            if (!$membership || $membership->isEmpty()) {
-                throw new Exception('Membership not found. Skipping project : ' . $project->getId());
+        $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) {
+            /**
+             * Skip user projects with id 'console'
+             */
+            if ($project->getId() === 'console') {
+                Console::info("Skipping project console");
+                return;
             }
 
-            $userInternalId = $membership->getAttribute('userInternalId', null);
-            if ($userInternalId) {
-                $user = $dbForConsole->findOne('users', [
-                    Query::equal('_id', [$userInternalId]),
-                ]);
+            Console::log("Getting stats for {$project->getId()}");
 
-                $stats['email'] = $user->getAttribute('email', null);
-                $stats['name'] = $user->getAttribute('name', null);
-            }
-        }
+            try {
+                $db = $project->getAttribute('database');
+                $adapter = $pools
+                    ->get($db)
+                    ->pop()
+                    ->getResource();
 
-        /** Get Domains */
-        $stats['domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT);
+                $dbForProject = new Database($adapter, $cache);
+                $dbForProject->setDefaultDatabase('appwrite');
+                $dbForProject->setNamespace('_' . $project->getInternalId());
 
-        /** Get Platforms */
-        $platforms = $dbForConsole->find('platforms', [
-            Query::equal('projectInternalId', [$project->getInternalId()]),
-            Query::limit(APP_LIMIT_COUNT)
-        ]);
+                $statsPerProject = [];
 
-        $stats['platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
-            return $platform['type'] === 'web';
-        }));
+                $statsPerProject['time'] = microtime(true);
 
-        $stats['platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
-            return $platform['type'] === 'android';
-        }));
+                /** Get Project ID */
+                $statsPerProject['project_id'] = $project->getId();
 
-        $stats['platforms_iOS'] = sizeof(array_filter($platforms, function ($platform) {
-            return str_contains($platform['type'], 'apple');
-        }));
+                /** Get project created time */
+                $statsPerProject['project_created'] = $project->getAttribute('$createdAt');
 
-        $stats['platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
-            return str_contains($platform['type'], 'flutter');
-        }));
+                /** Get Project Name */
+                $statsPerProject['project_name'] = $project->getAttribute('name');
 
-        /** Get Usage stats */
-        $range = '90d';
-        $periods = [
-            '90d' => [
-                'period' => '1d',
-                'limit' => 90,
-            ],
-        ];
+                /** Get Total Functions */
+                $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
 
-        $metrics = array_values($this->usageStats);
-        Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-            foreach ($metrics as $metric) {
-                $limit = $periods[$range]['limit'];
-                $period = $periods[$range]['period'];
+                /** Get Total Deployments */
+                $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
 
-                $requestDocs = $dbForProject->find('stats', [
-                    Query::equal('period', [$period]),
-                    Query::equal('metric', [$metric]),
-                    Query::limit($limit),
-                    Query::orderDesc('time'),
-                ]);
+                /** Get Total Teams */
+                $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
 
-                $stats[$metric] = [];
-                foreach ($requestDocs as $requestDoc) {
-                    $stats[$metric][] = [
-                        'value' => $requestDoc->getAttribute('value'),
-                        'date' => $requestDoc->getAttribute('time'),
-                    ];
+                /** Get Total Members */
+                $teamInternalId = $project->getAttribute('teamInternalId', null);
+                if ($teamInternalId) {
+                    $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [
+                        Query::equal('teamInternalId', [$teamInternalId])
+                    ], APP_LIMIT_COUNT);
+                } else {
+                    $statsPerProject['custom_organization_members'] = 0;
                 }
 
-                $stats[$metric] = array_reverse($stats[$metric]);
-                // Calculate aggregate of each metric
-                $stats[$metric] = array_sum(array_column($stats[$metric], 'value'));
+                /** Get Email and Name of the project owner */
+                if ($teamInternalId) {
+                    $membership = $dbForConsole->findOne('memberships', [
+                        Query::equal('teamInternalId', [$teamInternalId]),
+                    ]);
+
+                    if (!$membership || $membership->isEmpty()) {
+                        throw new Exception('Membership not found. Skipping project : ' . $project->getId());
+                    }
+
+                    $userInternalId = $membership->getAttribute('userInternalId', null);
+                    if ($userInternalId) {
+                        $user = $dbForConsole->findOne('users', [
+                            Query::equal('_id', [$userInternalId]),
+                        ]);
+
+                        $statsPerProject['email'] = $user->getAttribute('email', null);
+                        $statsPerProject['name'] = $user->getAttribute('name', null);
+                    }
+                }
+
+                /** Get Domains */
+                $statsPerProject['custom_domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT);
+
+                /** Get Platforms */
+                $platforms = $dbForConsole->find('platforms', [
+                    Query::equal('projectInternalId', [$project->getInternalId()]),
+                    Query::limit(APP_LIMIT_COUNT)
+                ]);
+
+                $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
+                    return $platform['type'] === 'web';
+                }));
+
+                $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
+                    return $platform['type'] === 'android';
+                }));
+
+                $statsPerProject['custom_platforms_iOS'] = sizeof(array_filter($platforms, function ($platform) {
+                    return str_contains($platform['type'], 'apple');
+                }));
+
+                $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
+                    return str_contains($platform['type'], 'flutter');
+                }));
+
+                /** Get Usage $statsPerProject */
+                $periods = [
+                    'infinity' => [
+                        'period' => '1d',
+                        'limit' => 90,
+                    ],
+                    '24h' => [
+                        'period' => '1h',
+                        'limit' => 24,
+                    ],
+                ];
+
+                Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) {
+                    foreach ($this->metrics as $key => $metric) {
+                        foreach ($periods as $periodKey => $periodValue) {
+                            $limit = $periodValue['limit'];
+                            $period = $periodValue['period'];
+
+                            $requestDocs = $dbForProject->find('stats', [
+                                Query::equal('period', [$period]),
+                                Query::equal('metric', [$metric]),
+                                Query::limit($limit),
+                                Query::orderDesc('time'),
+                            ]);
+
+                            $statsPerProject[$key . '_' . $periodKey] = [];
+                            foreach ($requestDocs as $requestDoc) {
+                                $statsPerProject[$key . '_' . $periodKey][] = [
+                                    'value' => $requestDoc->getAttribute('value'),
+                                    'date' => $requestDoc->getAttribute('time'),
+                                ];
+                            }
+
+                            $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]);
+                            // Calculate aggregate of each metric
+                            $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value'));
+                        }
+                    }
+                });
+
+                if (isset($statsPerProject['email'])) {
+                    /** Send data to mixpanel */
+                    $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
+                        'name' => $statsPerProject['name'],
+                        'email' => $statsPerProject['email']
+                    ]);
+
+                    if (!$res) {
+                        Console::error('Failed to create user profile for project: ' . $project->getId());
+                    }
+
+                    $event = new Event();
+                    $event
+                        ->setName('Project Daily Usage')
+                        ->setProps($statsPerProject);
+                    $res = $this->mixpanel->createEvent($event);
+                    if (!$res) {
+                        Console::error('Failed to create event for project: ' . $project->getId());
+                    }
+                }
+            } catch (Exception $e) {
+                Console::error('Failed to send stats for project: ' . $project->getId());
+                Console::error($e->getMessage());
+            } finally {
+                $pools
+                    ->get($db)
+                    ->reclaim();
             }
         });
-
-        return $stats;
     }
 
-    public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void
+    public function action(Group $pools, Cache $cache, Database $dbForConsole): void
     {
 
         Console::title('Cloud Hamster V1');
@@ -181,7 +241,7 @@ class Hamster extends Action
 
         $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
 
-        $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00');// (hour:minutes)
+        $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
         $now = new \DateTime();
         $now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
         $next = new \DateTime($now->format("Y-m-d $jobInitTime"));
@@ -198,90 +258,25 @@ class Hamster extends Action
 
         Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
 
-        Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $sleep) {
+        Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) {
             $now = date('d-m-Y H:i:s', time());
             Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds");
             $loopStart = microtime(true);
 
             /* Initialise new Utopia app */
             $app = new App('UTC');
-            $console = $app->getResource('console');
 
-            /** Database connections */
-            $totalProjects = $dbForConsole->count('projects') + 1;
-            Console::success("Found a total of: {$totalProjects} projects");
+            Console::info('Getting stats for all projects');
+            $this->getStatsPerProject($pools, $cache, $dbForConsole);
+            Console::success('Completed getting stats for all projects');
 
-            $projects = [$console];
-            $count = 0;
-            $limit = 30;
-            $sum = 30;
-            $offset = 0;
-            while (!empty($projects)) {
-                foreach ($projects as $project) {
-                    /**
-                     * Skip user projects with id 'console'
-                     */
-                    if ($project->getId() === 'console') {
-                        continue;
-                    }
-
-                    Console::info("Getting stats for {$project->getId()}");
-
-                    try {
-                        $db = $project->getAttribute('database');
-                        $adapter = $pools
-                            ->get($db)
-                            ->pop()
-                            ->getResource();
-
-                        $dbForProject = new Database($adapter, $cache);
-                        $dbForProject->setDefaultDatabase('appwrite');
-                        $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                        $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project);
-
-                        if (isset($statsPerProject['email'])) {
-                            /** Send data to mixpanel */
-                            $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
-                                'name' => $statsPerProject['name'],
-                                'email' => $statsPerProject['email']
-                            ]);
-
-                            if (!$res) {
-                                Console::error('Failed to create user profile for project: ' . $project->getId());
-                            }
-
-                            $event = new Event();
-                            $event
-                                ->setName('Appwrite Cloud Project Stats')
-                                ->setProps($statsPerProject);
-                            $res = $this->mixpanel->createEvent($event);
-                            if (!$res) {
-                                Console::error('Failed to create event for project: ' . $project->getId());
-                            }
-                        }
-                    } catch (\Throwable $th) {
-                        Console::error('Failed to get stats for project ("' . $project->getId() . '") with error: ' . $th->getMessage());
-                    } finally {
-                        $pools
-                            ->get($db)
-                            ->reclaim();
-                    }
-                }
-
-                $sum = \count($projects);
-
-                $projects = $dbForConsole->find('projects', [
-                    Query::limit($limit),
-                    Query::offset($offset),
-                ]);
-
-                $offset = $offset + $limit;
-                $count = $count + $sum;
-
-                Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...');
-            }
+            Console::info('Getting stats for all organizations');
+            $this->getStatsPerOrganization($dbForConsole);
+            Console::success('Completed getting stats for all organizations');
 
+            Console::info('Getting stats for all users');
+            $this->getStatsPerUser($dbForConsole);
+            Console::success('Completed getting stats for all users');
 
             $pools
                 ->get('console')
@@ -292,4 +287,125 @@ class Hamster extends Action
             Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
         }, $sleep, $delay);
     }
+
+    protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback)
+    {
+        $count = 0;
+        $chunk = 0;
+        $limit = 50;
+        $results = [];
+        $sum = $limit;
+
+        $executionStart = \microtime(true);
+
+        while ($sum === $limit) {
+            $chunk++;
+
+            $results = $dbForConsole->find($collection, \array_merge([Query::limit($limit)]));
+
+            $sum = count($results);
+
+            Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents');
+
+            foreach ($results as $document) {
+                call_user_func($callback, $dbForConsole, $document);
+                $count++;
+            }
+        }
+
+        $executionEnd = \microtime(true);
+
+        Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
+    }
+
+    protected function getStatsPerOrganization(Database $dbForConsole)
+    {
+
+        $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) {
+            $statsPerOrganization = [];
+
+            /** Organization name */
+            $statsPerOrganization['name'] = $document->getAttribute('name');
+
+            /** Get Email and of the organization owner */
+            $membership = $dbForConsole->findOne('memberships', [
+                Query::equal('teamInternalId', [$document->getInternalId()]),
+            ]);
+
+            if (!$membership || $membership->isEmpty()) {
+                throw new Exception('Membership not found. Skipping organization : ' . $document->getId());
+            }
+
+            $userInternalId = $membership->getAttribute('userInternalId', null);
+            if ($userInternalId) {
+                $user = $dbForConsole->findOne('users', [
+                    Query::equal('_id', [$userInternalId]),
+                ]);
+
+                $statsPerOrganization['email'] = $user->getAttribute('email', null);
+            }
+
+            /** Organization Creation Date */
+            $statsPerOrganization['created'] = $document->getAttribute('$createdAt');
+
+            /** Number of team members */
+            $statsPerOrganization['members'] = $document->getAttribute('total');
+
+            /** Number of projects in this organization */
+            $statsPerOrganization['projects'] = $dbForConsole->count('projects', [
+                Query::equal('teamId', [$document->getId()]),
+                Query::limit(APP_LIMIT_COUNT)
+            ]);
+
+            if (!isset($statsPerOrganization['email'])) {
+                throw new Exception('Email not found. Skipping organization : ' . $document->getId());
+            }
+
+            $event = new Event();
+            $event
+                ->setName('Organization Daily Usage')
+                ->setProps($statsPerOrganization);
+            $res = $this->mixpanel->createEvent($event);
+            if (!$res) {
+                throw new Exception('Failed to create event for organization : ' . $document->getId());
+            }
+        });
+    }
+
+    protected function getStatsPerUser(Database $dbForConsole)
+    {
+        $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) {
+            $statsPerUser = [];
+
+            /** Organization name */
+            $statsPerUser['name'] = $document->getAttribute('name');
+
+            /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */
+            $statsPerUser['email'] = $document->getAttribute('email');
+
+            /** Organization Creation Date */
+            $statsPerUser['created'] = $document->getAttribute('$createdAt');
+
+            /** Number of teams this user is a part of */
+            $statsPerUser['memberships'] = $dbForConsole->count('memberships', [
+                Query::equal('userInternalId', [$document->getInternalId()]),
+                Query::limit(APP_LIMIT_COUNT)
+            ]);
+
+            if (!isset($statsPerUser['email'])) {
+                throw new Exception('User has no email: ' . $document->getId());
+            }
+
+            /** Send data to mixpanel */
+            $event = new Event();
+            $event
+                ->setName('User Daily Usage')
+                ->setProps($statsPerUser);
+            $res = $this->mixpanel->createEvent($event);
+
+            if (!$res) {
+                throw new Exception('Failed to create user profile for user: ' . $document->getId());
+            }
+        });
+    }
 }