diff --git a/.env b/.env index 804b8c999e..14738d1a3b 100644 --- a/.env +++ b/.env @@ -46,8 +46,8 @@ _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= _APP_HAMSTER_INTERVAL=86400 -_APP_HAMSTER_TIME=21:00 -_APP_MIXPANEL_TOKEN= +_APP_HAMSTER_TIME=12:31 +_APP_MIXPANEL_TOKEN=bce512333a58ec62f44541328607f53c _APP_SMS_PROVIDER=sms://username:password@mock _APP_SMS_FROM=+123456789 _APP_STORAGE_LIMIT=30000000 diff --git a/composer.json b/composer.json index f947592f9e..1eec45940f 100644 --- a/composer.json +++ b/composer.json @@ -44,7 +44,7 @@ "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.11.*", "utopia-php/abuse": "0.16.*", - "utopia-php/analytics": "0.10.1", + "utopia-php/analytics": "0.10.2", "utopia-php/audit": "0.17.*", "utopia-php/cache": "0.8.*", "utopia-php/cli": "0.15.*", diff --git a/composer.lock b/composer.lock index a54754a036..d86ee62b7f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f409ee69f8040b1928cbc618ff3e8a43", + "content-hash": "ca2a083ff1c0d0c77942674400137793", "packages": [ { "name": "adhocore/jwt", @@ -1583,16 +1583,16 @@ }, { "name": "utopia-php/analytics", - "version": "0.10.1", + "version": "0.10.2", "source": { "type": "git", "url": "https://github.com/utopia-php/analytics.git", - "reference": "70ada5e6b192ae27e6d5467899a4cdd886003835" + "reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/analytics/zipball/70ada5e6b192ae27e6d5467899a4cdd886003835", - "reference": "70ada5e6b192ae27e6d5467899a4cdd886003835", + "url": "https://api.github.com/repos/utopia-php/analytics/zipball/14c805114736f44c26d6d24b176a2f8b93d86a1f", + "reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f", "shasum": "" }, "require": { @@ -1623,9 +1623,9 @@ ], "support": { "issues": "https://github.com/utopia-php/analytics/issues", - "source": "https://github.com/utopia-php/analytics/tree/0.10.1" + "source": "https://github.com/utopia-php/analytics/tree/0.10.2" }, - "time": "2023-03-17T14:42:35+00:00" + "time": "2023-03-22T12:01:09+00:00" }, { "name": "utopia-php/audit", diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index cf484f2703..cc89321704 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Tasks; use Exception; +use finfo; use Utopia\App; use Utopia\Platform\Action; use Utopia\Cache\Cache; @@ -14,22 +15,23 @@ use Utopia\Analytics\Adapter\Mixpanel; use Utopia\Analytics\Event; use Utopia\Database\Document; use Utopia\Pools\Group; +use Utopia\Pools\Pool; use Utopia\Registry\Registry; class Hamster extends Action { - private array $usageStats = [ - 'files' => 'files.$all.count.total', - 'buckets' => 'buckets.$all.count.total', - 'databases' => 'databases.$all.count.total', - 'documents' => 'documents.$all.count.total', - 'collections' => 'collections.$all.count.total', - 'storage' => 'project.$all.storage.size', - 'requests' => 'project.$all.network.requests', - 'bandwidth' => 'project.$all.network.bandwidth', - 'users' => 'users.$all.count.total', - 'sessions' => 'sessions.email.requests.create', - 'executions' => 'executions.$all.compute.total', + private array $metrics = [ + 'usage_files' => 'files.$all.count.total', + 'usage_buckets' => 'buckets.$all.count.total', + 'usage_databases' => 'databases.$all.count.total', + 'usage_documents' => 'documents.$all.count.total', + 'usage_collections' => 'collections.$all.count.total', + 'usage_storage' => 'project.$all.storage.size', + 'usage_requests' => 'project.$all.network.requests', + 'usage_bandwidth' => 'project.$all.network.bandwidth', + 'usage_users' => 'users.$all.count.total', + 'usage_sessions' => 'sessions.email.requests.create', + 'usage_executions' => 'executions.$all.compute.total', ]; protected string $directory = '/usr/local'; @@ -51,129 +53,188 @@ class Hamster extends Action $this ->desc('Get stats for projects') - ->inject('register') ->inject('pools') ->inject('cache') ->inject('dbForConsole') - ->callback(function (Registry $register, Group $pools, Cache $cache, Database $dbForConsole) { - $this->action($register, $pools, $cache, $dbForConsole); + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole) { + $this->action($pools, $cache, $dbForConsole); }); } - private function getStats(Database $dbForConsole, Database $dbForProject, Document $project): array + private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole) { - $stats = []; - - $stats['time'] = microtime(true); - - /** Get Project ID */ - $stats['projectId'] = $project->getId(); - - /** Get Project Name */ - $stats['projectName'] = $project->getAttribute('name'); - - /** Get Total Functions */ - $stats['functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); - - /** Get Total Deployments */ - $stats['deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); - - /** Get Total Members */ - $teamInternalId = $project->getAttribute('teamInternalId', null); - if ($teamInternalId) { - $stats['members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ], APP_LIMIT_COUNT); - } else { - $stats['members'] = 0; - } - - /** Get Email and Name of the project owner */ - if ($teamInternalId) { - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new Exception('Membership not found. Skipping project : ' . $project->getId()); + $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache){ + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + Console::info("Skipping project console"); + return; } - $userInternalId = $membership->getAttribute('userInternalId', null); - if ($userInternalId) { - $user = $dbForConsole->findOne('users', [ - Query::equal('_id', [$userInternalId]), - ]); + Console::log("Getting stats for {$project->getId()}"); - $stats['email'] = $user->getAttribute('email', null); - $stats['name'] = $user->getAttribute('name', null); - } - } + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); - /** Get Domains */ - $stats['domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT); + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); - /** Get Platforms */ - $platforms = $dbForConsole->find('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); + $statsPerProject = []; - $stats['platforms_web'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'web'; - })); + $statsPerProject['time'] = microtime(true); - $stats['platforms_android'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'android'; - })); + /** Get Project ID */ + $statsPerProject['project_id'] = $project->getId(); - $stats['platforms_iOS'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'apple'); - })); + /** Get project created time */ + $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); - $stats['platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'flutter'); - })); + /** Get Project Name */ + $statsPerProject['project_name'] = $project->getAttribute('name'); - /** Get Usage stats */ - $range = '90d'; - $periods = [ - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + /** Get Total Functions */ + $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); - $metrics = array_values($this->usageStats); - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; + /** Get Total Deployments */ + $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); + /** Get Total Teams */ + $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; + /** Get Total Members */ + $teamInternalId = $project->getAttribute('teamInternalId', null); + if ($teamInternalId) { + $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]) + ], APP_LIMIT_COUNT); + } else { + $statsPerProject['custom_organization_members'] = 0; } - $stats[$metric] = array_reverse($stats[$metric]); - // Calculate aggregate of each metric - $stats[$metric] = array_sum(array_column($stats[$metric], 'value')); + /** Get Email and Name of the project owner */ + if ($teamInternalId) { + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new Exception('Membership not found. Skipping project : ' . $project->getId()); + } + + $userInternalId = $membership->getAttribute('userInternalId', null); + if ($userInternalId) { + $user = $dbForConsole->findOne('users', [ + Query::equal('_id', [$userInternalId]), + ]); + + $statsPerProject['email'] = $user->getAttribute('email', null); + $statsPerProject['name'] = $user->getAttribute('name', null); + } + } + + /** Get Domains */ + $statsPerProject['custom_domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT); + + /** Get Platforms */ + $platforms = $dbForConsole->find('platforms', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'web'; + })); + + $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'android'; + })); + + $statsPerProject['custom_platforms_iOS'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'apple'); + })); + + $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'flutter'); + })); + + /** Get Usage $statsPerProject */ + $periods = [ + 'infinity' => [ + 'period' => '1d', + 'limit' => 90, + ], + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + ]; + + Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { + foreach ($this->metrics as $key => $metric) { + foreach ($periods as $periodKey => $periodValue) { + $limit = $periodValue['limit']; + $period = $periodValue['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $statsPerProject[$key . '_' . $periodKey] = []; + foreach ($requestDocs as $requestDoc) { + $statsPerProject[$key . '_' . $periodKey][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); + // Calculate aggregate of each metric + $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); + } + } + }); + + if (isset($statsPerProject['email'])) { + /** Send data to mixpanel */ + $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ + 'name' => $statsPerProject['name'], + 'email' => $statsPerProject['email'] + ]); + + if (!$res) { + Console::error('Failed to create user profile for project: ' . $project->getId()); + } + + $event = new Event(); + $event + ->setName('Project Daily Usage') + ->setProps($statsPerProject); + $res = $this->mixpanel->createEvent($event); + if (!$res) { + Console::error('Failed to create event for project: ' . $project->getId()); + } + } + } catch (Exception $e) { + Console::error('Failed to send stats for project: ' . $project->getId()); + Console::error($e->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); } }); - - return $stats; } - public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void + public function action(Group $pools, Cache $cache, Database $dbForConsole): void { Console::title('Cloud Hamster V1'); @@ -181,7 +242,7 @@ class Hamster extends Action $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default) - $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00');// (hour:minutes) + $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes) $now = new \DateTime(); $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); $next = new \DateTime($now->format("Y-m-d $jobInitTime")); @@ -198,90 +259,25 @@ class Hamster extends Action Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']'); - Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $sleep) { + Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) { $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds"); $loopStart = microtime(true); /* Initialise new Utopia app */ $app = new App('UTC'); - $console = $app->getResource('console'); - /** Database connections */ - $totalProjects = $dbForConsole->count('projects') + 1; - Console::success("Found a total of: {$totalProjects} projects"); + Console::info('Getting stats for all projects'); + $this->getStatsPerProject($pools, $cache, $dbForConsole); + Console::success('Completed getting stats for all projects'); - $projects = [$console]; - $count = 0; - $limit = 30; - $sum = 30; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - Console::info("Getting stats for {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project); - - if (isset($statsPerProject['email'])) { - /** Send data to mixpanel */ - $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ - 'name' => $statsPerProject['name'], - 'email' => $statsPerProject['email'] - ]); - - if (!$res) { - Console::error('Failed to create user profile for project: ' . $project->getId()); - } - - $event = new Event(); - $event - ->setName('Appwrite Cloud Project Stats') - ->setProps($statsPerProject); - $res = $this->mixpanel->createEvent($event); - if (!$res) { - Console::error('Failed to create event for project: ' . $project->getId()); - } - } - } catch (\Throwable $th) { - Console::error('Failed to get stats for project ("' . $project->getId() . '") with error: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - - Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); - } + Console::info('Getting stats for all organizations'); + $this->getStatsPerOrganization($dbForConsole); + Console::success('Completed getting stats for all organizations'); + Console::info('Getting stats for all users'); + $this->getStatsPerUser($dbForConsole); + Console::success('Completed getting stats for all users'); $pools ->get('console') @@ -292,4 +288,125 @@ class Hamster extends Action Console::info("[{$now}] Cloud Stats took {$loopTook} seconds"); }, $sleep, $delay); } + + protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback) + { + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $dbForConsole->find($collection, \array_merge([Query::limit($limit)])); + + $sum = count($results); + + Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents'); + + foreach ($results as $document) { + call_user_func($callback, $dbForConsole, $document); + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); + } + + protected function getStatsPerOrganization(Database $dbForConsole) + { + + $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) { + $statsPerOrganization = []; + + /** Organization name */ + $statsPerOrganization['name'] = $document->getAttribute('name'); + + /** Get Email and of the organization owner */ + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$document->getInternalId()]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new Exception('Membership not found. Skipping organization : ' . $document->getId()); + } + + $userInternalId = $membership->getAttribute('userInternalId', null); + if ($userInternalId) { + $user = $dbForConsole->findOne('users', [ + Query::equal('_id', [$userInternalId]), + ]); + + $statsPerOrganization['email'] = $user->getAttribute('email', null); + } + + /** Organization Creation Date */ + $statsPerOrganization['created'] = $document->getAttribute('$createdAt'); + + /** Number of team members */ + $statsPerOrganization['members'] = $document->getAttribute('total'); + + /** Number of projects in this organization */ + $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ + Query::equal('teamId', [$document->getId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerOrganization['email'])) { + throw new Exception('Email not found. Skipping organization : ' . $document->getId()); + } + + $event = new Event(); + $event + ->setName('Organization Daily Usage') + ->setProps($statsPerOrganization); + $res = $this->mixpanel->createEvent($event); + if (!$res) { + throw new Exception('Failed to create event for organization : ' . $document->getId()); + } + }); + } + + protected function getStatsPerUser(Database $dbForConsole) + { + $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) { + $statsPerUser = []; + + /** Organization name */ + $statsPerUser['name'] = $document->getAttribute('name'); + + /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ + $statsPerUser['email'] = $document->getAttribute('email'); + + /** Organization Creation Date */ + $statsPerUser['created'] = $document->getAttribute('$createdAt'); + + /** Number of teams this user is a part of */ + $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerUser['email'])) { + throw new Exception('User has no email: ' . $document->getId()); + } + + /** Send data to mixpanel */ + $event = new Event(); + $event + ->setName('User Daily Usage') + ->setProps($statsPerUser); + $res = $this->mixpanel->createEvent($event); + + if (!$res) { + throw new Exception('Failed to create user profile for user: ' . $document->getId()); + } + }); + } }