From dea3e74b6adb0f983a00d08e5a026d7dad50477f Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 28 Nov 2023 10:19:55 +0000 Subject: [PATCH] Implement Job based hamster --- Dockerfile | 3 +- app/cli.php | 4 + app/init.php | 4 + app/worker.php | 4 + bin/worker-hamster | 3 + docker-compose.yml | 31 ++ src/Appwrite/Event/Event.php | 3 + src/Appwrite/Event/Hamster.php | 153 +++++++ src/Appwrite/Platform/Services/Workers.php | 2 + src/Appwrite/Platform/Tasks/Hamster.php | 373 +++-------------- src/Appwrite/Platform/Workers/Hamster.php | 454 +++++++++++++++++++++ 11 files changed, 708 insertions(+), 326 deletions(-) create mode 100644 bin/worker-hamster create mode 100644 src/Appwrite/Event/Hamster.php create mode 100644 src/Appwrite/Platform/Workers/Hamster.php diff --git a/Dockerfile b/Dockerfile index 059c499bd..599e4ea70 100755 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,8 @@ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats + chmod +x /usr/local/bin/calc-tier-stats && \ + chmod +x /usr/local/bin/worker-hamster # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/cli.php b/app/cli.php index 643a615c4..003f3a1f7 100644 --- a/app/cli.php +++ b/app/cli.php @@ -6,6 +6,7 @@ require_once __DIR__ . '/controllers/general.php'; use Appwrite\Event\Delete; use Appwrite\Event\Certificate; use Appwrite\Event\Func; +use Appwrite\Event\Hamster; use Appwrite\Platform\Appwrite; use Utopia\CLI\CLI; use Utopia\Database\Validator\Authorization; @@ -154,6 +155,9 @@ CLI::setResource('queue', function (Group $pools) { CLI::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); +CLI::setResource('queueForHamster', function (Connection $queue) { + return new Hamster($queue); +}, ['queue']); CLI::setResource('queueForDeletes', function (Connection $queue) { return new Delete($queue); }, ['queue']); diff --git a/app/init.php b/app/init.php index 2c0219eec..c30eb77e8 100644 --- a/app/init.php +++ b/app/init.php @@ -72,6 +72,7 @@ use Ahc\Jwt\JWTException; use Appwrite\Event\Build; use Appwrite\Event\Certificate; use Appwrite\Event\Func; +use Appwrite\Event\Hamster; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; @@ -916,6 +917,9 @@ App::setResource('queueForCertificates', function (Connection $queue) { App::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); +App::setResource('queueForHamster', function (Connection $queue) { + return new Hamster($queue); +}, ['queue']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); diff --git a/app/worker.php b/app/worker.php index 32a8b9804..4f7355311 100644 --- a/app/worker.php +++ b/app/worker.php @@ -9,6 +9,7 @@ use Appwrite\Event\Certificate; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Func; +use Appwrite\Event\Hamster; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; @@ -155,6 +156,9 @@ Server::setResource('queueForCertificates', function (Connection $queue) { Server::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); +Server::setResource('queueForHamster', function (Connection $queue) { + return new Hamster($queue); +}, ['queue']); Server::setResource('logger', function (Registry $register) { return $register->get('logger'); }, ['register']); diff --git a/bin/worker-hamster b/bin/worker-hamster new file mode 100644 index 000000000..b388dd13c --- /dev/null +++ b/bin/worker-hamster @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php hamster $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 42091e5e4..da9b0e51e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -717,6 +717,37 @@ services: environment: - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-worker-hamster: + entrypoint: worker-hamster + <<: *x-logging + container_name: appwrite-worker-hamster + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_LOGGING_PROVIDER + - _APP_MIXPANEL_TOKEN + openruntimes-executor: container_name: openruntimes-executor hostname: appwrite-executor diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 46b430d12..fc12c5b5b 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -42,6 +42,9 @@ class Event public const MIGRATIONS_QUEUE_NAME = 'v1-migrations'; public const MIGRATIONS_CLASS_NAME = 'MigrationsV1'; + public const HAMSTER_QUEUE_NAME = 'v1-hamster'; + public const HAMSTER_CLASS_NAME = 'HamsterV1'; + protected string $queue = ''; protected string $class = ''; protected string $event = ''; diff --git a/src/Appwrite/Event/Hamster.php b/src/Appwrite/Event/Hamster.php new file mode 100644 index 000000000..9ae730367 --- /dev/null +++ b/src/Appwrite/Event/Hamster.php @@ -0,0 +1,153 @@ +setQueue(Event::HAMSTER_QUEUE_NAME) + ->setClass(Event::HAMSTER_CLASS_NAME); + } + + /** + * Sets the type for the hamster event. + * + * @param string $type + * @return self + */ + public function setType(string $type): self + { + $this->type = $type; + + return $this; + } + + /** + * Returns the set type for the hamster event. + * + * @return string + */ + public function getType(): string + { + return $this->type; + } + + /** + * Sets the project for the hamster event. + * + * @param Document $project + */ + public function setProject(Document $project): self + { + $this->project = $project; + + return $this; + } + + /** + * Returns the set project for the hamster event. + * + * @return Document + */ + public function getProject(): Document + { + return $this->project; + } + + /** + * Sets the organization for the hamster event. + * + * @param Document $organization + */ + public function setOrganization(Document $organization): self + { + $this->organization = $organization; + + return $this; + } + + /** + * Returns the set organization for the hamster event. + * + * @return string + */ + public function getOrganization(): Document + { + return $this->organization; + } + + /** + * Sets the user for the hamster event. + * + * @param Document $user + */ + public function setUser(Document $user): self + { + $this->user = $user; + + return $this; + } + + /** + * Returns the set user for the hamster event. + * + * @return Document + */ + public function getUser(): Document + { + return $this->user; + } + + /** + * Executes the function event and sends it to the functions worker. + * + * @return string|bool + * @throws \InvalidArgumentException + */ + public function trigger(): string|bool + { + if ($this->paused) { + return false; + } + + $client = new Client($this->queue, $this->connection); + + $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; + + return $client->enqueue([ + 'type' => $this->type, + 'project' => $this->project, + 'organization' => $this->organization, + 'user' => $this->user, + 'events' => $events, + ]); + } + + /** + * Generate a function event from a base event + * + * @param Event $event + * + * @return self + * + */ + public function from(Event $event): self + { + $this->event = $event->getEvent(); + $this->params = $event->getParams(); + return $this; + } +} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 07fc25434..c5a051476 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -12,6 +12,7 @@ use Appwrite\Platform\Workers\Databases; use Appwrite\Platform\Workers\Functions; use Appwrite\Platform\Workers\Builds; use Appwrite\Platform\Workers\Deletes; +use Appwrite\Platform\Workers\Hamster; use Appwrite\Platform\Workers\Migrations; class Workers extends Service @@ -30,6 +31,7 @@ class Workers extends Service ->addAction(Builds::getName(), new Builds()) ->addAction(Deletes::getName(), new Deletes()) ->addAction(Migrations::getName(), new Migrations()) + ->addAction(Hamster::getName(), new Hamster()) ; } diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 1d5d3b0b2..fea4c88ad 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Tasks; +use Appwrite\Event\Hamster as EventHamster; use Appwrite\Network\Validator\Origin; use Exception; use Utopia\App; @@ -19,20 +20,6 @@ use Utopia\Pools\Group; class Hamster extends Action { - private array $metrics = [ - 'usage_files' => 'files.$all.count.total', - 'usage_buckets' => 'buckets.$all.count.total', - 'usage_databases' => 'databases.$all.count.total', - 'usage_documents' => 'documents.$all.count.total', - 'usage_collections' => 'collections.$all.count.total', - 'usage_storage' => 'project.$all.storage.size', - 'usage_requests' => 'project.$all.network.requests', - 'usage_bandwidth' => 'project.$all.network.bandwidth', - 'usage_users' => 'users.$all.count.total', - 'usage_sessions' => 'sessions.email.requests.create', - 'usage_executions' => 'executions.$all.compute.total', - ]; - protected string $directory = '/usr/local'; protected string $path; @@ -60,233 +47,20 @@ class Hamster extends Action }); } - private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole) + private function getStatsPerProject(Group $pools, Database $dbForConsole) { - $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - Console::info("Skipping project console"); - return; - } + $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools) { + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); - Console::log("Getting stats for {$project->getId()}"); + $hamsterTask = new EventHamster($connection); - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); + $hamsterTask + ->setType('project') + ->setProject($project) + ->trigger(); - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $statsPerProject = []; - - $statsPerProject['time'] = microtime(true); - - /** Get Project ID */ - $statsPerProject['project_id'] = $project->getId(); - - /** Get project created time */ - $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); - - /** Get Project Name */ - $statsPerProject['project_name'] = $project->getAttribute('name'); - - /** Total Project Variables */ - $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT); - - /** Total Migrations */ - $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); - - /** Get Custom SMTP */ - $smtp = $project->getAttribute('smtp', null); - if ($smtp) { - $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled'; - - /** Get Custom Templates Count */ - $templates = array_keys($project->getAttribute('templates', [])); - $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'email'); - }); - $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'sms'); - }); - } - - /** Get total relationship attributes */ - $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [ - Query::equal('type', ['relationship']) - ], APP_LIMIT_COUNT); - - /** Get Total Functions */ - $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); - - foreach (\array_keys(Config::getParam('runtimes')) as $runtime) { - $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [ - Query::equal('runtime', [$runtime]), - ], APP_LIMIT_COUNT); - } - - /** Get Total Deployments */ - $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [ - Query::equal('type', ['manual']) - ], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [ - Query::equal('type', ['vcs']) - ], APP_LIMIT_COUNT); - - /** Get VCS repos connected */ - $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [ - Query::equal('projectInternalId', [$project->getInternalId()]) - ], APP_LIMIT_COUNT); - - /** Get Total Teams */ - $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); - - /** Get Total Members */ - $teamInternalId = $project->getAttribute('teamInternalId', null); - if ($teamInternalId) { - $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ], APP_LIMIT_COUNT); - } else { - $statsPerProject['custom_organization_members'] = 0; - } - - /** Get Email and Name of the project owner */ - if ($teamInternalId) { - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new Exception('Membership not found. Skipping project : ' . $project->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerProject['email'] = $user->getAttribute('email', null); - $statsPerProject['name'] = $user->getAttribute('name', null); - } - } - - /** Get Domains */ - $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Platforms */ - $platforms = $dbForConsole->find('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'web'; - })); - - $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'android'; - })); - - $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'apple'); - })); - - $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'flutter'); - })); - - $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX]; - - foreach ($flutterPlatforms as $flutterPlatform) { - $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) { - return $platform['type'] === $flutterPlatform; - })); - } - - $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Usage $statsPerProject */ - $periods = [ - 'infinity' => [ - 'period' => '1d', - 'limit' => 90, - ], - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - ]; - - Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { - foreach ($this->metrics as $key => $metric) { - foreach ($periods as $periodKey => $periodValue) { - $limit = $periodValue['limit']; - $period = $periodValue['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $statsPerProject[$key . '_' . $periodKey] = []; - foreach ($requestDocs as $requestDoc) { - $statsPerProject[$key . '_' . $periodKey][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); - // Calculate aggregate of each metric - $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); - } - } - }); - - if (isset($statsPerProject['email'])) { - /** Send data to mixpanel */ - $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ - 'name' => $statsPerProject['name'], - 'email' => $statsPerProject['email'] - ]); - - if (!$res) { - Console::error('Failed to create user profile for project: ' . $project->getId()); - } - } - - $event = new Event(); - $event - ->setName('Project Daily Usage') - ->setProps($statsPerProject); - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - Console::error('Failed to create event for project: ' . $project->getId()); - } - } catch (Exception $e) { - Console::error('Failed to send stats for project: ' . $project->getId()); - Console::error($e->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } + $queue->reclaim(); }); } @@ -305,6 +79,8 @@ class Hamster extends Action $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); $delay = $next->getTimestamp() - $now->getTimestamp(); + $delay = 5; + /** * If time passed for the target day. */ @@ -323,17 +99,17 @@ class Hamster extends Action /* Initialise new Utopia app */ $app = new App('UTC'); - Console::info('Getting stats for all projects'); - $this->getStatsPerProject($pools, $cache, $dbForConsole); - Console::success('Completed getting stats for all projects'); + Console::info('Queuing stats for all projects'); + $this->getStatsPerProject($pools, $dbForConsole); + Console::success('Completed queuing stats for all projects'); - Console::info('Getting stats for all organizations'); - $this->getStatsPerOrganization($dbForConsole); - Console::success('Completed getting stats for all organizations'); + Console::info('Queuing stats for all organizations'); + $this->getStatsPerOrganization($pools, $dbForConsole); + Console::success('Completed queuing stats for all organizations'); - Console::info('Getting stats for all users'); - $this->getStatsPerUser($dbForConsole); - Console::success('Completed getting stats for all users'); + Console::info('Queuing stats for all users'); + $this->getStatsPerUser($pools, $dbForConsole); + Console::success('Completed queuing stats for all users'); $pools ->get('console') @@ -378,96 +154,43 @@ class Hamster extends Action Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } - protected function getStatsPerOrganization(Database $dbForConsole) + protected function getStatsPerOrganization(Group $pools, Database $dbForConsole) { - $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) { + $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($pools) { try { - $statsPerOrganization = []; - - /** Organization name */ - $statsPerOrganization['name'] = $document->getAttribute('name'); - - /** Get Email and of the organization owner */ - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$document->getInternalId()]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new Exception('Membership not found. Skipping organization : ' . $document->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerOrganization['email'] = $user->getAttribute('email', null); - } - - /** Organization Creation Date */ - $statsPerOrganization['created'] = $document->getAttribute('$createdAt'); - - /** Number of team members */ - $statsPerOrganization['members'] = $document->getAttribute('total'); - - /** Number of projects in this organization */ - $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ - Query::equal('teamId', [$document->getId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerOrganization['email'])) { - throw new Exception('Email not found. Skipping organization : ' . $document->getId()); - } - - $event = new Event(); - $event - ->setName('Organization Daily Usage') - ->setProps($statsPerOrganization); - $res = $this->mixpanel->createEvent($event); - if (!$res) { - throw new Exception('Failed to create event for organization : ' . $document->getId()); - } + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + + $hamsterTask = new EventHamster($connection); + + $hamsterTask + ->setType('organization') + ->setOrganization($organization) + ->trigger(); + + $queue->reclaim(); } catch (Exception $e) { Console::error($e->getMessage()); } }); } - protected function getStatsPerUser(Database $dbForConsole) + protected function getStatsPerUser(Group $pools, Database $dbForConsole) { - $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) { + $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($pools) { try { - $statsPerUser = []; - - /** Organization name */ - $statsPerUser['name'] = $document->getAttribute('name'); - - /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ - $statsPerUser['email'] = $document->getAttribute('email'); - - /** Organization Creation Date */ - $statsPerUser['created'] = $document->getAttribute('$createdAt'); - - /** Number of teams this user is a part of */ - $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ - Query::equal('userInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerUser['email'])) { - throw new Exception('User has no email: ' . $document->getId()); - } - - /** Send data to mixpanel */ - $event = new Event(); - $event - ->setName('User Daily Usage') - ->setProps($statsPerUser); - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - throw new Exception('Failed to create user profile for user: ' . $document->getId()); - } + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + + $hamsterTask = new EventHamster($connection); + + $hamsterTask + ->setType('user') + ->setUser($user) + ->trigger(); + + $queue->reclaim(); } catch (Exception $e) { Console::error($e->getMessage()); } diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php new file mode 100644 index 000000000..d1c03744c --- /dev/null +++ b/src/Appwrite/Platform/Workers/Hamster.php @@ -0,0 +1,454 @@ + 'files.$all.count.total', + 'usage_buckets' => 'buckets.$all.count.total', + 'usage_databases' => 'databases.$all.count.total', + 'usage_documents' => 'documents.$all.count.total', + 'usage_collections' => 'collections.$all.count.total', + 'usage_storage' => 'project.$all.storage.size', + 'usage_requests' => 'project.$all.network.requests', + 'usage_bandwidth' => 'project.$all.network.bandwidth', + 'usage_users' => 'users.$all.count.total', + 'usage_sessions' => 'sessions.email.requests.create', + 'usage_executions' => 'executions.$all.compute.total', + ]; + + protected string $directory = '/usr/local'; + + protected string $path; + + protected string $date; + + protected Mixpanel $mixpanel; + + public static function getName(): string + { + return 'hamster'; + } + + /** + * @throws \Exception + */ + public function __construct() + { + $this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', '')); + + $this + ->desc('Hamster worker') + ->inject('message') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('queueForHamster') + ->inject('queueForEvents') + ->inject('usage') + ->inject('log') + ->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole, EventHamster $queueForHamster, Event $queueForEvents, Stats $usage, Log $log) => $this->action($message, $pools, $cache, $dbForConsole, $queueForHamster, $queueForEvents, $usage, $log)); + } + + /** + * @param Message $message + * @param Group $pools + * @param Cache $cache + * @param Database $dbForConsole + * @param EventHamster $queueForHamster + * @param Event $queueForEvents + * @param Stats $usage + * @param Log $log + * @return void + * @throws Authorization + * @throws Structure + * @throws \Utopia\Database\Exception + * @throws Conflict + */ + public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole, EventHamster $queueForHamster, Event $queueForEvents, Stats $usage, Log $log): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new \Exception('Missing payload'); + } + + + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new \Exception('Missing payload'); + } + + $type = $payload['type'] ?? ''; + + switch ($type) { + case 'project': + $this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole, $log); + break; + case 'organization': + $this->getStatsForOrganization(new Document($payload['organization']), $pools, $cache, $dbForConsole, $log); + break; + case 'user': + $this->getStatsPerUser(new Document($payload['user']), $dbForConsole); + break; + } + } + + /** + * @param Document $project + * @param Group $pools + * @param Cache $cache + * @param Database $dbForConsole + * @param Log $log + * @throws \Utopia\Database\Exception + */ + private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole, Log $log): void + { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + Console::info("Skipping project console"); + return; + } + + Console::log("Getting stats for {$project->getId()}"); + + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + $statsPerProject = []; + + $statsPerProject['time'] = microtime(true); + + /** Get Project ID */ + $statsPerProject['project_id'] = $project->getId(); + + /** Get project created time */ + $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); + + /** Get Project Name */ + $statsPerProject['project_name'] = $project->getAttribute('name'); + + /** Total Project Variables */ + $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT); + + /** Total Migrations */ + $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); + + /** Get Custom SMTP */ + $smtp = $project->getAttribute('smtp', null); + if ($smtp) { + $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled'; + + /** Get Custom Templates Count */ + $templates = array_keys($project->getAttribute('templates', [])); + $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) { + return str_contains($template, 'email'); + }); + $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) { + return str_contains($template, 'sms'); + }); + } + + /** Get total relationship attributes */ + $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [ + Query::equal('type', ['relationship']) + ], APP_LIMIT_COUNT); + + /** Get Total Functions */ + $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); + + foreach (\array_keys(Config::getParam('runtimes')) as $runtime) { + $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [ + Query::equal('runtime', [$runtime]), + ], APP_LIMIT_COUNT); + } + + /** Get Total Deployments */ + $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); + $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [ + Query::equal('type', ['manual']) + ], APP_LIMIT_COUNT); + $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [ + Query::equal('type', ['vcs']) + ], APP_LIMIT_COUNT); + + /** Get VCS repos connected */ + $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ], APP_LIMIT_COUNT); + + /** Get Total Teams */ + $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); + + /** Get Total Members */ + $teamInternalId = $project->getAttribute('teamInternalId', null); + if ($teamInternalId) { + $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]) + ], APP_LIMIT_COUNT); + } else { + $statsPerProject['custom_organization_members'] = 0; + } + + /** Get Email and Name of the project owner */ + if ($teamInternalId) { + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new \Exception('Membership not found. Skipping project : ' . $project->getId()); + } + + $userId = $membership->getAttribute('userId', null); + if ($userId) { + $user = $dbForConsole->getDocument('users', $userId); + $statsPerProject['email'] = $user->getAttribute('email', null); + $statsPerProject['name'] = $user->getAttribute('name', null); + } + } + + /** Get Domains */ + $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + /** Get Platforms */ + $platforms = $dbForConsole->find('platforms', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'web'; + })); + + $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { + return $platform['type'] === 'android'; + })); + + $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'apple'); + })); + + $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { + return str_contains($platform['type'], 'flutter'); + })); + + $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX]; + + foreach ($flutterPlatforms as $flutterPlatform) { + $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) { + return $platform['type'] === $flutterPlatform; + })); + } + + $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + /** Get Usage $statsPerProject */ + $periods = [ + 'infinity' => [ + 'period' => '1d', + 'limit' => 90, + ], + '24h' => [ + 'period' => '1h', + 'limit' => 24, + ], + ]; + + Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { + foreach ($this->metrics as $key => $metric) { + foreach ($periods as $periodKey => $periodValue) { + $limit = $periodValue['limit']; + $period = $periodValue['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $statsPerProject[$key . '_' . $periodKey] = []; + foreach ($requestDocs as $requestDoc) { + $statsPerProject[$key . '_' . $periodKey][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); + // Calculate aggregate of each metric + $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); + } + } + }); + + if (isset($statsPerProject['email'])) { + /** Send data to mixpanel */ + $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ + 'name' => $statsPerProject['name'], + 'email' => $statsPerProject['email'] + ]); + + if (!$res) { + Console::error('Failed to create user profile for project: ' . $project->getId()); + } + } + + $event = new AnalyticsEvent(); + $event + ->setName('Project Daily Usage') + ->setProps($statsPerProject); + $res = $this->mixpanel->createEvent($event); + + if (!$res) { + Console::error('Failed to create event for project: ' . $project->getId()); + } + } catch (\Exception $e) { + Console::error('Failed to send stats for project: ' . $project->getId()); + Console::error($e->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + /** + * @param Document $organization + * @param Group $pools + * @param Cache $cache + * @param Database $dbForConsole + * @param Log $log + * @throws \Utopia\Database\Exception + */ + private function getStatsForOrganization(Document $organization, Group $pools, Cache $cache, Database $dbForConsole, Log $log): void + { + try { + $statsPerOrganization = []; + + /** Organization name */ + $statsPerOrganization['name'] = $organization->getAttribute('name'); + + /** Get Email and of the organization owner */ + $membership = $dbForConsole->findOne('memberships', [ + Query::equal('teamInternalId', [$organization->getInternalId()]), + ]); + + if (!$membership || $membership->isEmpty()) { + throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId()); + } + + $userId = $membership->getAttribute('userId', null); + if ($userId) { + $user = $dbForConsole->getDocument('users', $userId); + $statsPerOrganization['email'] = $user->getAttribute('email', null); + } + + + /** Organization Creation Date */ + $statsPerOrganization['created'] = $organization->getAttribute('$createdAt'); + + /** Number of team members */ + $statsPerOrganization['members'] = $organization->getAttribute('total'); + + /** Number of projects in this organization */ + $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ + Query::equal('teamId', [$organization->getId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerOrganization['email'])) { + throw new \Exception('Email not found. Skipping organization : ' . $organization->getId()); + } + + $event = new AnalyticsEvent(); + $event + ->setName('Organization Daily Usage') + ->setProps($statsPerOrganization); + $res = $this->mixpanel->createEvent($event); + if (!$res) { + throw new \Exception('Failed to create event for organization : ' . $organization->getId()); + } + } catch (\Exception $e) { + Console::error($e->getMessage()); + } + } + + protected function getStatsPerUser(Document $user, Database $dbForConsole) + { + try { + $statsPerUser = []; + + /** Organization name */ + $statsPerUser['name'] = $user->getAttribute('name'); + + /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ + $statsPerUser['email'] = $user->getAttribute('email'); + + /** Organization Creation Date */ + $statsPerUser['created'] = $user->getAttribute('$createdAt'); + + /** Number of teams this user is a part of */ + $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ + Query::equal('userInternalId', [$user->getInternalId()]), + Query::limit(APP_LIMIT_COUNT) + ]); + + if (!isset($statsPerUser['email'])) { + throw new \Exception('User has no email: ' . $user->getId()); + } + + /** Send data to mixpanel */ + $event = new AnalyticsEvent(); + $event + ->setName('User Daily Usage') + ->setProps($statsPerUser); + $res = $this->mixpanel->createEvent($event); + + if (!$res) { + throw new \Exception('Failed to create user profile for user: ' . $user->getId()); + } + } catch (\Exception $e) { + Console::error($e->getMessage()); + } + } +}