CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-17 11:12:41 +12:00
Code Issues Projects Releases Wiki Activity

Implement Job based hamster

Browse source
This commit is contained in:
Bradley Schofield 2023-11-28 10:19:55 +00:00
parent 5a715ff68c
commit dea3e74b6a
11 changed files with 708 additions and 326 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Dockerfile
View file

@ -105,7 +105,8 @@ RUN chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/delete-orphaned-projects && \
chmod +x /usr/local/bin/clear-card-cache && \
chmod +x /usr/local/bin/calc-users-stats && \
chmod +x /usr/local/bin/calc-tier-stats
chmod +x /usr/local/bin/calc-tier-stats && \
chmod +x /usr/local/bin/worker-hamster
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

4
app/cli.php
View file

@ -6,6 +6,7 @@ require_once __DIR__ . '/controllers/general.php';
use Appwrite\Event\Delete;
use Appwrite\Event\Certificate;
use Appwrite\Event\Func;
use Appwrite\Event\Hamster;
use Appwrite\Platform\Appwrite;
use Utopia\CLI\CLI;
use Utopia\Database\Validator\Authorization;
@ -154,6 +155,9 @@ CLI::setResource('queue', function (Group $pools) {
CLI::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
CLI::setResource('queueForHamster', function (Connection $queue) {
return new Hamster($queue);
}, ['queue']);
CLI::setResource('queueForDeletes', function (Connection $queue) {
return new Delete($queue);
}, ['queue']);

4
app/init.php
View file

@ -72,6 +72,7 @@ use Ahc\Jwt\JWTException;
use Appwrite\Event\Build;
use Appwrite\Event\Certificate;
use Appwrite\Event\Func;
use Appwrite\Event\Hamster;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy;
@ -916,6 +917,9 @@ App::setResource('queueForCertificates', function (Connection $queue) {
App::setResource('queueForMigrations', function (Connection $queue) {
return new Migration($queue);
}, ['queue']);
App::setResource('queueForHamster', function (Connection $queue) {
return new Hamster($queue);
}, ['queue']);
App::setResource('usage', function ($register) {
return new Stats($register->get('statsd'));
}, ['register']);

4
app/worker.php
View file

@ -9,6 +9,7 @@ use Appwrite\Event\Certificate;
use Appwrite\Event\Database as EventDatabase;
use Appwrite\Event\Delete;
use Appwrite\Event\Func;
use Appwrite\Event\Hamster;
use Appwrite\Event\Mail;
use Appwrite\Event\Messaging;
use Appwrite\Event\Migration;
@ -155,6 +156,9 @@ Server::setResource('queueForCertificates', function (Connection $queue) {
Server::setResource('queueForMigrations', function (Connection $queue) {
return new Migration($queue);
}, ['queue']);
Server::setResource('queueForHamster', function (Connection $queue) {
return new Hamster($queue);
}, ['queue']);
Server::setResource('logger', function (Registry $register) {
return $register->get('logger');
}, ['register']);

3
bin/worker-hamster Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/worker.php hamster $@

31
docker-compose.yml
View file

@ -717,6 +717,37 @@ services:
environment:
- _APP_ASSISTANT_OPENAI_API_KEY
appwrite-worker-hamster:
entrypoint: worker-hamster
<<: *x-logging
container_name: appwrite-worker-hamster
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_CONFIG
- _APP_LOGGING_PROVIDER
- _APP_MIXPANEL_TOKEN
openruntimes-executor:
container_name: openruntimes-executor
hostname: appwrite-executor

3
src/Appwrite/Event/Event.php
View file

@ -42,6 +42,9 @@ class Event
public const MIGRATIONS_QUEUE_NAME = 'v1-migrations';
public const MIGRATIONS_CLASS_NAME = 'MigrationsV1';
public const HAMSTER_QUEUE_NAME = 'v1-hamster';
public const HAMSTER_CLASS_NAME = 'HamsterV1';
protected string $queue = '';
protected string $class = '';
protected string $event = '';

153
src/Appwrite/Event/Hamster.php Normal file
View file

@ -0,0 +1,153 @@
<?php
namespace Appwrite\Event;
use Utopia\Database\Document;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class Hamster extends Event
{
protected string $type = '';
protected ?Document $project = null;
protected ?Document $organization = null;
protected ?Document $user = null;
public function __construct(protected Connection $connection)
{
parent::__construct($connection);
$this
->setQueue(Event::HAMSTER_QUEUE_NAME)
->setClass(Event::HAMSTER_CLASS_NAME);
}
/**
* Sets the type for the hamster event.
*
* @param string $type
* @return self
*/
public function setType(string $type): self
{
$this->type = $type;
return $this;
}
/**
* Returns the set type for the hamster event.
*
* @return string
*/
public function getType(): string
{
return $this->type;
}
/**
* Sets the project for the hamster event.
*
* @param Document $project
*/
public function setProject(Document $project): self
{
$this->project = $project;
return $this;
}
/**
* Returns the set project for the hamster event.
*
* @return Document
*/
public function getProject(): Document
{
return $this->project;
}
/**
* Sets the organization for the hamster event.
*
* @param Document $organization
*/
public function setOrganization(Document $organization): self
{
$this->organization = $organization;
return $this;
}
/**
* Returns the set organization for the hamster event.
*
* @return string
*/
public function getOrganization(): Document
{
return $this->organization;
}
/**
* Sets the user for the hamster event.
*
* @param Document $user
*/
public function setUser(Document $user): self
{
$this->user = $user;
return $this;
}
/**
* Returns the set user for the hamster event.
*
* @return Document
*/
public function getUser(): Document
{
return $this->user;
}
/**
* Executes the function event and sends it to the functions worker.
*
* @return string|bool
* @throws \InvalidArgumentException
*/
public function trigger(): string|bool
{
if ($this->paused) {
return false;
}
$client = new Client($this->queue, $this->connection);
$events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null;
return $client->enqueue([
'type' => $this->type,
'project' => $this->project,
'organization' => $this->organization,
'user' => $this->user,
'events' => $events,
]);
}
/**
* Generate a function event from a base event
*
* @param Event $event
*
* @return self
*
*/
public function from(Event $event): self
{
$this->event = $event->getEvent();
$this->params = $event->getParams();
return $this;
}
}

2
src/Appwrite/Platform/Services/Workers.php
View file

@ -12,6 +12,7 @@ use Appwrite\Platform\Workers\Databases;
use Appwrite\Platform\Workers\Functions;
use Appwrite\Platform\Workers\Builds;
use Appwrite\Platform\Workers\Deletes;
use Appwrite\Platform\Workers\Hamster;
use Appwrite\Platform\Workers\Migrations;
class Workers extends Service
@ -30,6 +31,7 @@ class Workers extends Service
->addAction(Builds::getName(), new Builds())
->addAction(Deletes::getName(), new Deletes())
->addAction(Migrations::getName(), new Migrations())
->addAction(Hamster::getName(), new Hamster())
;
}

373
src/Appwrite/Platform/Tasks/Hamster.php
View file

@ -2,6 +2,7 @@
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Hamster as EventHamster;
use Appwrite\Network\Validator\Origin;
use Exception;
use Utopia\App;
@ -19,20 +20,6 @@ use Utopia\Pools\Group;
class Hamster extends Action
{
private array $metrics = [
'usage_files' => 'files.$all.count.total',
'usage_buckets' => 'buckets.$all.count.total',
'usage_databases' => 'databases.$all.count.total',
'usage_documents' => 'documents.$all.count.total',
'usage_collections' => 'collections.$all.count.total',
'usage_storage' => 'project.$all.storage.size',
'usage_requests' => 'project.$all.network.requests',
'usage_bandwidth' => 'project.$all.network.bandwidth',
'usage_users' => 'users.$all.count.total',
'usage_sessions' => 'sessions.email.requests.create',
'usage_executions' => 'executions.$all.compute.total',
];
protected string $directory = '/usr/local';
protected string $path;
@ -60,233 +47,20 @@ class Hamster extends Action
});
}
private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole)
private function getStatsPerProject(Group $pools, Database $dbForConsole)
{
$this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
Console::info("Skipping project console");
return;
}
$this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools) {
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
Console::log("Getting stats for {$project->getId()}");
$hamsterTask = new EventHamster($connection);
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$hamsterTask
->setType('project')
->setProject($project)
->trigger();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$statsPerProject = [];
$statsPerProject['time'] = microtime(true);
/** Get Project ID */
$statsPerProject['project_id'] = $project->getId();
/** Get project created time */
$statsPerProject['project_created'] = $project->getAttribute('$createdAt');
/** Get Project Name */
$statsPerProject['project_name'] = $project->getAttribute('name');
/** Total Project Variables */
$statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT);
/** Total Migrations */
$statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT);
/** Get Custom SMTP */
$smtp = $project->getAttribute('smtp', null);
if ($smtp) {
$statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled';
/** Get Custom Templates Count */
$templates = array_keys($project->getAttribute('templates', []));
$statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) {
return str_contains($template, 'email');
});
$statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) {
return str_contains($template, 'sms');
});
}
/** Get total relationship attributes */
$statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [
Query::equal('type', ['relationship'])
], APP_LIMIT_COUNT);
/** Get Total Functions */
$statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
foreach (\array_keys(Config::getParam('runtimes')) as $runtime) {
$statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [
Query::equal('runtime', [$runtime]),
], APP_LIMIT_COUNT);
}
/** Get Total Deployments */
$statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
$statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [
Query::equal('type', ['manual'])
], APP_LIMIT_COUNT);
$statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [
Query::equal('type', ['vcs'])
], APP_LIMIT_COUNT);
/** Get VCS repos connected */
$statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [
Query::equal('projectInternalId', [$project->getInternalId()])
], APP_LIMIT_COUNT);
/** Get Total Teams */
$statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
], APP_LIMIT_COUNT);
} else {
$statsPerProject['custom_organization_members'] = 0;
}
/** Get Email and Name of the project owner */
if ($teamInternalId) {
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$teamInternalId]),
]);
if (!$membership || $membership->isEmpty()) {
throw new Exception('Membership not found. Skipping project : ' . $project->getId());
}
$userId = $membership->getAttribute('userId', null);
if ($userId) {
$user = $dbForConsole->getDocument('users', $userId);
$statsPerProject['email'] = $user->getAttribute('email', null);
$statsPerProject['name'] = $user->getAttribute('name', null);
}
}
/** Get Domains */
$statsPerProject['custom_domains'] = $dbForConsole->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
/** Get Platforms */
$platforms = $dbForConsole->find('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
$statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'web';
}));
$statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'android';
}));
$statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'apple');
}));
$statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'flutter');
}));
$flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX];
foreach ($flutterPlatforms as $flutterPlatform) {
$statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) {
return $platform['type'] === $flutterPlatform;
}));
}
$statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
/** Get Usage $statsPerProject */
$periods = [
'infinity' => [
'period' => '1d',
'limit' => 90,
],
'24h' => [
'period' => '1h',
'limit' => 24,
],
];
Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) {
foreach ($this->metrics as $key => $metric) {
foreach ($periods as $periodKey => $periodValue) {
$limit = $periodValue['limit'];
$period = $periodValue['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$statsPerProject[$key . '_' . $periodKey] = [];
foreach ($requestDocs as $requestDoc) {
$statsPerProject[$key . '_' . $periodKey][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]);
// Calculate aggregate of each metric
$statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value'));
}
}
});
if (isset($statsPerProject['email'])) {
/** Send data to mixpanel */
$res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
'name' => $statsPerProject['name'],
'email' => $statsPerProject['email']
]);
if (!$res) {
Console::error('Failed to create user profile for project: ' . $project->getId());
}
}
$event = new Event();
$event
->setName('Project Daily Usage')
->setProps($statsPerProject);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
Console::error('Failed to create event for project: ' . $project->getId());
}
} catch (Exception $e) {
Console::error('Failed to send stats for project: ' . $project->getId());
Console::error($e->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
$queue->reclaim();
});
}
@ -305,6 +79,8 @@ class Hamster extends Action
$next->setTimezone(new \DateTimeZone(date_default_timezone_get()));
$delay = $next->getTimestamp() - $now->getTimestamp();
$delay = 5;
/**
* If time passed for the target day.
*/
@ -323,17 +99,17 @@ class Hamster extends Action
/* Initialise new Utopia app */
$app = new App('UTC');
Console::info('Getting stats for all projects');
$this->getStatsPerProject($pools, $cache, $dbForConsole);
Console::success('Completed getting stats for all projects');
Console::info('Queuing stats for all projects');
$this->getStatsPerProject($pools, $dbForConsole);
Console::success('Completed queuing stats for all projects');
Console::info('Getting stats for all organizations');
$this->getStatsPerOrganization($dbForConsole);
Console::success('Completed getting stats for all organizations');
Console::info('Queuing stats for all organizations');
$this->getStatsPerOrganization($pools, $dbForConsole);
Console::success('Completed queuing stats for all organizations');
Console::info('Getting stats for all users');
$this->getStatsPerUser($dbForConsole);
Console::success('Completed getting stats for all users');
Console::info('Queuing stats for all users');
$this->getStatsPerUser($pools, $dbForConsole);
Console::success('Completed queuing stats for all users');
$pools
->get('console')
@ -378,96 +154,43 @@ class Hamster extends Action
Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
protected function getStatsPerOrganization(Database $dbForConsole)
protected function getStatsPerOrganization(Group $pools, Database $dbForConsole)
{
$this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) {
$this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($pools) {
try {
$statsPerOrganization = [];
/** Organization name */
$statsPerOrganization['name'] = $document->getAttribute('name');
/** Get Email and of the organization owner */
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$document->getInternalId()]),
]);
if (!$membership || $membership->isEmpty()) {
throw new Exception('Membership not found. Skipping organization : ' . $document->getId());
}
$userId = $membership->getAttribute('userId', null);
if ($userId) {
$user = $dbForConsole->getDocument('users', $userId);
$statsPerOrganization['email'] = $user->getAttribute('email', null);
}
/** Organization Creation Date */
$statsPerOrganization['created'] = $document->getAttribute('$createdAt');
/** Number of team members */
$statsPerOrganization['members'] = $document->getAttribute('total');
/** Number of projects in this organization */
$statsPerOrganization['projects'] = $dbForConsole->count('projects', [
Query::equal('teamId', [$document->getId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerOrganization['email'])) {
throw new Exception('Email not found. Skipping organization : ' . $document->getId());
}
$event = new Event();
$event
->setName('Organization Daily Usage')
->setProps($statsPerOrganization);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new Exception('Failed to create event for organization : ' . $document->getId());
}
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
$hamsterTask = new EventHamster($connection);
$hamsterTask
->setType('organization')
->setOrganization($organization)
->trigger();
$queue->reclaim();
} catch (Exception $e) {
Console::error($e->getMessage());
}
});
}
protected function getStatsPerUser(Database $dbForConsole)
protected function getStatsPerUser(Group $pools, Database $dbForConsole)
{
$this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) {
$this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($pools) {
try {
$statsPerUser = [];
/** Organization name */
$statsPerUser['name'] = $document->getAttribute('name');
/** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */
$statsPerUser['email'] = $document->getAttribute('email');
/** Organization Creation Date */
$statsPerUser['created'] = $document->getAttribute('$createdAt');
/** Number of teams this user is a part of */
$statsPerUser['memberships'] = $dbForConsole->count('memberships', [
Query::equal('userInternalId', [$document->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerUser['email'])) {
throw new Exception('User has no email: ' . $document->getId());
}
/** Send data to mixpanel */
$event = new Event();
$event
->setName('User Daily Usage')
->setProps($statsPerUser);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new Exception('Failed to create user profile for user: ' . $document->getId());
}
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
$hamsterTask = new EventHamster($connection);
$hamsterTask
->setType('user')
->setUser($user)
->trigger();
$queue->reclaim();
} catch (Exception $e) {
Console::error($e->getMessage());
}

454
src/Appwrite/Platform/Workers/Hamster.php Normal file
View file

@ -0,0 +1,454 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Usage\Stats;
use Appwrite\Event\Event;
use Appwrite\Event\Hamster as EventHamster;
use Appwrite\Network\Validator\Origin;
use Utopia\Analytics\Adapter\Mixpanel;
use Utopia\Analytics\Event as AnalyticsEvent;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Platform\Action;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Document;
use Utopia\Queue\Message;
use Utopia\Logger\Log;
use Utopia\Pools\Group;
class Hamster extends Action
{
private array $metrics = [
'usage_files' => 'files.$all.count.total',
'usage_buckets' => 'buckets.$all.count.total',
'usage_databases' => 'databases.$all.count.total',
'usage_documents' => 'documents.$all.count.total',
'usage_collections' => 'collections.$all.count.total',
'usage_storage' => 'project.$all.storage.size',
'usage_requests' => 'project.$all.network.requests',
'usage_bandwidth' => 'project.$all.network.bandwidth',
'usage_users' => 'users.$all.count.total',
'usage_sessions' => 'sessions.email.requests.create',
'usage_executions' => 'executions.$all.compute.total',
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
protected Mixpanel $mixpanel;
public static function getName(): string
{
return 'hamster';
}
/**
* @throws \Exception
*/
public function __construct()
{
$this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', ''));
$this
->desc('Hamster worker')
->inject('message')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('queueForHamster')
->inject('queueForEvents')
->inject('usage')
->inject('log')
->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole, EventHamster $queueForHamster, Event $queueForEvents, Stats $usage, Log $log) => $this->action($message, $pools, $cache, $dbForConsole, $queueForHamster, $queueForEvents, $usage, $log));
}
/**
* @param Message $message
* @param Group $pools
* @param Cache $cache
* @param Database $dbForConsole
* @param EventHamster $queueForHamster
* @param Event $queueForEvents
* @param Stats $usage
* @param Log $log
* @return void
* @throws Authorization
* @throws Structure
* @throws \Utopia\Database\Exception
* @throws Conflict
*/
public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole, EventHamster $queueForHamster, Event $queueForEvents, Stats $usage, Log $log): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new \Exception('Missing payload');
}
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new \Exception('Missing payload');
}
$type = $payload['type'] ?? '';
switch ($type) {
case 'project':
$this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole, $log);
break;
case 'organization':
$this->getStatsForOrganization(new Document($payload['organization']), $pools, $cache, $dbForConsole, $log);
break;
case 'user':
$this->getStatsPerUser(new Document($payload['user']), $dbForConsole);
break;
}
}
/**
* @param Document $project
* @param Group $pools
* @param Cache $cache
* @param Database $dbForConsole
* @param Log $log
* @throws \Utopia\Database\Exception
*/
private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole, Log $log): void
{
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
Console::info("Skipping project console");
return;
}
Console::log("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$statsPerProject = [];
$statsPerProject['time'] = microtime(true);
/** Get Project ID */
$statsPerProject['project_id'] = $project->getId();
/** Get project created time */
$statsPerProject['project_created'] = $project->getAttribute('$createdAt');
/** Get Project Name */
$statsPerProject['project_name'] = $project->getAttribute('name');
/** Total Project Variables */
$statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT);
/** Total Migrations */
$statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT);
/** Get Custom SMTP */
$smtp = $project->getAttribute('smtp', null);
if ($smtp) {
$statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled';
/** Get Custom Templates Count */
$templates = array_keys($project->getAttribute('templates', []));
$statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) {
return str_contains($template, 'email');
});
$statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) {
return str_contains($template, 'sms');
});
}
/** Get total relationship attributes */
$statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [
Query::equal('type', ['relationship'])
], APP_LIMIT_COUNT);
/** Get Total Functions */
$statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
foreach (\array_keys(Config::getParam('runtimes')) as $runtime) {
$statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [
Query::equal('runtime', [$runtime]),
], APP_LIMIT_COUNT);
}
/** Get Total Deployments */
$statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
$statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [
Query::equal('type', ['manual'])
], APP_LIMIT_COUNT);
$statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [
Query::equal('type', ['vcs'])
], APP_LIMIT_COUNT);
/** Get VCS repos connected */
$statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [
Query::equal('projectInternalId', [$project->getInternalId()])
], APP_LIMIT_COUNT);
/** Get Total Teams */
$statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
], APP_LIMIT_COUNT);
} else {
$statsPerProject['custom_organization_members'] = 0;
}
/** Get Email and Name of the project owner */
if ($teamInternalId) {
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$teamInternalId]),
]);
if (!$membership || $membership->isEmpty()) {
throw new \Exception('Membership not found. Skipping project : ' . $project->getId());
}
$userId = $membership->getAttribute('userId', null);
if ($userId) {
$user = $dbForConsole->getDocument('users', $userId);
$statsPerProject['email'] = $user->getAttribute('email', null);
$statsPerProject['name'] = $user->getAttribute('name', null);
}
}
/** Get Domains */
$statsPerProject['custom_domains'] = $dbForConsole->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
/** Get Platforms */
$platforms = $dbForConsole->find('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
$statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'web';
}));
$statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'android';
}));
$statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'apple');
}));
$statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'flutter');
}));
$flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX];
foreach ($flutterPlatforms as $flutterPlatform) {
$statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) {
return $platform['type'] === $flutterPlatform;
}));
}
$statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
/** Get Usage $statsPerProject */
$periods = [
'infinity' => [
'period' => '1d',
'limit' => 90,
],
'24h' => [
'period' => '1h',
'limit' => 24,
],
];
Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) {
foreach ($this->metrics as $key => $metric) {
foreach ($periods as $periodKey => $periodValue) {
$limit = $periodValue['limit'];
$period = $periodValue['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$statsPerProject[$key . '_' . $periodKey] = [];
foreach ($requestDocs as $requestDoc) {
$statsPerProject[$key . '_' . $periodKey][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]);
// Calculate aggregate of each metric
$statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value'));
}
}
});
if (isset($statsPerProject['email'])) {
/** Send data to mixpanel */
$res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
'name' => $statsPerProject['name'],
'email' => $statsPerProject['email']
]);
if (!$res) {
Console::error('Failed to create user profile for project: ' . $project->getId());
}
}
$event = new AnalyticsEvent();
$event
->setName('Project Daily Usage')
->setProps($statsPerProject);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
Console::error('Failed to create event for project: ' . $project->getId());
}
} catch (\Exception $e) {
Console::error('Failed to send stats for project: ' . $project->getId());
Console::error($e->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
/**
* @param Document $organization
* @param Group $pools
* @param Cache $cache
* @param Database $dbForConsole
* @param Log $log
* @throws \Utopia\Database\Exception
*/
private function getStatsForOrganization(Document $organization, Group $pools, Cache $cache, Database $dbForConsole, Log $log): void
{
try {
$statsPerOrganization = [];
/** Organization name */
$statsPerOrganization['name'] = $organization->getAttribute('name');
/** Get Email and of the organization owner */
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$organization->getInternalId()]),
]);
if (!$membership || $membership->isEmpty()) {
throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId());
}
$userId = $membership->getAttribute('userId', null);
if ($userId) {
$user = $dbForConsole->getDocument('users', $userId);
$statsPerOrganization['email'] = $user->getAttribute('email', null);
}
/** Organization Creation Date */
$statsPerOrganization['created'] = $organization->getAttribute('$createdAt');
/** Number of team members */
$statsPerOrganization['members'] = $organization->getAttribute('total');
/** Number of projects in this organization */
$statsPerOrganization['projects'] = $dbForConsole->count('projects', [
Query::equal('teamId', [$organization->getId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerOrganization['email'])) {
throw new \Exception('Email not found. Skipping organization : ' . $organization->getId());
}
$event = new AnalyticsEvent();
$event
->setName('Organization Daily Usage')
->setProps($statsPerOrganization);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new \Exception('Failed to create event for organization : ' . $organization->getId());
}
} catch (\Exception $e) {
Console::error($e->getMessage());
}
}
protected function getStatsPerUser(Document $user, Database $dbForConsole)
{
try {
$statsPerUser = [];
/** Organization name */
$statsPerUser['name'] = $user->getAttribute('name');
/** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */
$statsPerUser['email'] = $user->getAttribute('email');
/** Organization Creation Date */
$statsPerUser['created'] = $user->getAttribute('$createdAt');
/** Number of teams this user is a part of */
$statsPerUser['memberships'] = $dbForConsole->count('memberships', [
Query::equal('userInternalId', [$user->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerUser['email'])) {
throw new \Exception('User has no email: ' . $user->getId());
}
/** Send data to mixpanel */
$event = new AnalyticsEvent();
$event
->setName('User Daily Usage')
->setProps($statsPerUser);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new \Exception('Failed to create user profile for user: ' . $user->getId());
}
} catch (\Exception $e) {
Console::error($e->getMessage());
}
}
}