CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-17 11:12:41 +12:00
Code Issues Projects Releases Wiki Activity

calc tier stats

Browse source
This commit is contained in:
shimon 2023-05-30 18:06:51 +03:00
parent 37f690c3bd
commit 1225c8b7c3
5 changed files with 301 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -120,7 +120,7 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
chmod +x /usr/local/bin/clear-card-cache && \
chmod +x /usr/local/bin/calc-users-stats && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/calc-tier-stats && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/usage && \
chmod +x /usr/local/bin/install && \

2
app/console

@ -1 +1 @@
Subproject commit 3a0c8f0334d402a6e27c8b5c0512f3d60080ddbd
Subproject commit 2fac7c1f390637f7dfc11882dcad6fb6508c83f1

3
bin/calc-tier-stats Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php calc-tier-stats $@

5
src/Appwrite/Platform/Services/Tasks.php
View file

@ -20,6 +20,7 @@ use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
use Appwrite\Platform\Tasks\CalcUsersStats;
use Appwrite\Platform\Tasks\CalcTierStats;
class Tasks extends Service
{
@ -43,6 +44,8 @@ class Tasks extends Service
->addAction(SDKs::getName(), new SDKs())
->addAction(VolumeSync::getName(), new VolumeSync())
->addAction(Specs::getName(), new Specs())
->addAction(CalcUsersStats::getName(), new CalcUsersStats());
->addAction(CalcUsersStats::getName(), new CalcUsersStats())
->addAction(CalctierStats::getName(), new CalcTierStats())
;
}
}

292
src/Appwrite/Platform/Tasks/CalcTierStats.php Normal file
View file

@ -0,0 +1,292 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use Utopia\App;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
class CalcTierStats extends Action
{
private array $columns = [
'Project ID',
'Organization ID',
'Users',
'Teams',
'Requests',
'Bandwidth',
'Domains',
'Webhooks',
'Platforms',
'Buckets',
'Files',
'Storage',
'Max file size',
'Databases',
'Functions',
'Deployments',
'Executions',
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
private array $usageStats = [
'project.$all.network.requests' => 'Requests',
'project.$all.network.bandwidth' => 'Bandwidth',
];
public static function getName(): string
{
return 'calc-tier-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
});
}
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
//docker compose exec -t appwrite calc-tier-stats
Console::title('Cloud free tier stats calculation V1');
Console::success(APP_NAME . ' cloud free tier stats calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
///** Get Project Name */
//$stats['Project Name'] = $project->getAttribute('name');
/** Get Organization Name and Id */
//$teamId = $project->getAttribute('teamId', null);
//$teamName = null;
//if ($teamId) {
//$team = $dbForConsole->getDocument('teams', $teamId);
//$teamName = $team->getAttribute('name');
// }
$stats['Organization ID'] = $project->getAttribute('teamInternalId', null);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$stats['Users'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
]);
} else {
$stats['Users'] = 0;
}
/** Get Total internal Teams */
$stats['Teams'] = $dbForProject->count('teams', []);
/** Get Usage stats */
$range = '90d';
$periods = [
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$tmp = [];
$metrics = $this->usageStats;
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
foreach ($metrics as $metric => $name) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$tmp[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$tmp[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$tmp[$metric] = array_reverse($tmp[$metric]);
$tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
}
});
foreach ($tmp as $key => $value) {
$stats[$metrics[$key]] = $value;
}
/** Get Domains */
$stats['Domains'] = $dbForConsole->count('domains', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
/** Get Webhooks */
$stats['Webhooks'] = $dbForConsole->count('webhooks', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
/** Get Platforms */
$stats['Platforms'] = $dbForConsole->count('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
/** Get Files & Buckets */
$filesCount = 0;
$filesSum = 0;
$maxFileSize = 0;
$buckets = $dbForProject->find('buckets', []);
$counter = 0;
foreach ($buckets as $bucket) {
$filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', [], 0);
$filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
$file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
$maxFileSize = $file->getAttribute('sizeOriginal');
}
$counter++;
}
$stats['Buckets'] = $counter;
$stats['Files'] = $filesCount;
$stats['Storage'] = $filesSum;
$stats['Max file size'] = $maxFileSize;
/** Get Total Functions */
$stats['Databases'] = $dbForProject->count('databases', []);
/** Get Total Functions */
$stats['Functions'] = $dbForProject->count('functions', []);
/** Get Total Deployments */
$stats['Deployments'] = $dbForProject->count('deployments', []);
/** Get Total Executions */
$stats['Executions'] = $dbForProject->count('executions', []);
$csv->insertOne(array_values($stats));
} catch (\Throwable $th) {
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
//
// /** @var PHPMailer $mail */
// $mail = $register->get('smtp');
//
// $mail->clearAddresses();
// $mail->clearAllRecipients();
// $mail->clearReplyTos();
// $mail->clearAttachments();
// $mail->clearBCCs();
// $mail->clearCCs();
//
// try {
// /** Addresses */
// $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
// $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
//
// foreach ($recipients as $recipient) {
// $mail->addAddress($recipient);
// }
//
// /** Attachments */
// $mail->addAttachment($this->path);
//
// /** Content */
// $mail->Subject = "Cloud Report for {$this->date}";
// $mail->Body = "Please find the daily cloud report atttached";
// $mail->send();
// Console::success('Email has been sent!');
// } catch (Exception $e) {
// Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
// }
}
}