1
0
Fork 0
mirror of synced 2024-06-14 00:34:51 +12:00
appwrite/src/Appwrite/Platform/Workers/Usage.php

248 lines
11 KiB
PHP
Raw Normal View History

2023-10-25 21:38:44 +13:00
<?php
namespace Appwrite\Platform\Workers;
use Exception;
use Appwrite\Event\UsageDump;
2024-02-08 21:26:45 +13:00
use Utopia\App;
2023-10-25 21:38:44 +13:00
use Utopia\CLI\Console;
2024-01-19 05:52:20 +13:00
use Utopia\Database\DateTime;
2023-10-25 21:38:44 +13:00
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
class Usage extends Action
{
2024-01-28 22:28:59 +13:00
private array $stats = [];
private int $lastTriggeredTime = 0;
private int $keys = 0;
private const INFINITY_PERIOD = '_inf_';
2024-02-08 21:26:45 +13:00
private const KEYS_THRESHOLD = 10000;
2024-01-28 22:28:59 +13:00
2023-10-25 21:38:44 +13:00
public static function getName(): string
{
return 'usage';
}
/**
* @throws Exception
*/
public function __construct()
{
$this
2023-12-25 07:38:15 +13:00
->desc('Usage worker')
->inject('message')
->inject('getProjectDB')
2024-01-28 22:28:59 +13:00
->inject('queueForUsageDump')
->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) {
$this->action($message, $getProjectDB, $queueForUsageDump);
2023-12-25 07:38:15 +13:00
});
2024-01-28 22:28:59 +13:00
$this->lastTriggeredTime = time();
2023-10-25 21:38:44 +13:00
}
/**
* @param Message $message
2023-12-25 07:38:15 +13:00
* @param callable $getProjectDB
2024-01-28 22:28:59 +13:00
* @param UsageDump $queueForUsageDump
2023-10-25 21:38:44 +13:00
* @return void
* @throws \Utopia\Database\Exception
2023-12-25 07:38:15 +13:00
* @throws Exception
2023-10-25 21:38:44 +13:00
*/
2024-01-28 22:28:59 +13:00
public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void
2023-10-25 21:38:44 +13:00
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
2024-02-08 21:26:45 +13:00
//Todo Figure out way to preserve keys when the container is being recreated @shimonewman
2023-10-25 21:38:44 +13:00
2024-02-09 07:51:54 +13:00
$aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
2023-10-25 21:38:44 +13:00
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
continue;
}
$this->reduce(
2023-12-25 07:38:15 +13:00
project: $project,
2023-10-25 21:38:44 +13:00
document: new Document($document),
metrics: $payload['metrics'],
2023-12-25 07:38:15 +13:00
getProjectDB: $getProjectDB
2023-10-25 21:38:44 +13:00
);
}
2024-01-28 22:28:59 +13:00
$this->stats[$projectId]['project'] = $project;
2023-10-25 21:38:44 +13:00
foreach ($payload['metrics'] ?? [] as $metric) {
2024-01-28 22:28:59 +13:00
$this->keys++;
if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
$this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
2023-10-25 21:38:44 +13:00
continue;
}
2024-01-28 22:28:59 +13:00
$this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
2023-10-25 21:38:44 +13:00
}
2024-02-10 00:22:48 +13:00
// If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
2024-01-28 22:28:59 +13:00
if (
$this->keys >= self::KEYS_THRESHOLD ||
2024-02-08 21:26:45 +13:00
(time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0)
2024-01-28 22:28:59 +13:00
) {
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
2024-02-09 08:08:29 +13:00
2024-01-28 22:28:59 +13:00
$queueForUsageDump
2024-01-31 00:24:57 +13:00
->setStats($this->stats)
2024-01-28 22:28:59 +13:00
->trigger();
2024-01-31 00:24:57 +13:00
$this->stats = [];
2024-01-28 22:28:59 +13:00
$this->keys = 0;
$this->lastTriggeredTime = time();
2023-10-25 21:38:44 +13:00
}
}
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
* When we remove a parent document we need to deduct his children aggregation from the project scope.
2023-12-25 07:38:15 +13:00
* @param Document $project
2023-10-25 21:38:44 +13:00
* @param Document $document
* @param array $metrics
2023-12-25 07:38:15 +13:00
* @param callable $getProjectDB
2023-10-25 21:38:44 +13:00
* @return void
*/
2023-12-25 07:38:15 +13:00
private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
2023-10-25 21:38:44 +13:00
{
2023-12-25 07:38:15 +13:00
$dbForProject = $getProjectDB($project);
2023-10-25 21:38:44 +13:00
2023-12-25 07:38:15 +13:00
try {
2023-10-25 21:38:44 +13:00
switch (true) {
case $document->getCollection() === 'users': // users
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
if (!empty($sessions)) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_SESSIONS,
'value' => ($sessions * -1),
2023-10-25 21:38:44 +13:00
];
}
break;
case $document->getCollection() === 'databases': // databases
2024-02-01 23:21:50 +13:00
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
2023-10-25 21:38:44 +13:00
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
'value' => ($collections['value'] * -1),
];
}
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
2024-02-01 23:21:50 +13:00
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
2023-10-25 21:38:44 +13:00
if (!empty($documents['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
2023-10-25 21:38:44 +13:00
];
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
'value' => ($documents['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
break;
case $document->getCollection() === 'buckets':
2024-02-01 23:21:50 +13:00
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
2023-10-25 21:38:44 +13:00
if (!empty($files['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_FILES,
'value' => ($files['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
if (!empty($storage['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_FILES_STORAGE,
'value' => ($storage['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
break;
case $document->getCollection() === 'functions':
2024-02-01 23:21:50 +13:00
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
2023-10-25 21:38:44 +13:00
if (!empty($deployments['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_DEPLOYMENTS,
'value' => ($deployments['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
if (!empty($deploymentsStorage['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_DEPLOYMENTS_STORAGE,
'value' => ($deploymentsStorage['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
if (!empty($builds['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_BUILDS,
'value' => ($builds['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_BUILDS_STORAGE,
'value' => ($buildsStorage['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
if (!empty($buildsCompute['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_BUILDS_COMPUTE,
'value' => ($buildsCompute['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
if (!empty($executions['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_EXECUTIONS,
'value' => ($executions['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
if (!empty($executionsCompute['value'])) {
$metrics[] = [
2023-12-25 07:38:15 +13:00
'key' => METRIC_EXECUTIONS_COMPUTE,
'value' => ($executionsCompute['value'] * -1),
2023-10-25 21:38:44 +13:00
];
}
break;
default:
break;
}
} catch (\Throwable $e) {
2023-12-25 07:38:15 +13:00
console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
2023-10-25 21:38:44 +13:00
}
}
}