CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-09-28 07:21:35 +12:00
Code Issues Projects Releases Wiki Activity

new metrics aggregation

Browse source
This commit is contained in:
Damodar Lohani 2022-08-08 01:33:34 +00:00
parent 73817a311b
commit 6ea4098679
3 changed files with 226 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
composer.lock generated
View file

@ -5370,5 +5370,5 @@
"platform-overrides": {
"php": "8.0"
},
"plugin-api-version": "2.2.0"
"plugin-api-version": "2.3.0"
}

30
src/Appwrite/Stats/Usage.php
View file

@ -188,17 +188,9 @@ class Usage
],
];
protected array $periods = [
[
'key' => '30m',
'multiplier' => 1800,
'startTime' => '-24 hours',
],
[
'key' => '1d',
'multiplier' => 86400,
'startTime' => '-90 days',
],
protected array $period = [
'key' => '30m',
'startTime' => '-24 hours',
];
public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null)
@ -342,15 +334,13 @@ class Usage
public function collect(): void
{
foreach ($this->metrics as $metric => $options) { //for each metrics
foreach ($this->periods as $period) { // aggregate data for each period
try {
$this->syncFromInfluxDB($metric, $options, $period);
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e);
} else {
throw $e;
}
try {
$this->syncFromInfluxDB($metric, $options, $this->period);
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e);
} else {
throw $e;
}
}
}

230
src/Appwrite/Stats/UsageDB.php
View file

@ -4,14 +4,45 @@ namespace Appwrite\Stats;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
class UsageDB extends Usage
{
protected array $periods = [
[
'key' => '30m',
'multiplier' => 1800,
],
[
'key' => '1d',
'multiplier' => 86400,
],
];
public function __construct(Database $database, callable $errorHandler = null)
{
$this->database = $database;
$this->errorHandler = $errorHandler;
}
/**
* Create Per Period Metric
* Create given metric for each defined period
*
* @param string $projectId
* @param string $metric
* @param int $value
*
* @return void
*/
private function createPerPeriodMetric(string $projectId, string $metric, int $value): void
{
foreach ($this->periods as $options) {
$period = $options['key'];
$time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']);
$this->createOrUpdateMetric($projectId, $metric, $period, $time, $value);
}
}
/**
* Create or Update Mertic
* Create or update each metric in the stats collection for the given project
@ -22,11 +53,8 @@ class UsageDB extends Usage
*
* @return void
*/
private function createOrUpdateMetric(string $projectId, string $metric, int $value): void
private function createOrUpdateMetric(string $projectId, string $metric, string $period, int $time, int $value): void
{
foreach ($this->periods as $options) {
$period = $options['key'];
$time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']);
$id = \md5("{$time}_{$period}_{$metric}");
$this->database->setNamespace('_' . $projectId);
@ -55,7 +83,6 @@ class UsageDB extends Usage
throw $e;
}
}
}
}
/**
@ -114,13 +141,13 @@ class UsageDB extends Usage
*
* @return int
*/
private function sum(string $projectId, string $collection, string $attribute, string $metric): int
private function sum(string $projectId, string $collection, string $attribute, string $metric, array $queries = []): int
{
$this->database->setNamespace('_' . $projectId);
try {
$sum = (int) $this->database->sum($collection, $attribute);
$this->createOrUpdateMetric($projectId, $metric, $sum);
$sum = (int) $this->database->sum($collection, $attribute, $queries);
$this->createPerPeriodMetric($projectId, $metric, $sum);
return $sum;
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
@ -147,7 +174,7 @@ class UsageDB extends Usage
try {
$count = $this->database->count($collection);
$this->createOrUpdateMetric($projectId, $metric, $count);
$this->createPerPeriodMetric($projectId, $metric, $count);
return $count;
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
@ -214,10 +241,10 @@ class UsageDB extends Usage
$projectFilesTotal += $sum;
});
$this->createOrUpdateMetric($projectId, 'storage.files.count', $projectFilesCount);
$this->createOrUpdateMetric($projectId, 'storage.files.total', $projectFilesTotal);
$this->createPerPeriodMetric($projectId, 'storage.files.count', $projectFilesCount);
$this->createPerPeriodMetric($projectId, 'storage.files.total', $projectFilesTotal);
$this->createOrUpdateMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal);
$this->createPerPeriodMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal);
}
/**
@ -251,11 +278,176 @@ class UsageDB extends Usage
$databaseDocumentsCount += $count;
});
$this->createOrUpdateMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount);
$this->createPerPeriodMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount);
});
$this->createOrUpdateMetric($projectId, 'databases.collections.count', $projectCollectionsCount);
$this->createOrUpdateMetric($projectId, 'databases.documents.count', $projectDocumentsCount);
$this->createPerPeriodMetric($projectId, 'databases.collections.count', $projectCollectionsCount);
$this->createPerPeriodMetric($projectId, 'databases.documents.count', $projectDocumentsCount);
}
protected function aggregateDatabaseMetrics(string $projectId): void
{
$this->database->setNamespace('_' . $projectId);
$databasesGeneralMetrics = [
'databases.create',
'databases.read',
'databases.update',
'databases.delete',
'databases.collections.create',
'databases.collections.read',
'databases.collections.update',
'databases.collections.delete',
'databases.documents.create',
'databases.documents.read',
'databases.documents.update',
'databases.documents.delete',
];
foreach($databasesGeneralMetrics as $metric) {
$this->aggregateDailyMetric($projectId, $metric);
}
$databasesDatabaseMetrics = [
'databases.databaseId.collections.create',
'databases.databaseId.collections.read',
'databases.databaseId.collections.update',
'databases.databaseId.collections.delete',
'databases.databaseId.documents.create',
'databases.databaseId.documents.read',
'databases.databaseId.documents.update',
'databases.databaseId.documents.delete',
];
$this->foreachDocument($projectId, 'databases', [], function(Document $database) use ($databasesDatabaseMetrics, $projectId) {
$databaseId = $database->getId();
foreach ($databasesDatabaseMetrics as $metric) {
$metric = str_replace('databaseId', $databaseId, $metric);
$this->aggregateDailyMetric($projectId, $metric);
}
$databasesCollectionMetrics = [
'databases.' . $databaseId . '.collections.collectionId.documents.create',
'databases.' . $databaseId . '.collections.collectionId.documents.read',
'databases.' . $databaseId . '.collections.collectionId.documents.update',
'databases.' . $databaseId . '.collections.collectionId.documents.delete',
];
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function(Document $collection) use ($databasesCollectionMetrics, $projectId) {
$collectionId = $collection->getId();
foreach ($databasesCollectionMetrics as $metric) {
$metric = str_replace('collectionId', $collectionId, $metric);
$this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($projectId, $metric);
}
});
});
}
protected function aggregateStorageMetrics(string $projectId): void
{
$this->database->setNamespace('_' . $projectId);
$storageGeneralMetrics = [
'storage.buckets.create',
'storage.buckets.read',
'storage.buckets.update',
'storage.buckets.delete',
'storage.files.create',
'storage.files.read',
'storage.files.update',
'storage.files.delete',
];
foreach($storageGeneralMetrics as $metric) {
$this->aggregateDailyMetric($projectId, $metric);
}
$storageBucketMetrics = [
'storage.buckets.bucketId.files.create',
'storage.buckets.bucketId.files.read',
'storage.buckets.bucketId.files.update',
'storage.buckets.bucketId.files.delete',
];
$this->foreachDocument($projectId, 'buckets', [], function(Document $bucket) use ($storageBucketMetrics, $projectId) {
$bucketId = $bucket->getId();
foreach ($storageBucketMetrics as $metric) {
$metric = str_replace('bucketId', $bucketId, $metric);
$this->aggregateDailyMetric($projectId, $metric);
}
});
}
protected function aggregateFunctionMetrics(string $projectId): void
{
$this->database->setNamespace('_' . $projectId);
$this->aggregateDailyMetric($projectId, 'executions');
$functionMetrics = [
'functions.functionId.executions',
'functions.functionId.compute',
'function.functionId.failure',
];
$this->foreachDocument($projectId, 'functions', [], function(Document $function) use ($functionMetrics, $projectId) {
$functionId = $function->getId();
foreach ($functionMetrics as $metric) {
$metric = str_replace('functionId', $functionId, $metric);
$this->aggregateDailyMetric($projectId, $metric);
}
});
}
protected function aggregateUsersMetrics(string $projectId): void
{
$metrics = [
'users.create',
'users.read',
'users.update',
'users.delete',
'users.sessions.create',
'users.sessions.delete'
];
foreach($metrics as $metric) {
$this->aggregateDailyMetric($projectId, $metric);
}
}
protected function aggregateGeneralMetrics(string $projectId): void
{
$this->aggregateDailyMetric($projectId, 'requests');
$this->aggregateDailyMetric($projectId, 'network');
}
protected function aggregateDailyMetric(string $projectId, string $metric): void
{
$beginOfDay = strtotime("today");
$endOfDay = strtotime("tomorrow", $beginOfDay) - 1;
$this->database->setNamespace('_' . $projectId);
$value = (int) $this->database->sum('stats', 'value', [
new Query('metric', Query::TYPE_EQUAL, [$metric]),
new Query('period', Query::TYPE_EQUAL, ['30m']),
new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfDay]),
new Query('time', Query::TYPE_LESSEREQUAL, [$endOfDay]),
]);
$this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value);
}
protected function aggregateMonthlyMetric(string $projectId, string $metric): void
{
$beginOfMonth = strtotime("first day of the month");
$endOfMonth = strtotime("last day of the month");
$this->database->setNamespace('_' . $projectId);
$value = (int) $this->database->sum('stats', 'value', [
new Query('metric', Query::TYPE_EQUAL, [$metric]),
new Query('period', Query::TYPE_EQUAL, ['1d']),
new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfMonth]),
new Query('time', Query::TYPE_LESSEREQUAL, [$endOfMonth]),
]);
$this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value);
}
/**
@ -272,6 +464,14 @@ class UsageDB extends Usage
$this->usersStats($projectId);
$this->databaseStats($projectId);
$this->storageStats($projectId);
// Aggregate new metrics from already collected usage metrics
// for lower time period (1day and 1 month metric from 30 minute metrics)
$this->aggregateGeneralMetrics($projectId);
$this->aggregateFunctionMetrics($projectId);
$this->aggregateDatabaseMetrics($projectId);
$this->aggregateStorageMetrics($projectId);
$this->aggregateUsersMetrics($projectId);
});
}
}