usage refactor setups
This commit is contained in:
parent
d7d3d6c808
commit
7affd0579e
6 changed files with 715 additions and 687 deletions
|
@ -1,512 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace Appwrite\Stats;
|
|
||||||
|
|
||||||
use Utopia\Database\Database;
|
|
||||||
use Utopia\Database\Document;
|
|
||||||
use Utopia\Database\Query;
|
|
||||||
|
|
||||||
class UsageDB extends Usage
|
|
||||||
{
|
|
||||||
protected array $periods = [
|
|
||||||
[
|
|
||||||
'key' => '30m',
|
|
||||||
'multiplier' => 1800,
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'key' => '1d',
|
|
||||||
'multiplier' => 86400,
|
|
||||||
],
|
|
||||||
];
|
|
||||||
|
|
||||||
public function __construct(Database $database, callable $errorHandler = null)
|
|
||||||
{
|
|
||||||
$this->database = $database;
|
|
||||||
$this->errorHandler = $errorHandler;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Create Per Period Metric
|
|
||||||
* Create given metric for each defined period
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
* @param string $metric
|
|
||||||
* @param int $value
|
|
||||||
*
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
private function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void
|
|
||||||
{
|
|
||||||
foreach ($this->periods as $options) {
|
|
||||||
$period = $options['key'];
|
|
||||||
$time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']);
|
|
||||||
$this->createOrUpdateMetric($projectId, $metric, $period, $time, $value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Required for billing
|
|
||||||
if ($monthly) {
|
|
||||||
$time = strtotime("first day of the month");
|
|
||||||
$this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create or Update Mertic
|
|
||||||
* Create or update each metric in the stats collection for the given project
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
* @param string $metric
|
|
||||||
* @param int $value
|
|
||||||
*
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
private function createOrUpdateMetric(string $projectId, string $metric, string $period, int $time, int $value): void
|
|
||||||
{
|
|
||||||
$id = \md5("{$time}_{$period}_{$metric}");
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
try {
|
|
||||||
$document = $this->database->getDocument('stats', $id);
|
|
||||||
if ($document->isEmpty()) {
|
|
||||||
$this->database->createDocument('stats', new Document([
|
|
||||||
'$id' => $id,
|
|
||||||
'period' => $period,
|
|
||||||
'time' => $time,
|
|
||||||
'metric' => $metric,
|
|
||||||
'value' => $value,
|
|
||||||
'type' => 1,
|
|
||||||
]));
|
|
||||||
} else {
|
|
||||||
$this->database->updateDocument(
|
|
||||||
'stats',
|
|
||||||
$document->getId(),
|
|
||||||
$document->setAttribute('value', $value)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (\Exception$e) { // if projects are deleted this might fail
|
|
||||||
if (is_callable($this->errorHandler)) {
|
|
||||||
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
|
|
||||||
} else {
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Foreach Document
|
|
||||||
* Call provided callback for each document in the collection
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
* @param string $collection
|
|
||||||
* @param array $queries
|
|
||||||
* @param callable $callback
|
|
||||||
*
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
private function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
|
|
||||||
{
|
|
||||||
$limit = 50;
|
|
||||||
$results = [];
|
|
||||||
$sum = $limit;
|
|
||||||
$latestDocument = null;
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
while ($sum === $limit) {
|
|
||||||
try {
|
|
||||||
$results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument);
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
if (is_callable($this->errorHandler)) {
|
|
||||||
call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}");
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (empty($results)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
$sum = count($results);
|
|
||||||
|
|
||||||
foreach ($results as $document) {
|
|
||||||
if (is_callable($callback)) {
|
|
||||||
$callback($document);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$latestDocument = $results[array_key_last($results)];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sum
|
|
||||||
* Calculate sum of a attribute of documents in collection
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
* @param string $collection
|
|
||||||
* @param string $attribute
|
|
||||||
* @param string $metric
|
|
||||||
*
|
|
||||||
* @return int
|
|
||||||
*/
|
|
||||||
private function sum(string $projectId, string $collection, string $attribute, string $metric, int $multiplier = 1): int
|
|
||||||
{
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
try {
|
|
||||||
$sum = $this->database->sum($collection, $attribute);
|
|
||||||
$sum = (int) ($sum * $multiplier);
|
|
||||||
$this->createPerPeriodMetric($projectId, $metric, $sum);
|
|
||||||
return $sum;
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
if (is_callable($this->errorHandler)) {
|
|
||||||
call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}");
|
|
||||||
} else {
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Count
|
|
||||||
* Count number of documents in collection
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
* @param string $collection
|
|
||||||
* @param string $metric
|
|
||||||
*
|
|
||||||
* @return int
|
|
||||||
*/
|
|
||||||
private function count(string $projectId, string $collection, string $metric): int
|
|
||||||
{
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
try {
|
|
||||||
$count = $this->database->count($collection);
|
|
||||||
$this->createPerPeriodMetric($projectId, $metric, $count);
|
|
||||||
return $count;
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
if (is_callable($this->errorHandler)) {
|
|
||||||
call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}");
|
|
||||||
} else {
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Deployments Total
|
|
||||||
* Total sum of storage used by deployments
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
*
|
|
||||||
* @return int
|
|
||||||
*/
|
|
||||||
private function deploymentsTotal(string $projectId): int
|
|
||||||
{
|
|
||||||
return $this->sum($projectId, 'deployments', 'size', 'stroage.deployments.total');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Users Stats
|
|
||||||
* Metric: users.count
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
*
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
private function usersStats(string $projectId): void
|
|
||||||
{
|
|
||||||
$this->count($projectId, 'users', 'users.count');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Storage Stats
|
|
||||||
* Metrics: storage.total, storage.files.total, storage.buckets.{bucketId}.files.total,
|
|
||||||
* storage.buckets.count, storage.files.count, storage.buckets.{bucketId}.files.count
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
*
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
private function storageStats(string $projectId): void
|
|
||||||
{
|
|
||||||
$deploymentsTotal = $this->deploymentsTotal($projectId);
|
|
||||||
|
|
||||||
$projectFilesTotal = 0;
|
|
||||||
$projectFilesCount = 0;
|
|
||||||
|
|
||||||
$metric = 'storage.buckets.count';
|
|
||||||
$this->count($projectId, 'buckets', $metric);
|
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) {
|
|
||||||
$metric = "storage.buckets.{$bucket->getId()}.files.count";
|
|
||||||
|
|
||||||
$count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric);
|
|
||||||
$projectFilesCount += $count;
|
|
||||||
|
|
||||||
$metric = "storage.buckets.{$bucket->getId()}.files.total";
|
|
||||||
$sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric);
|
|
||||||
$projectFilesTotal += $sum;
|
|
||||||
});
|
|
||||||
|
|
||||||
$this->createPerPeriodMetric($projectId, 'storage.files.count', $projectFilesCount);
|
|
||||||
$this->createPerPeriodMetric($projectId, 'storage.files.total', $projectFilesTotal);
|
|
||||||
|
|
||||||
$this->createPerPeriodMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute Stats
|
|
||||||
* Metrics: functions.executionTime, functions.buildTime, functions.compute,
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
*
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
private function computeStats(string $projectId): void
|
|
||||||
{
|
|
||||||
$executionTotal = $this->sum($projectId, 'executions', 'time', 'functions.executionTime', 1000); // in ms
|
|
||||||
$buildTotal = $this->sum($projectId, 'builds', 'duration', 'functions.buildTime', 1000); // in ms
|
|
||||||
|
|
||||||
$this->createPerPeriodMetric($projectId, 'functions.compute', $executionTotal + $buildTotal); //in ms
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Database Stats
|
|
||||||
* Collect all database stats
|
|
||||||
* Metrics: database.collections.count, database.collections.{collectionId}.documents.count,
|
|
||||||
* database.documents.count
|
|
||||||
*
|
|
||||||
* @param string $projectId
|
|
||||||
*
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
private function databaseStats(string $projectId): void
|
|
||||||
{
|
|
||||||
$projectDocumentsCount = 0;
|
|
||||||
$projectCollectionsCount = 0;
|
|
||||||
|
|
||||||
$this->count($projectId, 'databases', 'databases.count');
|
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) {
|
|
||||||
$metric = "databases.{$database->getId()}.collections.count";
|
|
||||||
$count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric);
|
|
||||||
$projectCollectionsCount += $count;
|
|
||||||
$databaseDocumentsCount = 0;
|
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) {
|
|
||||||
$metric = "databases.{$database->getId()}.collections.{$collection->getId()}.documents.count";
|
|
||||||
|
|
||||||
$count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric);
|
|
||||||
$projectDocumentsCount += $count;
|
|
||||||
$databaseDocumentsCount += $count;
|
|
||||||
});
|
|
||||||
|
|
||||||
$this->createPerPeriodMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount);
|
|
||||||
});
|
|
||||||
|
|
||||||
$this->createPerPeriodMetric($projectId, 'databases.collections.count', $projectCollectionsCount);
|
|
||||||
$this->createPerPeriodMetric($projectId, 'databases.documents.count', $projectDocumentsCount);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function aggregateDatabaseMetrics(string $projectId): void
|
|
||||||
{
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
$databasesGeneralMetrics = [
|
|
||||||
'databases.create',
|
|
||||||
'databases.read',
|
|
||||||
'databases.update',
|
|
||||||
'databases.delete',
|
|
||||||
'databases.collections.create',
|
|
||||||
'databases.collections.read',
|
|
||||||
'databases.collections.update',
|
|
||||||
'databases.collections.delete',
|
|
||||||
'databases.documents.create',
|
|
||||||
'databases.documents.read',
|
|
||||||
'databases.documents.update',
|
|
||||||
'databases.documents.delete',
|
|
||||||
];
|
|
||||||
|
|
||||||
foreach ($databasesGeneralMetrics as $metric) {
|
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
|
||||||
}
|
|
||||||
|
|
||||||
$databasesDatabaseMetrics = [
|
|
||||||
'databases.databaseId.collections.create',
|
|
||||||
'databases.databaseId.collections.read',
|
|
||||||
'databases.databaseId.collections.update',
|
|
||||||
'databases.databaseId.collections.delete',
|
|
||||||
'databases.databaseId.documents.create',
|
|
||||||
'databases.databaseId.documents.read',
|
|
||||||
'databases.databaseId.documents.update',
|
|
||||||
'databases.databaseId.documents.delete',
|
|
||||||
];
|
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) {
|
|
||||||
$databaseId = $database->getId();
|
|
||||||
foreach ($databasesDatabaseMetrics as $metric) {
|
|
||||||
$metric = str_replace('databaseId', $databaseId, $metric);
|
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
|
||||||
}
|
|
||||||
|
|
||||||
$databasesCollectionMetrics = [
|
|
||||||
'databases.' . $databaseId . '.collections.collectionId.documents.create',
|
|
||||||
'databases.' . $databaseId . '.collections.collectionId.documents.read',
|
|
||||||
'databases.' . $databaseId . '.collections.collectionId.documents.update',
|
|
||||||
'databases.' . $databaseId . '.collections.collectionId.documents.delete',
|
|
||||||
];
|
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) {
|
|
||||||
$collectionId = $collection->getId();
|
|
||||||
foreach ($databasesCollectionMetrics as $metric) {
|
|
||||||
$metric = str_replace('collectionId', $collectionId, $metric);
|
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function aggregateStorageMetrics(string $projectId): void
|
|
||||||
{
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
$storageGeneralMetrics = [
|
|
||||||
'storage.buckets.create',
|
|
||||||
'storage.buckets.read',
|
|
||||||
'storage.buckets.update',
|
|
||||||
'storage.buckets.delete',
|
|
||||||
'storage.files.create',
|
|
||||||
'storage.files.read',
|
|
||||||
'storage.files.update',
|
|
||||||
'storage.files.delete',
|
|
||||||
];
|
|
||||||
|
|
||||||
foreach ($storageGeneralMetrics as $metric) {
|
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
|
||||||
}
|
|
||||||
|
|
||||||
$storageBucketMetrics = [
|
|
||||||
'storage.buckets.bucketId.files.create',
|
|
||||||
'storage.buckets.bucketId.files.read',
|
|
||||||
'storage.buckets.bucketId.files.update',
|
|
||||||
'storage.buckets.bucketId.files.delete',
|
|
||||||
];
|
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) {
|
|
||||||
$bucketId = $bucket->getId();
|
|
||||||
foreach ($storageBucketMetrics as $metric) {
|
|
||||||
$metric = str_replace('bucketId', $bucketId, $metric);
|
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function aggregateFunctionMetrics(string $projectId): void
|
|
||||||
{
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
|
|
||||||
$this->aggregateDailyMetric($projectId, 'functions.executions');
|
|
||||||
$this->aggregateDailyMetric($projectId, 'functions.builds');
|
|
||||||
$this->aggregateDailyMetric($projectId, 'functions.failures');
|
|
||||||
|
|
||||||
$functionMetrics = [
|
|
||||||
'functions.functionId.executions',
|
|
||||||
'functions.functionId.builds',
|
|
||||||
'functions.functionId.compute',
|
|
||||||
'function.functionId.executions.failure',
|
|
||||||
'function.functionId.builds.failure',
|
|
||||||
];
|
|
||||||
|
|
||||||
$this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) {
|
|
||||||
$functionId = $function->getId();
|
|
||||||
foreach ($functionMetrics as $metric) {
|
|
||||||
$metric = str_replace('functionId', $functionId, $metric);
|
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function aggregateUsersMetrics(string $projectId): void
|
|
||||||
{
|
|
||||||
$metrics = [
|
|
||||||
'users.create',
|
|
||||||
'users.read',
|
|
||||||
'users.update',
|
|
||||||
'users.delete',
|
|
||||||
'users.sessions.create',
|
|
||||||
'users.sessions.delete'
|
|
||||||
];
|
|
||||||
|
|
||||||
foreach ($metrics as $metric) {
|
|
||||||
$this->aggregateDailyMetric($projectId, $metric);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function aggregateGeneralMetrics(string $projectId): void
|
|
||||||
{
|
|
||||||
$this->aggregateDailyMetric($projectId, 'requests');
|
|
||||||
$this->aggregateDailyMetric($projectId, 'network');
|
|
||||||
$this->aggregateDailyMetric($projectId, 'inbound');
|
|
||||||
$this->aggregateDailyMetric($projectId, 'outbound');
|
|
||||||
|
|
||||||
//Required for billing
|
|
||||||
$this->aggregateMonthlyMetric($projectId, 'inbound');
|
|
||||||
$this->aggregateMonthlyMetric($projectId, 'outbound');
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function aggregateDailyMetric(string $projectId, string $metric): void
|
|
||||||
{
|
|
||||||
$beginOfDay = strtotime("today");
|
|
||||||
$endOfDay = strtotime("tomorrow", $beginOfDay) - 1;
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
$value = (int) $this->database->sum('stats', 'value', [
|
|
||||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
|
||||||
new Query('period', Query::TYPE_EQUAL, ['30m']),
|
|
||||||
new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfDay]),
|
|
||||||
new Query('time', Query::TYPE_LESSEREQUAL, [$endOfDay]),
|
|
||||||
]);
|
|
||||||
$this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function aggregateMonthlyMetric(string $projectId, string $metric): void
|
|
||||||
{
|
|
||||||
$beginOfMonth = strtotime("first day of the month");
|
|
||||||
$endOfMonth = strtotime("last day of the month");
|
|
||||||
$this->database->setNamespace('_' . $projectId);
|
|
||||||
$value = (int) $this->database->sum('stats', 'value', [
|
|
||||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
|
||||||
new Query('period', Query::TYPE_EQUAL, ['1d']),
|
|
||||||
new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfMonth]),
|
|
||||||
new Query('time', Query::TYPE_LESSEREQUAL, [$endOfMonth]),
|
|
||||||
]);
|
|
||||||
$this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Collect Stats
|
|
||||||
* Collect all database related stats
|
|
||||||
*
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
public function collect(): void
|
|
||||||
{
|
|
||||||
$this->foreachDocument('console', 'projects', [], function (Document $project) {
|
|
||||||
$projectId = $project->getInternalId();
|
|
||||||
|
|
||||||
$this->usersStats($projectId);
|
|
||||||
$this->databaseStats($projectId);
|
|
||||||
$this->storageStats($projectId);
|
|
||||||
$this->computeStats($projectId);
|
|
||||||
|
|
||||||
// Aggregate new metrics from already collected usage metrics
|
|
||||||
// for lower time period (1day and 1 month metric from 30 minute metrics)
|
|
||||||
$this->aggregateGeneralMetrics($projectId);
|
|
||||||
$this->aggregateFunctionMetrics($projectId);
|
|
||||||
$this->aggregateDatabaseMetrics($projectId);
|
|
||||||
$this->aggregateStorageMetrics($projectId);
|
|
||||||
$this->aggregateUsersMetrics($projectId);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
7
src/Appwrite/Usage/Calculator.php
Normal file
7
src/Appwrite/Usage/Calculator.php
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Appwrite\Usage;
|
||||||
|
|
||||||
|
abstract class Calculator {
|
||||||
|
abstract public function collect(): void;
|
||||||
|
}
|
205
src/Appwrite/Usage/Calculators/Aggregator.php
Normal file
205
src/Appwrite/Usage/Calculators/Aggregator.php
Normal file
|
@ -0,0 +1,205 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Appwrite\Usage\Calculators;
|
||||||
|
|
||||||
|
use Utopia\Database\Database as UtopiaDatabase;
|
||||||
|
use Utopia\Database\Document;
|
||||||
|
use Utopia\Database\Query;
|
||||||
|
|
||||||
|
class Aggregator extends Database {
|
||||||
|
|
||||||
|
protected function aggregateDatabaseMetrics(string $projectId): void
|
||||||
|
{
|
||||||
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
|
||||||
|
$databasesGeneralMetrics = [
|
||||||
|
'databases.create',
|
||||||
|
'databases.read',
|
||||||
|
'databases.update',
|
||||||
|
'databases.delete',
|
||||||
|
'databases.collections.create',
|
||||||
|
'databases.collections.read',
|
||||||
|
'databases.collections.update',
|
||||||
|
'databases.collections.delete',
|
||||||
|
'databases.documents.create',
|
||||||
|
'databases.documents.read',
|
||||||
|
'databases.documents.update',
|
||||||
|
'databases.documents.delete',
|
||||||
|
];
|
||||||
|
|
||||||
|
foreach ($databasesGeneralMetrics as $metric) {
|
||||||
|
$this->aggregateDailyMetric($projectId, $metric);
|
||||||
|
}
|
||||||
|
|
||||||
|
$databasesDatabaseMetrics = [
|
||||||
|
'databases.databaseId.collections.create',
|
||||||
|
'databases.databaseId.collections.read',
|
||||||
|
'databases.databaseId.collections.update',
|
||||||
|
'databases.databaseId.collections.delete',
|
||||||
|
'databases.databaseId.documents.create',
|
||||||
|
'databases.databaseId.documents.read',
|
||||||
|
'databases.databaseId.documents.update',
|
||||||
|
'databases.databaseId.documents.delete',
|
||||||
|
];
|
||||||
|
|
||||||
|
$this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) {
|
||||||
|
$databaseId = $database->getId();
|
||||||
|
foreach ($databasesDatabaseMetrics as $metric) {
|
||||||
|
$metric = str_replace('databaseId', $databaseId, $metric);
|
||||||
|
$this->aggregateDailyMetric($projectId, $metric);
|
||||||
|
}
|
||||||
|
|
||||||
|
$databasesCollectionMetrics = [
|
||||||
|
'databases.' . $databaseId . '.collections.collectionId.documents.create',
|
||||||
|
'databases.' . $databaseId . '.collections.collectionId.documents.read',
|
||||||
|
'databases.' . $databaseId . '.collections.collectionId.documents.update',
|
||||||
|
'databases.' . $databaseId . '.collections.collectionId.documents.delete',
|
||||||
|
];
|
||||||
|
|
||||||
|
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) {
|
||||||
|
$collectionId = $collection->getId();
|
||||||
|
foreach ($databasesCollectionMetrics as $metric) {
|
||||||
|
$metric = str_replace('collectionId', $collectionId, $metric);
|
||||||
|
$this->aggregateDailyMetric($projectId, $metric);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function aggregateStorageMetrics(string $projectId): void
|
||||||
|
{
|
||||||
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
|
||||||
|
$storageGeneralMetrics = [
|
||||||
|
'storage.buckets.create',
|
||||||
|
'storage.buckets.read',
|
||||||
|
'storage.buckets.update',
|
||||||
|
'storage.buckets.delete',
|
||||||
|
'storage.files.create',
|
||||||
|
'storage.files.read',
|
||||||
|
'storage.files.update',
|
||||||
|
'storage.files.delete',
|
||||||
|
];
|
||||||
|
|
||||||
|
foreach ($storageGeneralMetrics as $metric) {
|
||||||
|
$this->aggregateDailyMetric($projectId, $metric);
|
||||||
|
}
|
||||||
|
|
||||||
|
$storageBucketMetrics = [
|
||||||
|
'storage.buckets.bucketId.files.create',
|
||||||
|
'storage.buckets.bucketId.files.read',
|
||||||
|
'storage.buckets.bucketId.files.update',
|
||||||
|
'storage.buckets.bucketId.files.delete',
|
||||||
|
];
|
||||||
|
|
||||||
|
$this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) {
|
||||||
|
$bucketId = $bucket->getId();
|
||||||
|
foreach ($storageBucketMetrics as $metric) {
|
||||||
|
$metric = str_replace('bucketId', $bucketId, $metric);
|
||||||
|
$this->aggregateDailyMetric($projectId, $metric);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function aggregateFunctionMetrics(string $projectId): void
|
||||||
|
{
|
||||||
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
|
||||||
|
$this->aggregateDailyMetric($projectId, 'functions.executions');
|
||||||
|
$this->aggregateDailyMetric($projectId, 'functions.builds');
|
||||||
|
$this->aggregateDailyMetric($projectId, 'functions.failures');
|
||||||
|
|
||||||
|
$functionMetrics = [
|
||||||
|
'functions.functionId.executions',
|
||||||
|
'functions.functionId.builds',
|
||||||
|
'functions.functionId.compute',
|
||||||
|
'function.functionId.executions.failure',
|
||||||
|
'function.functionId.builds.failure',
|
||||||
|
];
|
||||||
|
|
||||||
|
$this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) {
|
||||||
|
$functionId = $function->getId();
|
||||||
|
foreach ($functionMetrics as $metric) {
|
||||||
|
$metric = str_replace('functionId', $functionId, $metric);
|
||||||
|
$this->aggregateDailyMetric($projectId, $metric);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function aggregateUsersMetrics(string $projectId): void
|
||||||
|
{
|
||||||
|
$metrics = [
|
||||||
|
'users.create',
|
||||||
|
'users.read',
|
||||||
|
'users.update',
|
||||||
|
'users.delete',
|
||||||
|
'users.sessions.create',
|
||||||
|
'users.sessions.delete'
|
||||||
|
];
|
||||||
|
|
||||||
|
foreach ($metrics as $metric) {
|
||||||
|
$this->aggregateDailyMetric($projectId, $metric);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function aggregateGeneralMetrics(string $projectId): void
|
||||||
|
{
|
||||||
|
$this->aggregateDailyMetric($projectId, 'requests');
|
||||||
|
$this->aggregateDailyMetric($projectId, 'network');
|
||||||
|
$this->aggregateDailyMetric($projectId, 'inbound');
|
||||||
|
$this->aggregateDailyMetric($projectId, 'outbound');
|
||||||
|
|
||||||
|
//Required for billing
|
||||||
|
$this->aggregateMonthlyMetric($projectId, 'inbound');
|
||||||
|
$this->aggregateMonthlyMetric($projectId, 'outbound');
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function aggregateDailyMetric(string $projectId, string $metric): void
|
||||||
|
{
|
||||||
|
$beginOfDay = strtotime("today");
|
||||||
|
$endOfDay = strtotime("tomorrow", $beginOfDay) - 1;
|
||||||
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
$value = (int) $this->database->sum('stats', 'value', [
|
||||||
|
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||||
|
new Query('period', Query::TYPE_EQUAL, ['30m']),
|
||||||
|
new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfDay]),
|
||||||
|
new Query('time', Query::TYPE_LESSEREQUAL, [$endOfDay]),
|
||||||
|
]);
|
||||||
|
$this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function aggregateMonthlyMetric(string $projectId, string $metric): void
|
||||||
|
{
|
||||||
|
$beginOfMonth = strtotime("first day of the month");
|
||||||
|
$endOfMonth = strtotime("last day of the month");
|
||||||
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
$value = (int) $this->database->sum('stats', 'value', [
|
||||||
|
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||||
|
new Query('period', Query::TYPE_EQUAL, ['1d']),
|
||||||
|
new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfMonth]),
|
||||||
|
new Query('time', Query::TYPE_LESSEREQUAL, [$endOfMonth]),
|
||||||
|
]);
|
||||||
|
$this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collect Stats
|
||||||
|
* Collect all database related stats
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function collect(): void
|
||||||
|
{
|
||||||
|
$this->foreachDocument('console', 'projects', [], function (Document $project) {
|
||||||
|
$projectId = $project->getInternalId();
|
||||||
|
|
||||||
|
// Aggregate new metrics from already collected usage metrics
|
||||||
|
// for lower time period (1day and 1 month metric from 30 minute metrics)
|
||||||
|
$this->aggregateGeneralMetrics($projectId);
|
||||||
|
$this->aggregateFunctionMetrics($projectId);
|
||||||
|
$this->aggregateDatabaseMetrics($projectId);
|
||||||
|
$this->aggregateStorageMetrics($projectId);
|
||||||
|
$this->aggregateUsersMetrics($projectId);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
318
src/Appwrite/Usage/Calculators/Database.php
Normal file
318
src/Appwrite/Usage/Calculators/Database.php
Normal file
|
@ -0,0 +1,318 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Appwrite\Usage\Calculators;
|
||||||
|
|
||||||
|
use Appwrite\Usage\Calculator;
|
||||||
|
use Utopia\Database\Database as UtopiaDatabase;
|
||||||
|
use Utopia\Database\Document;
|
||||||
|
use Utopia\Database\Query;
|
||||||
|
|
||||||
|
class Database extends Calculator
|
||||||
|
{
|
||||||
|
protected array $periods = [
|
||||||
|
[
|
||||||
|
'key' => '30m',
|
||||||
|
'multiplier' => 1800,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'key' => '1d',
|
||||||
|
'multiplier' => 86400,
|
||||||
|
],
|
||||||
|
];
|
||||||
|
|
||||||
|
public function __construct(UtopiaDatabase $database, callable $errorHandler = null)
|
||||||
|
{
|
||||||
|
$this->database = $database;
|
||||||
|
$this->errorHandler = $errorHandler;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create Per Period Metric
|
||||||
|
* Create given metric for each defined period
|
||||||
|
*
|
||||||
|
* @param string $projectId
|
||||||
|
* @param string $metric
|
||||||
|
* @param int $value
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
protected function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void
|
||||||
|
{
|
||||||
|
foreach ($this->periods as $options) {
|
||||||
|
$period = $options['key'];
|
||||||
|
$time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']);
|
||||||
|
$this->createOrUpdateMetric($projectId, $metric, $period, $time, $value);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Required for billing
|
||||||
|
if ($monthly) {
|
||||||
|
$time = strtotime("first day of the month");
|
||||||
|
$this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create or Update Mertic
|
||||||
|
* Create or update each metric in the stats collection for the given project
|
||||||
|
*
|
||||||
|
* @param string $projectId
|
||||||
|
* @param string $metric
|
||||||
|
* @param int $value
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
protected function createOrUpdateMetric(string $projectId, string $metric, string $period, int $time, int $value): void
|
||||||
|
{
|
||||||
|
$id = \md5("{$time}_{$period}_{$metric}");
|
||||||
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
|
||||||
|
try {
|
||||||
|
$document = $this->database->getDocument('stats', $id);
|
||||||
|
if ($document->isEmpty()) {
|
||||||
|
$this->database->createDocument('stats', new Document([
|
||||||
|
'$id' => $id,
|
||||||
|
'period' => $period,
|
||||||
|
'time' => $time,
|
||||||
|
'metric' => $metric,
|
||||||
|
'value' => $value,
|
||||||
|
'type' => 1,
|
||||||
|
]));
|
||||||
|
} else {
|
||||||
|
$this->database->updateDocument(
|
||||||
|
'stats',
|
||||||
|
$document->getId(),
|
||||||
|
$document->setAttribute('value', $value)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (\Exception$e) { // if projects are deleted this might fail
|
||||||
|
if (is_callable($this->errorHandler)) {
|
||||||
|
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
|
||||||
|
} else {
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Foreach Document
|
||||||
|
* Call provided callback for each document in the collection
|
||||||
|
*
|
||||||
|
* @param string $projectId
|
||||||
|
* @param string $collection
|
||||||
|
* @param array $queries
|
||||||
|
* @param callable $callback
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
protected function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
|
||||||
|
{
|
||||||
|
$limit = 50;
|
||||||
|
$results = [];
|
||||||
|
$sum = $limit;
|
||||||
|
$latestDocument = null;
|
||||||
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
|
||||||
|
while ($sum === $limit) {
|
||||||
|
try {
|
||||||
|
$results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument);
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
if (is_callable($this->errorHandler)) {
|
||||||
|
call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}");
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (empty($results)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$sum = count($results);
|
||||||
|
|
||||||
|
foreach ($results as $document) {
|
||||||
|
if (is_callable($callback)) {
|
||||||
|
$callback($document);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$latestDocument = $results[array_key_last($results)];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sum
|
||||||
|
* Calculate sum of a attribute of documents in collection
|
||||||
|
*
|
||||||
|
* @param string $projectId
|
||||||
|
* @param string $collection
|
||||||
|
* @param string $attribute
|
||||||
|
* @param string $metric
|
||||||
|
*
|
||||||
|
* @return int
|
||||||
|
*/
|
||||||
|
private function sum(string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int
|
||||||
|
{
|
||||||
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
|
||||||
|
try {
|
||||||
|
$sum = $this->database->sum($collection, $attribute);
|
||||||
|
$sum = (int) ($sum * $multiplier);
|
||||||
|
|
||||||
|
if(!is_null($metric)) {
|
||||||
|
$this->createPerPeriodMetric($projectId, $metric, $sum);
|
||||||
|
}
|
||||||
|
return $sum;
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
if (is_callable($this->errorHandler)) {
|
||||||
|
call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}");
|
||||||
|
} else {
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Count
|
||||||
|
* Count number of documents in collection
|
||||||
|
*
|
||||||
|
* @param string $projectId
|
||||||
|
* @param string $collection
|
||||||
|
* @param string? $metric
|
||||||
|
*
|
||||||
|
* @return int
|
||||||
|
*/
|
||||||
|
private function count(string $projectId, string $collection, string $metric = null): int
|
||||||
|
{
|
||||||
|
$this->database->setNamespace('_' . $projectId);
|
||||||
|
|
||||||
|
try {
|
||||||
|
$count = $this->database->count($collection);
|
||||||
|
if(!is_null($metric)) {
|
||||||
|
$this->createPerPeriodMetric($projectId, (string) $metric, $count);
|
||||||
|
}
|
||||||
|
return $count;
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
if (is_callable($this->errorHandler)) {
|
||||||
|
call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}");
|
||||||
|
} else {
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deployments Total
|
||||||
|
* Total sum of storage used by deployments
|
||||||
|
*
|
||||||
|
* @param string $projectId
|
||||||
|
*
|
||||||
|
* @return int
|
||||||
|
*/
|
||||||
|
private function deploymentsTotal(string $projectId): int
|
||||||
|
{
|
||||||
|
return $this->sum($projectId, 'deployments', 'size', 'deployments.$all.storage.size');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Users Stats
|
||||||
|
* Metric: users.count
|
||||||
|
*
|
||||||
|
* @param string $projectId
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
private function usersStats(string $projectId): void
|
||||||
|
{
|
||||||
|
$this->count($projectId, 'users', 'users.$all.count.total');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Storage Stats
|
||||||
|
* Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total,
|
||||||
|
* files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size
|
||||||
|
*
|
||||||
|
* @param string $projectId
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
private function storageStats(string $projectId): void
|
||||||
|
{
|
||||||
|
$projectFilesTotal = 0;
|
||||||
|
$projectFilesCount = 0;
|
||||||
|
|
||||||
|
$metric = 'buckets.$all.count.total';
|
||||||
|
$this->count($projectId, 'buckets', $metric);
|
||||||
|
|
||||||
|
$this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) {
|
||||||
|
$metric = "files.{$bucket->getId()}.count.total";
|
||||||
|
$count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric);
|
||||||
|
$projectFilesCount += $count;
|
||||||
|
|
||||||
|
$metric = "files.{$bucket->getId()}.storage.size";
|
||||||
|
$sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric);
|
||||||
|
$projectFilesTotal += $sum;
|
||||||
|
});
|
||||||
|
|
||||||
|
$this->createPerPeriodMetric($projectId, 'files.$all.count.total', $projectFilesCount);
|
||||||
|
$this->createPerPeriodMetric($projectId, 'files.$all.storage.size', $projectFilesTotal);
|
||||||
|
|
||||||
|
$deploymentsTotal = $this->deploymentsTotal($projectId);
|
||||||
|
$this->createPerPeriodMetric($projectId, 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Database Stats
|
||||||
|
* Collect all database stats
|
||||||
|
* Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total,
|
||||||
|
* documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total
|
||||||
|
*
|
||||||
|
* @param string $projectId
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
private function databaseStats(string $projectId): void
|
||||||
|
{
|
||||||
|
$projectDocumentsCount = 0;
|
||||||
|
$projectCollectionsCount = 0;
|
||||||
|
|
||||||
|
$this->count($projectId, 'databases', 'databases.$all.count.total');
|
||||||
|
|
||||||
|
$this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) {
|
||||||
|
$metric = "collections.{$database->getId()}.count.total";
|
||||||
|
$count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric);
|
||||||
|
$projectCollectionsCount += $count;
|
||||||
|
$databaseDocumentsCount = 0;
|
||||||
|
|
||||||
|
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) {
|
||||||
|
$metric = "documents.{$database->getId()}/{$collection->getId()}.count.total";
|
||||||
|
|
||||||
|
$count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric);
|
||||||
|
$projectDocumentsCount += $count;
|
||||||
|
$databaseDocumentsCount += $count;
|
||||||
|
});
|
||||||
|
|
||||||
|
$this->createPerPeriodMetric($projectId, "documents.{$database->getId()}.count.total", $databaseDocumentsCount);
|
||||||
|
});
|
||||||
|
|
||||||
|
$this->createPerPeriodMetric($projectId, 'collections.$all.count.total', $projectCollectionsCount);
|
||||||
|
$this->createPerPeriodMetric($projectId, 'documents.$all.count.total', $projectDocumentsCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collect Stats
|
||||||
|
* Collect all database related stats
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function collect(): void
|
||||||
|
{
|
||||||
|
$this->foreachDocument('console', 'projects', [], function (Document $project) {
|
||||||
|
$projectId = $project->getInternalId();
|
||||||
|
|
||||||
|
$this->usersStats($projectId);
|
||||||
|
$this->databaseStats($projectId);
|
||||||
|
$this->storageStats($projectId);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,14 +1,15 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
namespace Appwrite\Stats;
|
namespace Appwrite\Usage\Calculators;
|
||||||
|
|
||||||
|
use Appwrite\Usage\Calculator;
|
||||||
use Utopia\Database\Database;
|
use Utopia\Database\Database;
|
||||||
use Utopia\Database\Document;
|
use Utopia\Database\Document;
|
||||||
use InfluxDB\Database as InfluxDatabase;
|
use InfluxDB\Database as InfluxDatabase;
|
||||||
use DateTime;
|
use DateTime;
|
||||||
|
|
||||||
class Usage
|
class TimeSeries extends Calculator {
|
||||||
{
|
|
||||||
protected InfluxDatabase $influxDB;
|
protected InfluxDatabase $influxDB;
|
||||||
protected Database $database;
|
protected Database $database;
|
||||||
protected $errorHandler;
|
protected $errorHandler;
|
||||||
|
@ -16,154 +17,165 @@ class Usage
|
||||||
|
|
||||||
// all the mertics that we are collecting
|
// all the mertics that we are collecting
|
||||||
protected array $metrics = [
|
protected array $metrics = [
|
||||||
'requests' => [
|
'project.$all.network.requests' => [
|
||||||
'table' => 'appwrite_usage_requests_all',
|
'table' => 'appwrite_usage_network_requests',
|
||||||
],
|
],
|
||||||
'network' => [
|
'project.$all.network.bandwidth' => [
|
||||||
'table' => 'appwrite_usage_network_all',
|
'table' => 'appwrite_usage_network_bandwidth',
|
||||||
],
|
],
|
||||||
'inbound' => [
|
'project.$all.network.inbound' => [
|
||||||
'table' => 'appwrite_usage_network_inbound',
|
'table' => 'appwrite_usage_network_inbound',
|
||||||
],
|
],
|
||||||
'outbound' => [
|
'project.$all.network.outbound' => [
|
||||||
'table' => 'appwrite_usage_network_outbound',
|
'table' => 'appwrite_usage_network_outbound',
|
||||||
],
|
],
|
||||||
'databases.create' => [
|
/* Users service metrics */
|
||||||
'table' => 'appwrite_usage_databases_create',
|
'users.$all.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_users_requests_create',
|
||||||
],
|
],
|
||||||
'databases.read' => [
|
'users.$all.requests.read' => [
|
||||||
'table' => 'appwrite_usage_databases_read',
|
'table' => 'appwrite_usage_users_requests_read',
|
||||||
],
|
],
|
||||||
'databases.update' => [
|
'users.$all.requests.update' => [
|
||||||
'table' => 'appwrite_usage_databases_update',
|
'table' => 'appwrite_usage_users_requests_update',
|
||||||
],
|
],
|
||||||
'databases.delete' => [
|
'users.$all.requests.delete' => [
|
||||||
'table' => 'appwrite_usage_databases_delete',
|
'table' => 'appwrite_usage_users_requests_delete',
|
||||||
],
|
],
|
||||||
'databases.collections.create' => [
|
|
||||||
'table' => 'appwrite_usage_databases_collections_create',
|
'databases.$all.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_databases_requests_create',
|
||||||
],
|
],
|
||||||
'databases.collections.read' => [
|
'databases.$all.requests.read' => [
|
||||||
'table' => 'appwrite_usage_databases_collections_read',
|
'table' => 'appwrite_usage_databases_requests_read',
|
||||||
],
|
],
|
||||||
'databases.collections.update' => [
|
'databases.$all.requests.update' => [
|
||||||
'table' => 'appwrite_usage_databases_collections_update',
|
'table' => 'appwrite_usage_databases_requests_update',
|
||||||
],
|
],
|
||||||
'databases.collections.delete' => [
|
'databases.$all.requests.delete' => [
|
||||||
'table' => 'appwrite_usage_databases_collections_delete',
|
'table' => 'appwrite_usage_databases_requests_delete',
|
||||||
],
|
],
|
||||||
'databases.documents.create' => [
|
|
||||||
'table' => 'appwrite_usage_databases_documents_create',
|
'collections.$all.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_dollections_requests_create',
|
||||||
],
|
],
|
||||||
'databases.documents.read' => [
|
'collections.$all.requests.read' => [
|
||||||
'table' => 'appwrite_usage_databases_documents_read',
|
'table' => 'appwrite_usage_collections_requests_read',
|
||||||
],
|
],
|
||||||
'databases.documents.update' => [
|
'collections.$all.requests.update' => [
|
||||||
'table' => 'appwrite_usage_databases_documents_update',
|
'table' => 'appwrite_usage_collections_requests_update',
|
||||||
],
|
],
|
||||||
'databases.documents.delete' => [
|
'collections.$all.requests.delete' => [
|
||||||
'table' => 'appwrite_usage_databases_documents_delete',
|
'table' => 'appwrite_usage_collections_requests_delete',
|
||||||
],
|
],
|
||||||
'databases.databaseId.collections.create' => [
|
|
||||||
'table' => 'appwrite_usage_databases_collections_create',
|
'documents.$all.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_documents_requests_create',
|
||||||
|
],
|
||||||
|
'documents.$all.requests.read' => [
|
||||||
|
'table' => 'appwrite_usage_documents_requests_read',
|
||||||
|
],
|
||||||
|
'documents.$all.requests.update' => [
|
||||||
|
'table' => 'appwrite_usage_documents_requests_update',
|
||||||
|
],
|
||||||
|
'documents.$all.requests.delete' => [
|
||||||
|
'table' => 'appwrite_usage_documents_requests_delete',
|
||||||
|
],
|
||||||
|
|
||||||
|
'collections.databaseId.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_collections_requests_create',
|
||||||
'groupBy' => ['databaseId'],
|
'groupBy' => ['databaseId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.collections.read' => [
|
'collections.databaseId.requests.read' => [
|
||||||
'table' => 'appwrite_usage_databases_collections_read',
|
'table' => 'appwrite_usage_collections_requests_read',
|
||||||
'groupBy' => ['databaseId'],
|
'groupBy' => ['databaseId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.collections.update' => [
|
'collections.databaseId.requests.update' => [
|
||||||
'table' => 'appwrite_usage_databases_collections_update',
|
'table' => 'appwrite_usage_collections_requests_update',
|
||||||
'groupBy' => ['databaseId'],
|
'groupBy' => ['databaseId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.collections.delete' => [
|
'collections.databaseId.requests.delete' => [
|
||||||
'table' => 'appwrite_usage_databases_collections_delete',
|
'table' => 'appwrite_usage_collections_requests_delete',
|
||||||
'groupBy' => ['databaseId'],
|
'groupBy' => ['databaseId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.documents.create' => [
|
|
||||||
'table' => 'appwrite_usage_databases_documents_create',
|
'documents.databaseId.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_documents_requests_create',
|
||||||
'groupBy' => ['databaseId'],
|
'groupBy' => ['databaseId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.documents.read' => [
|
'documents.databaseId.requests.read' => [
|
||||||
'table' => 'appwrite_usage_databases_documents_read',
|
'table' => 'appwrite_usage_documents_requests_read',
|
||||||
'groupBy' => ['databaseId'],
|
'groupBy' => ['databaseId'],
|
||||||
],
|
],
|
||||||
'database.databaseId.documents.update' => [
|
'documents.databaseId.requests.update' => [
|
||||||
'table' => 'appwrite_usage_databases_documents_update',
|
'table' => 'appwrite_usage_documents_requests_update',
|
||||||
'groupBy' => ['databaseId'],
|
'groupBy' => ['databaseId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.documents.delete' => [
|
'documents.databaseId.requests.delete' => [
|
||||||
'table' => 'appwrite_usage_databases_documents_delete',
|
'table' => 'appwrite_usage_documents_requests_delete',
|
||||||
'groupBy' => ['databaseId'],
|
'groupBy' => ['databaseId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.collections.collectionId.documents.create' => [
|
|
||||||
'table' => 'appwrite_usage_databases_documents_create',
|
'documents.databaseId/collectionId.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_documents_requests_create',
|
||||||
'groupBy' => ['databaseId', 'collectionId'],
|
'groupBy' => ['databaseId', 'collectionId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.collections.collectionId.documents.read' => [
|
'documents.databaseId/collectionId.requests.read' => [
|
||||||
'table' => 'appwrite_usage_databases_documents_read',
|
'table' => 'appwrite_usage_documents_requests_read',
|
||||||
'groupBy' => ['databaseId', 'collectionId'],
|
'groupBy' => ['databaseId', 'collectionId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.collections.collectionId.documents.update' => [
|
'documents.databaseId/collectionId.requests.update' => [
|
||||||
'table' => 'appwrite_usage_databases_documents_update',
|
'table' => 'appwrite_usage_documents_requests_update',
|
||||||
'groupBy' => ['databaseId', 'collectionId'],
|
'groupBy' => ['databaseId', 'collectionId'],
|
||||||
],
|
],
|
||||||
'databases.databaseId.collections.collectionId.documents.delete' => [
|
'documents.databaseId/collectionId.requests.delete' => [
|
||||||
'table' => 'appwrite_usage_databases_documents_delete',
|
'table' => 'appwrite_usage_documents_requests_delete',
|
||||||
'groupBy' => ['databaseId', 'collectionId'],
|
'groupBy' => ['databaseId', 'collectionId'],
|
||||||
],
|
],
|
||||||
'storage.buckets.create' => [
|
|
||||||
'table' => 'appwrite_usage_storage_buckets_create',
|
'buckets.$all.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_buckets_requests_create',
|
||||||
],
|
],
|
||||||
'storage.buckets.read' => [
|
'buckets.$all.requests.read' => [
|
||||||
'table' => 'appwrite_usage_storage_buckets_read',
|
'table' => 'appwrite_usage_buckets_requests_read',
|
||||||
],
|
],
|
||||||
'storage.buckets.update' => [
|
'buckets.$all.requests.update' => [
|
||||||
'table' => 'appwrite_usage_storage_buckets_update',
|
'table' => 'appwrite_usage_buckets_requests_update',
|
||||||
],
|
],
|
||||||
'storage.buckets.delete' => [
|
'buckets.$all.requests.delete' => [
|
||||||
'table' => 'appwrite_usage_storage_buckets_delete',
|
'table' => 'appwrite_usage_buckets_requests_delete',
|
||||||
],
|
],
|
||||||
'storage.files.create' => [
|
|
||||||
'table' => 'appwrite_usage_storage_files_create',
|
'files.$all.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_files_requests_create',
|
||||||
],
|
],
|
||||||
'storage.files.read' => [
|
'files.$all.requests.read' => [
|
||||||
'table' => 'appwrite_usage_storage_files_read',
|
'table' => 'appwrite_usage_files_requests_read',
|
||||||
],
|
],
|
||||||
'storage.files.update' => [
|
'files.$all.requests.update' => [
|
||||||
'table' => 'appwrite_usage_storage_files_update',
|
'table' => 'appwrite_usage_files_requests_update',
|
||||||
],
|
],
|
||||||
'storage.files.delete' => [
|
'files.$all.requests.delete' => [
|
||||||
'table' => 'appwrite_usage_storage_files_delete',
|
'table' => 'appwrite_usage_files_requests_delete',
|
||||||
],
|
],
|
||||||
'storage.buckets.bucketId.files.create' => [
|
|
||||||
'table' => 'appwrite_usage_storage_files_create',
|
'files.bucketId.requests.create' => [
|
||||||
|
'table' => 'appwrite_usage_files_requests_create',
|
||||||
'groupBy' => ['bucketId'],
|
'groupBy' => ['bucketId'],
|
||||||
],
|
],
|
||||||
'storage.buckets.bucketId.files.read' => [
|
'files.bucketId.requests.read' => [
|
||||||
'table' => 'appwrite_usage_storage_files_read',
|
'table' => 'appwrite_usage_files_requests_read',
|
||||||
'groupBy' => ['bucketId'],
|
'groupBy' => ['bucketId'],
|
||||||
],
|
],
|
||||||
'storage.buckets.bucketId.files.update' => [
|
'files.bucketId.requests.update' => [
|
||||||
'table' => 'appwrite_usage_storage_files_update',
|
'table' => 'appwrite_usage_files_requests_update',
|
||||||
'groupBy' => ['bucketId'],
|
'groupBy' => ['bucketId'],
|
||||||
],
|
],
|
||||||
'storage.buckets.bucketId.files.delete' => [
|
'files.bucketId.requests.delete' => [
|
||||||
'table' => 'appwrite_usage_storage_files_delete',
|
'table' => 'appwrite_usage_files_requests_delete',
|
||||||
'groupBy' => ['bucketId'],
|
'groupBy' => ['bucketId'],
|
||||||
],
|
],
|
||||||
'users.create' => [
|
|
||||||
'table' => 'appwrite_usage_users_create',
|
|
||||||
],
|
|
||||||
'users.read' => [
|
|
||||||
'table' => 'appwrite_usage_users_read',
|
|
||||||
],
|
|
||||||
'users.update' => [
|
|
||||||
'table' => 'appwrite_usage_users_update',
|
|
||||||
],
|
|
||||||
'users.delete' => [
|
|
||||||
'table' => 'appwrite_usage_users_delete',
|
|
||||||
],
|
|
||||||
'users.sessions.create' => [
|
'users.sessions.create' => [
|
||||||
'table' => 'appwrite_usage_users_sessions_create',
|
'table' => 'appwrite_usage_users_sessions_create',
|
||||||
],
|
],
|
||||||
|
@ -378,4 +390,4 @@ class Usage
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -1,6 +1,6 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
namespace Appwrite\Stats;
|
namespace Appwrite\Usage;
|
||||||
|
|
||||||
use Utopia\App;
|
use Utopia\App;
|
||||||
|
|
||||||
|
@ -104,7 +104,84 @@ class Stats
|
||||||
$this->statsd->setNamespace($this->namespace);
|
$this->statsd->setNamespace($this->namespace);
|
||||||
|
|
||||||
if ($httpRequest >= 1) {
|
if ($httpRequest >= 1) {
|
||||||
$this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod));
|
$this->statsd->increment('network.requests' . $tags . ',method=' . \strtolower($httpMethod));
|
||||||
|
}
|
||||||
|
$this->statsd->count('network.inbound' . $tags, $networkRequestSize);
|
||||||
|
$this->statsd->count('network.outbound' . $tags, $networkResponseSize);
|
||||||
|
$this->statsd->count('network.bandwidth' . $tags, $networkRequestSize + $networkResponseSize);
|
||||||
|
|
||||||
|
$usersMetrics = [
|
||||||
|
'users.requests.create',
|
||||||
|
'users.requests.read',
|
||||||
|
'users.requests.update',
|
||||||
|
'users.requests.delete',
|
||||||
|
];
|
||||||
|
|
||||||
|
foreach ($usersMetrics as $metric) {
|
||||||
|
$value = $this->params[$metric] ?? 0;
|
||||||
|
if ($value >= 1) {
|
||||||
|
$this->statsd->increment($metric . $tags);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$dbMetrics = [
|
||||||
|
'databases.requests.create',
|
||||||
|
'databases.requests.read',
|
||||||
|
'databases.requests.update',
|
||||||
|
'databases.requests.delete',
|
||||||
|
'collections.requests.create',
|
||||||
|
'collections.requests.read',
|
||||||
|
'collections.requests.update',
|
||||||
|
'collections.requests.delete',
|
||||||
|
'documents.requests.create',
|
||||||
|
'documents.requests.read',
|
||||||
|
'documents.requests.update',
|
||||||
|
'documents.requests.delete',
|
||||||
|
];
|
||||||
|
|
||||||
|
foreach ($dbMetrics as $metric) {
|
||||||
|
$value = $this->params[$metric] ?? 0;
|
||||||
|
if ($value >= 1) {
|
||||||
|
$dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? '');
|
||||||
|
$this->statsd->increment($metric . $dbTags);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$storageMertics = [
|
||||||
|
'buckets.requests.create',
|
||||||
|
'buckets.requests.read',
|
||||||
|
'buckets.requests.update',
|
||||||
|
'buckets.requests.delete',
|
||||||
|
'files.requests.create',
|
||||||
|
'files.requests.read',
|
||||||
|
'files.requests.update',
|
||||||
|
'files.requests.delete',
|
||||||
|
];
|
||||||
|
|
||||||
|
foreach ($storageMertics as $metric) {
|
||||||
|
$value = $this->params[$metric] ?? 0;
|
||||||
|
if ($value >= 1) {
|
||||||
|
$storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? '');
|
||||||
|
$this->statsd->increment($metric . $storageTags);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$sessionsMetrics = [
|
||||||
|
'users.sessions.create',
|
||||||
|
'users.sessions.delete',
|
||||||
|
];
|
||||||
|
|
||||||
|
foreach ($sessionsMetrics as $metric) {
|
||||||
|
$value = $this->params[$metric] ?? 0;
|
||||||
|
if ($value >= 1) {
|
||||||
|
$sessionTags = $tags . ",provider=" . ($this->params['provider'] ?? '');
|
||||||
|
$this->statsd->count($metric . $sessionTags, $value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($storage >= 1) {
|
||||||
|
$storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? '');
|
||||||
|
$this->statsd->count('storage.all' . $storageTags, $storage);
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($functionExecution >= 1) {
|
if ($functionExecution >= 1) {
|
||||||
|
@ -119,85 +196,6 @@ class Stats
|
||||||
$this->statsd->count('functions.compute.time' . $tags . ',functionId=' . $functionId, $functionCompute);
|
$this->statsd->count('functions.compute.time' . $tags . ',functionId=' . $functionId, $functionCompute);
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->statsd->count('network.inbound' . $tags, $networkRequestSize);
|
|
||||||
$this->statsd->count('network.outbound' . $tags, $networkResponseSize);
|
|
||||||
$this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize);
|
|
||||||
|
|
||||||
$dbMetrics = [
|
|
||||||
'databases.create',
|
|
||||||
'databases.read',
|
|
||||||
'databases.update',
|
|
||||||
'databases.delete',
|
|
||||||
'databases.collections.create',
|
|
||||||
'databases.collections.read',
|
|
||||||
'databases.collections.update',
|
|
||||||
'databases.collections.delete',
|
|
||||||
'databases.documents.create',
|
|
||||||
'databases.documents.read',
|
|
||||||
'databases.documents.update',
|
|
||||||
'databases.documents.delete',
|
|
||||||
];
|
|
||||||
|
|
||||||
foreach ($dbMetrics as $metric) {
|
|
||||||
$value = $this->params[$metric] ?? 0;
|
|
||||||
if ($value >= 1) {
|
|
||||||
$tags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? '');
|
|
||||||
$this->statsd->increment($metric . $tags);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$storageMertics = [
|
|
||||||
'storage.buckets.create',
|
|
||||||
'storage.buckets.read',
|
|
||||||
'storage.buckets.update',
|
|
||||||
'storage.buckets.delete',
|
|
||||||
'storage.files.create',
|
|
||||||
'storage.files.read',
|
|
||||||
'storage.files.update',
|
|
||||||
'storage.files.delete',
|
|
||||||
];
|
|
||||||
|
|
||||||
foreach ($storageMertics as $metric) {
|
|
||||||
$value = $this->params[$metric] ?? 0;
|
|
||||||
if ($value >= 1) {
|
|
||||||
$tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? '');
|
|
||||||
$this->statsd->increment($metric . $tags);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$usersMetrics = [
|
|
||||||
'users.create',
|
|
||||||
'users.read',
|
|
||||||
'users.update',
|
|
||||||
'users.delete',
|
|
||||||
];
|
|
||||||
|
|
||||||
foreach ($usersMetrics as $metric) {
|
|
||||||
$value = $this->params[$metric] ?? 0;
|
|
||||||
if ($value >= 1) {
|
|
||||||
$tags = ",projectId={$projectId}";
|
|
||||||
$this->statsd->increment($metric . $tags);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$sessionsMetrics = [
|
|
||||||
'users.sessions.create',
|
|
||||||
'users.sessions.delete',
|
|
||||||
];
|
|
||||||
|
|
||||||
foreach ($sessionsMetrics as $metric) {
|
|
||||||
$value = $this->params[$metric] ?? 0;
|
|
||||||
if ($value >= 1) {
|
|
||||||
$tags = ",projectId={$projectId},provider=" . ($this->params['provider'] ?? '');
|
|
||||||
$this->statsd->count($metric . $tags, $value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($storage >= 1) {
|
|
||||||
$tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? '');
|
|
||||||
$this->statsd->count('storage.all' . $tags, $storage);
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->reset();
|
$this->reset();
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue