Merge pull request #3648 from appwrite/feat-refactor-usage-stats
This commit is contained in:
commit
d7d3d6c808
3 changed files with 267 additions and 66 deletions
2
composer.lock
generated
2
composer.lock
generated
|
@ -5370,5 +5370,5 @@
|
|||
"platform-overrides": {
|
||||
"php": "8.0"
|
||||
},
|
||||
"plugin-api-version": "2.2.0"
|
||||
"plugin-api-version": "2.3.0"
|
||||
}
|
||||
|
|
|
@ -28,9 +28,6 @@ class Usage
|
|||
'outbound' => [
|
||||
'table' => 'appwrite_usage_network_outbound',
|
||||
],
|
||||
'executions' => [
|
||||
'table' => 'appwrite_usage_executions_all',
|
||||
],
|
||||
'databases.create' => [
|
||||
'table' => 'appwrite_usage_databases_create',
|
||||
],
|
||||
|
@ -225,17 +222,9 @@ class Usage
|
|||
],
|
||||
];
|
||||
|
||||
protected array $periods = [
|
||||
[
|
||||
'key' => '30m',
|
||||
'multiplier' => 1800,
|
||||
'startTime' => '-24 hours',
|
||||
],
|
||||
[
|
||||
'key' => '1d',
|
||||
'multiplier' => 86400,
|
||||
'startTime' => '-90 days',
|
||||
],
|
||||
protected array $period = [
|
||||
'key' => '30m',
|
||||
'startTime' => '-24 hours',
|
||||
];
|
||||
|
||||
public function __construct(Database $database, InfluxDatabase $influxDB, callable $errorHandler = null)
|
||||
|
@ -378,15 +367,13 @@ class Usage
|
|||
public function collect(): void
|
||||
{
|
||||
foreach ($this->metrics as $metric => $options) { //for each metrics
|
||||
foreach ($this->periods as $period) { // aggregate data for each period
|
||||
try {
|
||||
$this->syncFromInfluxDB($metric, $options, $period);
|
||||
} catch (\Exception $e) {
|
||||
if (is_callable($this->errorHandler)) {
|
||||
call_user_func($this->errorHandler, $e);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
try {
|
||||
$this->syncFromInfluxDB($metric, $options, $this->period);
|
||||
} catch (\Exception $e) {
|
||||
if (is_callable($this->errorHandler)) {
|
||||
call_user_func($this->errorHandler, $e);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,14 +4,51 @@ namespace Appwrite\Stats;
|
|||
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
|
||||
class UsageDB extends Usage
|
||||
{
|
||||
protected array $periods = [
|
||||
[
|
||||
'key' => '30m',
|
||||
'multiplier' => 1800,
|
||||
],
|
||||
[
|
||||
'key' => '1d',
|
||||
'multiplier' => 86400,
|
||||
],
|
||||
];
|
||||
|
||||
public function __construct(Database $database, callable $errorHandler = null)
|
||||
{
|
||||
$this->database = $database;
|
||||
$this->errorHandler = $errorHandler;
|
||||
}
|
||||
/**
|
||||
* Create Per Period Metric
|
||||
* Create given metric for each defined period
|
||||
*
|
||||
* @param string $projectId
|
||||
* @param string $metric
|
||||
* @param int $value
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void
|
||||
{
|
||||
foreach ($this->periods as $options) {
|
||||
$period = $options['key'];
|
||||
$time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']);
|
||||
$this->createOrUpdateMetric($projectId, $metric, $period, $time, $value);
|
||||
}
|
||||
|
||||
// Required for billing
|
||||
if ($monthly) {
|
||||
$time = strtotime("first day of the month");
|
||||
$this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create or Update Mertic
|
||||
* Create or update each metric in the stats collection for the given project
|
||||
|
@ -22,38 +59,34 @@ class UsageDB extends Usage
|
|||
*
|
||||
* @return void
|
||||
*/
|
||||
private function createOrUpdateMetric(string $projectId, string $metric, int $value): void
|
||||
private function createOrUpdateMetric(string $projectId, string $metric, string $period, int $time, int $value): void
|
||||
{
|
||||
foreach ($this->periods as $options) {
|
||||
$period = $options['key'];
|
||||
$time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']);
|
||||
$id = \md5("{$time}_{$period}_{$metric}");
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
try {
|
||||
$document = $this->database->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$this->database->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'period' => $period,
|
||||
'time' => $time,
|
||||
'metric' => $metric,
|
||||
'value' => $value,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$this->database->updateDocument(
|
||||
'stats',
|
||||
$document->getId(),
|
||||
$document->setAttribute('value', $value)
|
||||
);
|
||||
}
|
||||
} catch (\Exception$e) { // if projects are deleted this might fail
|
||||
if (is_callable($this->errorHandler)) {
|
||||
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
try {
|
||||
$document = $this->database->getDocument('stats', $id);
|
||||
if ($document->isEmpty()) {
|
||||
$this->database->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'period' => $period,
|
||||
'time' => $time,
|
||||
'metric' => $metric,
|
||||
'value' => $value,
|
||||
'type' => 1,
|
||||
]));
|
||||
} else {
|
||||
$this->database->updateDocument(
|
||||
'stats',
|
||||
$document->getId(),
|
||||
$document->setAttribute('value', $value)
|
||||
);
|
||||
}
|
||||
} catch (\Exception$e) { // if projects are deleted this might fail
|
||||
if (is_callable($this->errorHandler)) {
|
||||
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -111,7 +144,6 @@ class UsageDB extends Usage
|
|||
* @param string $collection
|
||||
* @param string $attribute
|
||||
* @param string $metric
|
||||
* @param int $multiplier
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
|
@ -122,7 +154,7 @@ class UsageDB extends Usage
|
|||
try {
|
||||
$sum = $this->database->sum($collection, $attribute);
|
||||
$sum = (int) ($sum * $multiplier);
|
||||
$this->createOrUpdateMetric($projectId, $metric, $sum);
|
||||
$this->createPerPeriodMetric($projectId, $metric, $sum);
|
||||
return $sum;
|
||||
} catch (\Exception $e) {
|
||||
if (is_callable($this->errorHandler)) {
|
||||
|
@ -150,7 +182,7 @@ class UsageDB extends Usage
|
|||
|
||||
try {
|
||||
$count = $this->database->count($collection);
|
||||
$this->createOrUpdateMetric($projectId, $metric, $count);
|
||||
$this->createPerPeriodMetric($projectId, $metric, $count);
|
||||
return $count;
|
||||
} catch (\Exception $e) {
|
||||
if (is_callable($this->errorHandler)) {
|
||||
|
@ -218,16 +250,15 @@ class UsageDB extends Usage
|
|||
$projectFilesTotal += $sum;
|
||||
});
|
||||
|
||||
$this->createOrUpdateMetric($projectId, 'storage.files.count', $projectFilesCount);
|
||||
$this->createOrUpdateMetric($projectId, 'storage.files.total', $projectFilesTotal);
|
||||
$this->createPerPeriodMetric($projectId, 'storage.files.count', $projectFilesCount);
|
||||
$this->createPerPeriodMetric($projectId, 'storage.files.total', $projectFilesTotal);
|
||||
|
||||
$this->createOrUpdateMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal);
|
||||
$this->createPerPeriodMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal);
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage Stats
|
||||
* Metrics: storage.total, storage.files.total, storage.buckets.{bucketId}.files.total,
|
||||
* storage.buckets.count, storage.files.count, storage.buckets.{bucketId}.files.count
|
||||
* Compute Stats
|
||||
* Metrics: functions.executionTime, functions.buildTime, functions.compute,
|
||||
*
|
||||
* @param string $projectId
|
||||
*
|
||||
|
@ -238,7 +269,7 @@ class UsageDB extends Usage
|
|||
$executionTotal = $this->sum($projectId, 'executions', 'time', 'functions.executionTime', 1000); // in ms
|
||||
$buildTotal = $this->sum($projectId, 'builds', 'duration', 'functions.buildTime', 1000); // in ms
|
||||
|
||||
$this->createOrUpdateMetric($projectId, 'functions.compute', $executionTotal + $buildTotal); //in ms
|
||||
$this->createPerPeriodMetric($projectId, 'functions.compute', $executionTotal + $buildTotal); //in ms
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -272,11 +303,185 @@ class UsageDB extends Usage
|
|||
$databaseDocumentsCount += $count;
|
||||
});
|
||||
|
||||
$this->createOrUpdateMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount);
|
||||
$this->createPerPeriodMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount);
|
||||
});
|
||||
|
||||
$this->createOrUpdateMetric($projectId, 'databases.collections.count', $projectCollectionsCount);
|
||||
$this->createOrUpdateMetric($projectId, 'databases.documents.count', $projectDocumentsCount);
|
||||
$this->createPerPeriodMetric($projectId, 'databases.collections.count', $projectCollectionsCount);
|
||||
$this->createPerPeriodMetric($projectId, 'databases.documents.count', $projectDocumentsCount);
|
||||
}
|
||||
|
||||
protected function aggregateDatabaseMetrics(string $projectId): void
|
||||
{
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
$databasesGeneralMetrics = [
|
||||
'databases.create',
|
||||
'databases.read',
|
||||
'databases.update',
|
||||
'databases.delete',
|
||||
'databases.collections.create',
|
||||
'databases.collections.read',
|
||||
'databases.collections.update',
|
||||
'databases.collections.delete',
|
||||
'databases.documents.create',
|
||||
'databases.documents.read',
|
||||
'databases.documents.update',
|
||||
'databases.documents.delete',
|
||||
];
|
||||
|
||||
foreach ($databasesGeneralMetrics as $metric) {
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
}
|
||||
|
||||
$databasesDatabaseMetrics = [
|
||||
'databases.databaseId.collections.create',
|
||||
'databases.databaseId.collections.read',
|
||||
'databases.databaseId.collections.update',
|
||||
'databases.databaseId.collections.delete',
|
||||
'databases.databaseId.documents.create',
|
||||
'databases.databaseId.documents.read',
|
||||
'databases.databaseId.documents.update',
|
||||
'databases.databaseId.documents.delete',
|
||||
];
|
||||
|
||||
$this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) {
|
||||
$databaseId = $database->getId();
|
||||
foreach ($databasesDatabaseMetrics as $metric) {
|
||||
$metric = str_replace('databaseId', $databaseId, $metric);
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
}
|
||||
|
||||
$databasesCollectionMetrics = [
|
||||
'databases.' . $databaseId . '.collections.collectionId.documents.create',
|
||||
'databases.' . $databaseId . '.collections.collectionId.documents.read',
|
||||
'databases.' . $databaseId . '.collections.collectionId.documents.update',
|
||||
'databases.' . $databaseId . '.collections.collectionId.documents.delete',
|
||||
];
|
||||
|
||||
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) {
|
||||
$collectionId = $collection->getId();
|
||||
foreach ($databasesCollectionMetrics as $metric) {
|
||||
$metric = str_replace('collectionId', $collectionId, $metric);
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
protected function aggregateStorageMetrics(string $projectId): void
|
||||
{
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
$storageGeneralMetrics = [
|
||||
'storage.buckets.create',
|
||||
'storage.buckets.read',
|
||||
'storage.buckets.update',
|
||||
'storage.buckets.delete',
|
||||
'storage.files.create',
|
||||
'storage.files.read',
|
||||
'storage.files.update',
|
||||
'storage.files.delete',
|
||||
];
|
||||
|
||||
foreach ($storageGeneralMetrics as $metric) {
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
}
|
||||
|
||||
$storageBucketMetrics = [
|
||||
'storage.buckets.bucketId.files.create',
|
||||
'storage.buckets.bucketId.files.read',
|
||||
'storage.buckets.bucketId.files.update',
|
||||
'storage.buckets.bucketId.files.delete',
|
||||
];
|
||||
|
||||
$this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) {
|
||||
$bucketId = $bucket->getId();
|
||||
foreach ($storageBucketMetrics as $metric) {
|
||||
$metric = str_replace('bucketId', $bucketId, $metric);
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
protected function aggregateFunctionMetrics(string $projectId): void
|
||||
{
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
|
||||
$this->aggregateDailyMetric($projectId, 'functions.executions');
|
||||
$this->aggregateDailyMetric($projectId, 'functions.builds');
|
||||
$this->aggregateDailyMetric($projectId, 'functions.failures');
|
||||
|
||||
$functionMetrics = [
|
||||
'functions.functionId.executions',
|
||||
'functions.functionId.builds',
|
||||
'functions.functionId.compute',
|
||||
'function.functionId.executions.failure',
|
||||
'function.functionId.builds.failure',
|
||||
];
|
||||
|
||||
$this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) {
|
||||
$functionId = $function->getId();
|
||||
foreach ($functionMetrics as $metric) {
|
||||
$metric = str_replace('functionId', $functionId, $metric);
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
protected function aggregateUsersMetrics(string $projectId): void
|
||||
{
|
||||
$metrics = [
|
||||
'users.create',
|
||||
'users.read',
|
||||
'users.update',
|
||||
'users.delete',
|
||||
'users.sessions.create',
|
||||
'users.sessions.delete'
|
||||
];
|
||||
|
||||
foreach ($metrics as $metric) {
|
||||
$this->aggregateDailyMetric($projectId, $metric);
|
||||
}
|
||||
}
|
||||
|
||||
protected function aggregateGeneralMetrics(string $projectId): void
|
||||
{
|
||||
$this->aggregateDailyMetric($projectId, 'requests');
|
||||
$this->aggregateDailyMetric($projectId, 'network');
|
||||
$this->aggregateDailyMetric($projectId, 'inbound');
|
||||
$this->aggregateDailyMetric($projectId, 'outbound');
|
||||
|
||||
//Required for billing
|
||||
$this->aggregateMonthlyMetric($projectId, 'inbound');
|
||||
$this->aggregateMonthlyMetric($projectId, 'outbound');
|
||||
}
|
||||
|
||||
protected function aggregateDailyMetric(string $projectId, string $metric): void
|
||||
{
|
||||
$beginOfDay = strtotime("today");
|
||||
$endOfDay = strtotime("tomorrow", $beginOfDay) - 1;
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
$value = (int) $this->database->sum('stats', 'value', [
|
||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||
new Query('period', Query::TYPE_EQUAL, ['30m']),
|
||||
new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfDay]),
|
||||
new Query('time', Query::TYPE_LESSEREQUAL, [$endOfDay]),
|
||||
]);
|
||||
$this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value);
|
||||
}
|
||||
|
||||
protected function aggregateMonthlyMetric(string $projectId, string $metric): void
|
||||
{
|
||||
$beginOfMonth = strtotime("first day of the month");
|
||||
$endOfMonth = strtotime("last day of the month");
|
||||
$this->database->setNamespace('_' . $projectId);
|
||||
$value = (int) $this->database->sum('stats', 'value', [
|
||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||
new Query('period', Query::TYPE_EQUAL, ['1d']),
|
||||
new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfMonth]),
|
||||
new Query('time', Query::TYPE_LESSEREQUAL, [$endOfMonth]),
|
||||
]);
|
||||
$this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -289,10 +494,19 @@ class UsageDB extends Usage
|
|||
{
|
||||
$this->foreachDocument('console', 'projects', [], function (Document $project) {
|
||||
$projectId = $project->getInternalId();
|
||||
$this->computeStats($projectId);
|
||||
|
||||
$this->usersStats($projectId);
|
||||
$this->databaseStats($projectId);
|
||||
$this->storageStats($projectId);
|
||||
$this->computeStats($projectId);
|
||||
|
||||
// Aggregate new metrics from already collected usage metrics
|
||||
// for lower time period (1day and 1 month metric from 30 minute metrics)
|
||||
$this->aggregateGeneralMetrics($projectId);
|
||||
$this->aggregateFunctionMetrics($projectId);
|
||||
$this->aggregateDatabaseMetrics($projectId);
|
||||
$this->aggregateStorageMetrics($projectId);
|
||||
$this->aggregateUsersMetrics($projectId);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue