From 7affd0579e1ec42032ac41e56dd393109857a36f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Aug 2022 01:22:18 +0000 Subject: [PATCH] usage refactor setups --- src/Appwrite/Stats/UsageDB.php | 512 ------------------ src/Appwrite/Usage/Calculator.php | 7 + src/Appwrite/Usage/Calculators/Aggregator.php | 205 +++++++ src/Appwrite/Usage/Calculators/Database.php | 318 +++++++++++ .../Calculators/TimeSeries.php} | 200 +++---- src/Appwrite/{Stats => Usage}/Stats.php | 160 +++--- 6 files changed, 715 insertions(+), 687 deletions(-) delete mode 100644 src/Appwrite/Stats/UsageDB.php create mode 100644 src/Appwrite/Usage/Calculator.php create mode 100644 src/Appwrite/Usage/Calculators/Aggregator.php create mode 100644 src/Appwrite/Usage/Calculators/Database.php rename src/Appwrite/{Stats/Usage.php => Usage/Calculators/TimeSeries.php} (66%) rename src/Appwrite/{Stats => Usage}/Stats.php (70%) diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php deleted file mode 100644 index 1caef7acd..000000000 --- a/src/Appwrite/Stats/UsageDB.php +++ /dev/null @@ -1,512 +0,0 @@ - '30m', - 'multiplier' => 1800, - ], - [ - 'key' => '1d', - 'multiplier' => 86400, - ], - ]; - - public function __construct(Database $database, callable $errorHandler = null) - { - $this->database = $database; - $this->errorHandler = $errorHandler; - } - /** - * Create Per Period Metric - * Create given metric for each defined period - * - * @param string $projectId - * @param string $metric - * @param int $value - * - * @return void - */ - private function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void - { - foreach ($this->periods as $options) { - $period = $options['key']; - $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']); - $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value); - } - - // Required for billing - if ($monthly) { - $time = strtotime("first day of the month"); - $this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value); - } - } - - /** - * Create or Update Mertic - * Create or update each metric in the stats collection for the given project - * - * @param string $projectId - * @param string $metric - * @param int $value - * - * @return void - */ - private function createOrUpdateMetric(string $projectId, string $metric, string $period, int $time, int $value): void - { - $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_' . $projectId); - - try { - $document = $this->database->getDocument('stats', $id); - if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 1, - ])); - } else { - $this->database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); - } - } catch (\Exception$e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } - } - } - - /** - * Foreach Document - * Call provided callback for each document in the collection - * - * @param string $projectId - * @param string $collection - * @param array $queries - * @param callable $callback - * - * @return void - */ - private function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void - { - $limit = 50; - $results = []; - $sum = $limit; - $latestDocument = null; - $this->database->setNamespace('_' . $projectId); - - while ($sum === $limit) { - try { - $results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}"); - return; - } else { - throw $e; - } - } - if (empty($results)) { - return; - } - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - } - $latestDocument = $results[array_key_last($results)]; - } - } - - /** - * Sum - * Calculate sum of a attribute of documents in collection - * - * @param string $projectId - * @param string $collection - * @param string $attribute - * @param string $metric - * - * @return int - */ - private function sum(string $projectId, string $collection, string $attribute, string $metric, int $multiplier = 1): int - { - $this->database->setNamespace('_' . $projectId); - - try { - $sum = $this->database->sum($collection, $attribute); - $sum = (int) ($sum * $multiplier); - $this->createPerPeriodMetric($projectId, $metric, $sum); - return $sum; - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}"); - } else { - throw $e; - } - } - return 0; - } - - /** - * Count - * Count number of documents in collection - * - * @param string $projectId - * @param string $collection - * @param string $metric - * - * @return int - */ - private function count(string $projectId, string $collection, string $metric): int - { - $this->database->setNamespace('_' . $projectId); - - try { - $count = $this->database->count($collection); - $this->createPerPeriodMetric($projectId, $metric, $count); - return $count; - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}"); - } else { - throw $e; - } - } - return 0; - } - - /** - * Deployments Total - * Total sum of storage used by deployments - * - * @param string $projectId - * - * @return int - */ - private function deploymentsTotal(string $projectId): int - { - return $this->sum($projectId, 'deployments', 'size', 'stroage.deployments.total'); - } - - /** - * Users Stats - * Metric: users.count - * - * @param string $projectId - * - * @return void - */ - private function usersStats(string $projectId): void - { - $this->count($projectId, 'users', 'users.count'); - } - - /** - * Storage Stats - * Metrics: storage.total, storage.files.total, storage.buckets.{bucketId}.files.total, - * storage.buckets.count, storage.files.count, storage.buckets.{bucketId}.files.count - * - * @param string $projectId - * - * @return void - */ - private function storageStats(string $projectId): void - { - $deploymentsTotal = $this->deploymentsTotal($projectId); - - $projectFilesTotal = 0; - $projectFilesCount = 0; - - $metric = 'storage.buckets.count'; - $this->count($projectId, 'buckets', $metric); - - $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) { - $metric = "storage.buckets.{$bucket->getId()}.files.count"; - - $count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric); - $projectFilesCount += $count; - - $metric = "storage.buckets.{$bucket->getId()}.files.total"; - $sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric); - $projectFilesTotal += $sum; - }); - - $this->createPerPeriodMetric($projectId, 'storage.files.count', $projectFilesCount); - $this->createPerPeriodMetric($projectId, 'storage.files.total', $projectFilesTotal); - - $this->createPerPeriodMetric($projectId, 'storage.total', $projectFilesTotal + $deploymentsTotal); - } - - /** - * Compute Stats - * Metrics: functions.executionTime, functions.buildTime, functions.compute, - * - * @param string $projectId - * - * @return void - */ - private function computeStats(string $projectId): void - { - $executionTotal = $this->sum($projectId, 'executions', 'time', 'functions.executionTime', 1000); // in ms - $buildTotal = $this->sum($projectId, 'builds', 'duration', 'functions.buildTime', 1000); // in ms - - $this->createPerPeriodMetric($projectId, 'functions.compute', $executionTotal + $buildTotal); //in ms - } - - /** - * Database Stats - * Collect all database stats - * Metrics: database.collections.count, database.collections.{collectionId}.documents.count, - * database.documents.count - * - * @param string $projectId - * - * @return void - */ - private function databaseStats(string $projectId): void - { - $projectDocumentsCount = 0; - $projectCollectionsCount = 0; - - $this->count($projectId, 'databases', 'databases.count'); - - $this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) { - $metric = "databases.{$database->getId()}.collections.count"; - $count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric); - $projectCollectionsCount += $count; - $databaseDocumentsCount = 0; - - $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) { - $metric = "databases.{$database->getId()}.collections.{$collection->getId()}.documents.count"; - - $count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric); - $projectDocumentsCount += $count; - $databaseDocumentsCount += $count; - }); - - $this->createPerPeriodMetric($projectId, "databases.{$database->getId()}.documents.count", $databaseDocumentsCount); - }); - - $this->createPerPeriodMetric($projectId, 'databases.collections.count', $projectCollectionsCount); - $this->createPerPeriodMetric($projectId, 'databases.documents.count', $projectDocumentsCount); - } - - protected function aggregateDatabaseMetrics(string $projectId): void - { - $this->database->setNamespace('_' . $projectId); - - $databasesGeneralMetrics = [ - 'databases.create', - 'databases.read', - 'databases.update', - 'databases.delete', - 'databases.collections.create', - 'databases.collections.read', - 'databases.collections.update', - 'databases.collections.delete', - 'databases.documents.create', - 'databases.documents.read', - 'databases.documents.update', - 'databases.documents.delete', - ]; - - foreach ($databasesGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - } - - $databasesDatabaseMetrics = [ - 'databases.databaseId.collections.create', - 'databases.databaseId.collections.read', - 'databases.databaseId.collections.update', - 'databases.databaseId.collections.delete', - 'databases.databaseId.documents.create', - 'databases.databaseId.documents.read', - 'databases.databaseId.documents.update', - 'databases.databaseId.documents.delete', - ]; - - $this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) { - $databaseId = $database->getId(); - foreach ($databasesDatabaseMetrics as $metric) { - $metric = str_replace('databaseId', $databaseId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - } - - $databasesCollectionMetrics = [ - 'databases.' . $databaseId . '.collections.collectionId.documents.create', - 'databases.' . $databaseId . '.collections.collectionId.documents.read', - 'databases.' . $databaseId . '.collections.collectionId.documents.update', - 'databases.' . $databaseId . '.collections.collectionId.documents.delete', - ]; - - $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) { - $collectionId = $collection->getId(); - foreach ($databasesCollectionMetrics as $metric) { - $metric = str_replace('collectionId', $collectionId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - } - }); - }); - } - - protected function aggregateStorageMetrics(string $projectId): void - { - $this->database->setNamespace('_' . $projectId); - - $storageGeneralMetrics = [ - 'storage.buckets.create', - 'storage.buckets.read', - 'storage.buckets.update', - 'storage.buckets.delete', - 'storage.files.create', - 'storage.files.read', - 'storage.files.update', - 'storage.files.delete', - ]; - - foreach ($storageGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - } - - $storageBucketMetrics = [ - 'storage.buckets.bucketId.files.create', - 'storage.buckets.bucketId.files.read', - 'storage.buckets.bucketId.files.update', - 'storage.buckets.bucketId.files.delete', - ]; - - $this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) { - $bucketId = $bucket->getId(); - foreach ($storageBucketMetrics as $metric) { - $metric = str_replace('bucketId', $bucketId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - } - }); - } - - protected function aggregateFunctionMetrics(string $projectId): void - { - $this->database->setNamespace('_' . $projectId); - - $this->aggregateDailyMetric($projectId, 'functions.executions'); - $this->aggregateDailyMetric($projectId, 'functions.builds'); - $this->aggregateDailyMetric($projectId, 'functions.failures'); - - $functionMetrics = [ - 'functions.functionId.executions', - 'functions.functionId.builds', - 'functions.functionId.compute', - 'function.functionId.executions.failure', - 'function.functionId.builds.failure', - ]; - - $this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) { - $functionId = $function->getId(); - foreach ($functionMetrics as $metric) { - $metric = str_replace('functionId', $functionId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - } - }); - } - - protected function aggregateUsersMetrics(string $projectId): void - { - $metrics = [ - 'users.create', - 'users.read', - 'users.update', - 'users.delete', - 'users.sessions.create', - 'users.sessions.delete' - ]; - - foreach ($metrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - } - } - - protected function aggregateGeneralMetrics(string $projectId): void - { - $this->aggregateDailyMetric($projectId, 'requests'); - $this->aggregateDailyMetric($projectId, 'network'); - $this->aggregateDailyMetric($projectId, 'inbound'); - $this->aggregateDailyMetric($projectId, 'outbound'); - - //Required for billing - $this->aggregateMonthlyMetric($projectId, 'inbound'); - $this->aggregateMonthlyMetric($projectId, 'outbound'); - } - - protected function aggregateDailyMetric(string $projectId, string $metric): void - { - $beginOfDay = strtotime("today"); - $endOfDay = strtotime("tomorrow", $beginOfDay) - 1; - $this->database->setNamespace('_' . $projectId); - $value = (int) $this->database->sum('stats', 'value', [ - new Query('metric', Query::TYPE_EQUAL, [$metric]), - new Query('period', Query::TYPE_EQUAL, ['30m']), - new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfDay]), - new Query('time', Query::TYPE_LESSEREQUAL, [$endOfDay]), - ]); - $this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value); - } - - protected function aggregateMonthlyMetric(string $projectId, string $metric): void - { - $beginOfMonth = strtotime("first day of the month"); - $endOfMonth = strtotime("last day of the month"); - $this->database->setNamespace('_' . $projectId); - $value = (int) $this->database->sum('stats', 'value', [ - new Query('metric', Query::TYPE_EQUAL, [$metric]), - new Query('period', Query::TYPE_EQUAL, ['1d']), - new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfMonth]), - new Query('time', Query::TYPE_LESSEREQUAL, [$endOfMonth]), - ]); - $this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value); - } - - /** - * Collect Stats - * Collect all database related stats - * - * @return void - */ - public function collect(): void - { - $this->foreachDocument('console', 'projects', [], function (Document $project) { - $projectId = $project->getInternalId(); - - $this->usersStats($projectId); - $this->databaseStats($projectId); - $this->storageStats($projectId); - $this->computeStats($projectId); - - // Aggregate new metrics from already collected usage metrics - // for lower time period (1day and 1 month metric from 30 minute metrics) - $this->aggregateGeneralMetrics($projectId); - $this->aggregateFunctionMetrics($projectId); - $this->aggregateDatabaseMetrics($projectId); - $this->aggregateStorageMetrics($projectId); - $this->aggregateUsersMetrics($projectId); - }); - } -} diff --git a/src/Appwrite/Usage/Calculator.php b/src/Appwrite/Usage/Calculator.php new file mode 100644 index 000000000..66c1b9f8f --- /dev/null +++ b/src/Appwrite/Usage/Calculator.php @@ -0,0 +1,7 @@ +database->setNamespace('_' . $projectId); + + $databasesGeneralMetrics = [ + 'databases.create', + 'databases.read', + 'databases.update', + 'databases.delete', + 'databases.collections.create', + 'databases.collections.read', + 'databases.collections.update', + 'databases.collections.delete', + 'databases.documents.create', + 'databases.documents.read', + 'databases.documents.update', + 'databases.documents.delete', + ]; + + foreach ($databasesGeneralMetrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + + $databasesDatabaseMetrics = [ + 'databases.databaseId.collections.create', + 'databases.databaseId.collections.read', + 'databases.databaseId.collections.update', + 'databases.databaseId.collections.delete', + 'databases.databaseId.documents.create', + 'databases.databaseId.documents.read', + 'databases.databaseId.documents.update', + 'databases.databaseId.documents.delete', + ]; + + $this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) { + $databaseId = $database->getId(); + foreach ($databasesDatabaseMetrics as $metric) { + $metric = str_replace('databaseId', $databaseId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + + $databasesCollectionMetrics = [ + 'databases.' . $databaseId . '.collections.collectionId.documents.create', + 'databases.' . $databaseId . '.collections.collectionId.documents.read', + 'databases.' . $databaseId . '.collections.collectionId.documents.update', + 'databases.' . $databaseId . '.collections.collectionId.documents.delete', + ]; + + $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) { + $collectionId = $collection->getId(); + foreach ($databasesCollectionMetrics as $metric) { + $metric = str_replace('collectionId', $collectionId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + }); + } + + protected function aggregateStorageMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $storageGeneralMetrics = [ + 'storage.buckets.create', + 'storage.buckets.read', + 'storage.buckets.update', + 'storage.buckets.delete', + 'storage.files.create', + 'storage.files.read', + 'storage.files.update', + 'storage.files.delete', + ]; + + foreach ($storageGeneralMetrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + + $storageBucketMetrics = [ + 'storage.buckets.bucketId.files.create', + 'storage.buckets.bucketId.files.read', + 'storage.buckets.bucketId.files.update', + 'storage.buckets.bucketId.files.delete', + ]; + + $this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) { + $bucketId = $bucket->getId(); + foreach ($storageBucketMetrics as $metric) { + $metric = str_replace('bucketId', $bucketId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + } + + protected function aggregateFunctionMetrics(string $projectId): void + { + $this->database->setNamespace('_' . $projectId); + + $this->aggregateDailyMetric($projectId, 'functions.executions'); + $this->aggregateDailyMetric($projectId, 'functions.builds'); + $this->aggregateDailyMetric($projectId, 'functions.failures'); + + $functionMetrics = [ + 'functions.functionId.executions', + 'functions.functionId.builds', + 'functions.functionId.compute', + 'function.functionId.executions.failure', + 'function.functionId.builds.failure', + ]; + + $this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) { + $functionId = $function->getId(); + foreach ($functionMetrics as $metric) { + $metric = str_replace('functionId', $functionId, $metric); + $this->aggregateDailyMetric($projectId, $metric); + } + }); + } + + protected function aggregateUsersMetrics(string $projectId): void + { + $metrics = [ + 'users.create', + 'users.read', + 'users.update', + 'users.delete', + 'users.sessions.create', + 'users.sessions.delete' + ]; + + foreach ($metrics as $metric) { + $this->aggregateDailyMetric($projectId, $metric); + } + } + + protected function aggregateGeneralMetrics(string $projectId): void + { + $this->aggregateDailyMetric($projectId, 'requests'); + $this->aggregateDailyMetric($projectId, 'network'); + $this->aggregateDailyMetric($projectId, 'inbound'); + $this->aggregateDailyMetric($projectId, 'outbound'); + + //Required for billing + $this->aggregateMonthlyMetric($projectId, 'inbound'); + $this->aggregateMonthlyMetric($projectId, 'outbound'); + } + + protected function aggregateDailyMetric(string $projectId, string $metric): void + { + $beginOfDay = strtotime("today"); + $endOfDay = strtotime("tomorrow", $beginOfDay) - 1; + $this->database->setNamespace('_' . $projectId); + $value = (int) $this->database->sum('stats', 'value', [ + new Query('metric', Query::TYPE_EQUAL, [$metric]), + new Query('period', Query::TYPE_EQUAL, ['30m']), + new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfDay]), + new Query('time', Query::TYPE_LESSEREQUAL, [$endOfDay]), + ]); + $this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value); + } + + protected function aggregateMonthlyMetric(string $projectId, string $metric): void + { + $beginOfMonth = strtotime("first day of the month"); + $endOfMonth = strtotime("last day of the month"); + $this->database->setNamespace('_' . $projectId); + $value = (int) $this->database->sum('stats', 'value', [ + new Query('metric', Query::TYPE_EQUAL, [$metric]), + new Query('period', Query::TYPE_EQUAL, ['1d']), + new Query('time', Query::TYPE_GREATEREQUAL, [$beginOfMonth]), + new Query('time', Query::TYPE_LESSEREQUAL, [$endOfMonth]), + ]); + $this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value); + } + + /** + * Collect Stats + * Collect all database related stats + * + * @return void + */ + public function collect(): void + { + $this->foreachDocument('console', 'projects', [], function (Document $project) { + $projectId = $project->getInternalId(); + + // Aggregate new metrics from already collected usage metrics + // for lower time period (1day and 1 month metric from 30 minute metrics) + $this->aggregateGeneralMetrics($projectId); + $this->aggregateFunctionMetrics($projectId); + $this->aggregateDatabaseMetrics($projectId); + $this->aggregateStorageMetrics($projectId); + $this->aggregateUsersMetrics($projectId); + }); + } +} \ No newline at end of file diff --git a/src/Appwrite/Usage/Calculators/Database.php b/src/Appwrite/Usage/Calculators/Database.php new file mode 100644 index 000000000..43bf40558 --- /dev/null +++ b/src/Appwrite/Usage/Calculators/Database.php @@ -0,0 +1,318 @@ + '30m', + 'multiplier' => 1800, + ], + [ + 'key' => '1d', + 'multiplier' => 86400, + ], + ]; + + public function __construct(UtopiaDatabase $database, callable $errorHandler = null) + { + $this->database = $database; + $this->errorHandler = $errorHandler; + } + + /** + * Create Per Period Metric + * Create given metric for each defined period + * + * @param string $projectId + * @param string $metric + * @param int $value + * + * @return void + */ + protected function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void + { + foreach ($this->periods as $options) { + $period = $options['key']; + $time = (int) (floor(time() / $options['multiplier']) * $options['multiplier']); + $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value); + } + + // Required for billing + if ($monthly) { + $time = strtotime("first day of the month"); + $this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value); + } + } + + /** + * Create or Update Mertic + * Create or update each metric in the stats collection for the given project + * + * @param string $projectId + * @param string $metric + * @param int $value + * + * @return void + */ + protected function createOrUpdateMetric(string $projectId, string $metric, string $period, int $time, int $value): void + { + $id = \md5("{$time}_{$period}_{$metric}"); + $this->database->setNamespace('_' . $projectId); + + try { + $document = $this->database->getDocument('stats', $id); + if ($document->isEmpty()) { + $this->database->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => 1, + ])); + } else { + $this->database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + } catch (\Exception$e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; + } + } + } + + /** + * Foreach Document + * Call provided callback for each document in the collection + * + * @param string $projectId + * @param string $collection + * @param array $queries + * @param callable $callback + * + * @return void + */ + protected function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void + { + $limit = 50; + $results = []; + $sum = $limit; + $latestDocument = null; + $this->database->setNamespace('_' . $projectId); + + while ($sum === $limit) { + try { + $results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}"); + return; + } else { + throw $e; + } + } + if (empty($results)) { + return; + } + + $sum = count($results); + + foreach ($results as $document) { + if (is_callable($callback)) { + $callback($document); + } + } + $latestDocument = $results[array_key_last($results)]; + } + } + + /** + * Sum + * Calculate sum of a attribute of documents in collection + * + * @param string $projectId + * @param string $collection + * @param string $attribute + * @param string $metric + * + * @return int + */ + private function sum(string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int + { + $this->database->setNamespace('_' . $projectId); + + try { + $sum = $this->database->sum($collection, $attribute); + $sum = (int) ($sum * $multiplier); + + if(!is_null($metric)) { + $this->createPerPeriodMetric($projectId, $metric, $sum); + } + return $sum; + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}"); + } else { + throw $e; + } + } + return 0; + } + + /** + * Count + * Count number of documents in collection + * + * @param string $projectId + * @param string $collection + * @param string? $metric + * + * @return int + */ + private function count(string $projectId, string $collection, string $metric = null): int + { + $this->database->setNamespace('_' . $projectId); + + try { + $count = $this->database->count($collection); + if(!is_null($metric)) { + $this->createPerPeriodMetric($projectId, (string) $metric, $count); + } + return $count; + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}"); + } else { + throw $e; + } + } + return 0; + } + + /** + * Deployments Total + * Total sum of storage used by deployments + * + * @param string $projectId + * + * @return int + */ + private function deploymentsTotal(string $projectId): int + { + return $this->sum($projectId, 'deployments', 'size', 'deployments.$all.storage.size'); + } + + /** + * Users Stats + * Metric: users.count + * + * @param string $projectId + * + * @return void + */ + private function usersStats(string $projectId): void + { + $this->count($projectId, 'users', 'users.$all.count.total'); + } + + /** + * Storage Stats + * Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total, + * files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size + * + * @param string $projectId + * + * @return void + */ + private function storageStats(string $projectId): void + { + $projectFilesTotal = 0; + $projectFilesCount = 0; + + $metric = 'buckets.$all.count.total'; + $this->count($projectId, 'buckets', $metric); + + $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) { + $metric = "files.{$bucket->getId()}.count.total"; + $count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric); + $projectFilesCount += $count; + + $metric = "files.{$bucket->getId()}.storage.size"; + $sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric); + $projectFilesTotal += $sum; + }); + + $this->createPerPeriodMetric($projectId, 'files.$all.count.total', $projectFilesCount); + $this->createPerPeriodMetric($projectId, 'files.$all.storage.size', $projectFilesTotal); + + $deploymentsTotal = $this->deploymentsTotal($projectId); + $this->createPerPeriodMetric($projectId, 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal); + } + + /** + * Database Stats + * Collect all database stats + * Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total, + * documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total + * + * @param string $projectId + * + * @return void + */ + private function databaseStats(string $projectId): void + { + $projectDocumentsCount = 0; + $projectCollectionsCount = 0; + + $this->count($projectId, 'databases', 'databases.$all.count.total'); + + $this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) { + $metric = "collections.{$database->getId()}.count.total"; + $count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric); + $projectCollectionsCount += $count; + $databaseDocumentsCount = 0; + + $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) { + $metric = "documents.{$database->getId()}/{$collection->getId()}.count.total"; + + $count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric); + $projectDocumentsCount += $count; + $databaseDocumentsCount += $count; + }); + + $this->createPerPeriodMetric($projectId, "documents.{$database->getId()}.count.total", $databaseDocumentsCount); + }); + + $this->createPerPeriodMetric($projectId, 'collections.$all.count.total', $projectCollectionsCount); + $this->createPerPeriodMetric($projectId, 'documents.$all.count.total', $projectDocumentsCount); + } + + /** + * Collect Stats + * Collect all database related stats + * + * @return void + */ + public function collect(): void + { + $this->foreachDocument('console', 'projects', [], function (Document $project) { + $projectId = $project->getInternalId(); + + $this->usersStats($projectId); + $this->databaseStats($projectId); + $this->storageStats($projectId); + }); + } +} diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Usage/Calculators/TimeSeries.php similarity index 66% rename from src/Appwrite/Stats/Usage.php rename to src/Appwrite/Usage/Calculators/TimeSeries.php index 560bd0ff0..c5d4f75d9 100644 --- a/src/Appwrite/Stats/Usage.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -1,14 +1,15 @@ [ - 'table' => 'appwrite_usage_requests_all', + 'project.$all.network.requests' => [ + 'table' => 'appwrite_usage_network_requests', ], - 'network' => [ - 'table' => 'appwrite_usage_network_all', + 'project.$all.network.bandwidth' => [ + 'table' => 'appwrite_usage_network_bandwidth', ], - 'inbound' => [ + 'project.$all.network.inbound' => [ 'table' => 'appwrite_usage_network_inbound', ], - 'outbound' => [ + 'project.$all.network.outbound' => [ 'table' => 'appwrite_usage_network_outbound', ], - 'databases.create' => [ - 'table' => 'appwrite_usage_databases_create', + /* Users service metrics */ + 'users.$all.requests.create' => [ + 'table' => 'appwrite_usage_users_requests_create', ], - 'databases.read' => [ - 'table' => 'appwrite_usage_databases_read', + 'users.$all.requests.read' => [ + 'table' => 'appwrite_usage_users_requests_read', ], - 'databases.update' => [ - 'table' => 'appwrite_usage_databases_update', + 'users.$all.requests.update' => [ + 'table' => 'appwrite_usage_users_requests_update', ], - 'databases.delete' => [ - 'table' => 'appwrite_usage_databases_delete', + 'users.$all.requests.delete' => [ + 'table' => 'appwrite_usage_users_requests_delete', ], - 'databases.collections.create' => [ - 'table' => 'appwrite_usage_databases_collections_create', + + 'databases.$all.requests.create' => [ + 'table' => 'appwrite_usage_databases_requests_create', ], - 'databases.collections.read' => [ - 'table' => 'appwrite_usage_databases_collections_read', + 'databases.$all.requests.read' => [ + 'table' => 'appwrite_usage_databases_requests_read', ], - 'databases.collections.update' => [ - 'table' => 'appwrite_usage_databases_collections_update', + 'databases.$all.requests.update' => [ + 'table' => 'appwrite_usage_databases_requests_update', ], - 'databases.collections.delete' => [ - 'table' => 'appwrite_usage_databases_collections_delete', + 'databases.$all.requests.delete' => [ + 'table' => 'appwrite_usage_databases_requests_delete', ], - 'databases.documents.create' => [ - 'table' => 'appwrite_usage_databases_documents_create', + + 'collections.$all.requests.create' => [ + 'table' => 'appwrite_usage_dollections_requests_create', ], - 'databases.documents.read' => [ - 'table' => 'appwrite_usage_databases_documents_read', + 'collections.$all.requests.read' => [ + 'table' => 'appwrite_usage_collections_requests_read', ], - 'databases.documents.update' => [ - 'table' => 'appwrite_usage_databases_documents_update', + 'collections.$all.requests.update' => [ + 'table' => 'appwrite_usage_collections_requests_update', ], - 'databases.documents.delete' => [ - 'table' => 'appwrite_usage_databases_documents_delete', + 'collections.$all.requests.delete' => [ + 'table' => 'appwrite_usage_collections_requests_delete', ], - 'databases.databaseId.collections.create' => [ - 'table' => 'appwrite_usage_databases_collections_create', + + 'documents.$all.requests.create' => [ + 'table' => 'appwrite_usage_documents_requests_create', + ], + 'documents.$all.requests.read' => [ + 'table' => 'appwrite_usage_documents_requests_read', + ], + 'documents.$all.requests.update' => [ + 'table' => 'appwrite_usage_documents_requests_update', + ], + 'documents.$all.requests.delete' => [ + 'table' => 'appwrite_usage_documents_requests_delete', + ], + + 'collections.databaseId.requests.create' => [ + 'table' => 'appwrite_usage_collections_requests_create', 'groupBy' => ['databaseId'], ], - 'databases.databaseId.collections.read' => [ - 'table' => 'appwrite_usage_databases_collections_read', + 'collections.databaseId.requests.read' => [ + 'table' => 'appwrite_usage_collections_requests_read', 'groupBy' => ['databaseId'], ], - 'databases.databaseId.collections.update' => [ - 'table' => 'appwrite_usage_databases_collections_update', + 'collections.databaseId.requests.update' => [ + 'table' => 'appwrite_usage_collections_requests_update', 'groupBy' => ['databaseId'], ], - 'databases.databaseId.collections.delete' => [ - 'table' => 'appwrite_usage_databases_collections_delete', + 'collections.databaseId.requests.delete' => [ + 'table' => 'appwrite_usage_collections_requests_delete', 'groupBy' => ['databaseId'], ], - 'databases.databaseId.documents.create' => [ - 'table' => 'appwrite_usage_databases_documents_create', + + 'documents.databaseId.requests.create' => [ + 'table' => 'appwrite_usage_documents_requests_create', 'groupBy' => ['databaseId'], ], - 'databases.databaseId.documents.read' => [ - 'table' => 'appwrite_usage_databases_documents_read', + 'documents.databaseId.requests.read' => [ + 'table' => 'appwrite_usage_documents_requests_read', 'groupBy' => ['databaseId'], ], - 'database.databaseId.documents.update' => [ - 'table' => 'appwrite_usage_databases_documents_update', + 'documents.databaseId.requests.update' => [ + 'table' => 'appwrite_usage_documents_requests_update', 'groupBy' => ['databaseId'], ], - 'databases.databaseId.documents.delete' => [ - 'table' => 'appwrite_usage_databases_documents_delete', + 'documents.databaseId.requests.delete' => [ + 'table' => 'appwrite_usage_documents_requests_delete', 'groupBy' => ['databaseId'], ], - 'databases.databaseId.collections.collectionId.documents.create' => [ - 'table' => 'appwrite_usage_databases_documents_create', + + 'documents.databaseId/collectionId.requests.create' => [ + 'table' => 'appwrite_usage_documents_requests_create', 'groupBy' => ['databaseId', 'collectionId'], ], - 'databases.databaseId.collections.collectionId.documents.read' => [ - 'table' => 'appwrite_usage_databases_documents_read', + 'documents.databaseId/collectionId.requests.read' => [ + 'table' => 'appwrite_usage_documents_requests_read', 'groupBy' => ['databaseId', 'collectionId'], ], - 'databases.databaseId.collections.collectionId.documents.update' => [ - 'table' => 'appwrite_usage_databases_documents_update', + 'documents.databaseId/collectionId.requests.update' => [ + 'table' => 'appwrite_usage_documents_requests_update', 'groupBy' => ['databaseId', 'collectionId'], ], - 'databases.databaseId.collections.collectionId.documents.delete' => [ - 'table' => 'appwrite_usage_databases_documents_delete', + 'documents.databaseId/collectionId.requests.delete' => [ + 'table' => 'appwrite_usage_documents_requests_delete', 'groupBy' => ['databaseId', 'collectionId'], ], - 'storage.buckets.create' => [ - 'table' => 'appwrite_usage_storage_buckets_create', + + 'buckets.$all.requests.create' => [ + 'table' => 'appwrite_usage_buckets_requests_create', ], - 'storage.buckets.read' => [ - 'table' => 'appwrite_usage_storage_buckets_read', + 'buckets.$all.requests.read' => [ + 'table' => 'appwrite_usage_buckets_requests_read', ], - 'storage.buckets.update' => [ - 'table' => 'appwrite_usage_storage_buckets_update', + 'buckets.$all.requests.update' => [ + 'table' => 'appwrite_usage_buckets_requests_update', ], - 'storage.buckets.delete' => [ - 'table' => 'appwrite_usage_storage_buckets_delete', + 'buckets.$all.requests.delete' => [ + 'table' => 'appwrite_usage_buckets_requests_delete', ], - 'storage.files.create' => [ - 'table' => 'appwrite_usage_storage_files_create', + + 'files.$all.requests.create' => [ + 'table' => 'appwrite_usage_files_requests_create', ], - 'storage.files.read' => [ - 'table' => 'appwrite_usage_storage_files_read', + 'files.$all.requests.read' => [ + 'table' => 'appwrite_usage_files_requests_read', ], - 'storage.files.update' => [ - 'table' => 'appwrite_usage_storage_files_update', + 'files.$all.requests.update' => [ + 'table' => 'appwrite_usage_files_requests_update', ], - 'storage.files.delete' => [ - 'table' => 'appwrite_usage_storage_files_delete', + 'files.$all.requests.delete' => [ + 'table' => 'appwrite_usage_files_requests_delete', ], - 'storage.buckets.bucketId.files.create' => [ - 'table' => 'appwrite_usage_storage_files_create', + + 'files.bucketId.requests.create' => [ + 'table' => 'appwrite_usage_files_requests_create', 'groupBy' => ['bucketId'], ], - 'storage.buckets.bucketId.files.read' => [ - 'table' => 'appwrite_usage_storage_files_read', + 'files.bucketId.requests.read' => [ + 'table' => 'appwrite_usage_files_requests_read', 'groupBy' => ['bucketId'], ], - 'storage.buckets.bucketId.files.update' => [ - 'table' => 'appwrite_usage_storage_files_update', + 'files.bucketId.requests.update' => [ + 'table' => 'appwrite_usage_files_requests_update', 'groupBy' => ['bucketId'], ], - 'storage.buckets.bucketId.files.delete' => [ - 'table' => 'appwrite_usage_storage_files_delete', + 'files.bucketId.requests.delete' => [ + 'table' => 'appwrite_usage_files_requests_delete', 'groupBy' => ['bucketId'], ], - 'users.create' => [ - 'table' => 'appwrite_usage_users_create', - ], - 'users.read' => [ - 'table' => 'appwrite_usage_users_read', - ], - 'users.update' => [ - 'table' => 'appwrite_usage_users_update', - ], - 'users.delete' => [ - 'table' => 'appwrite_usage_users_delete', - ], + 'users.sessions.create' => [ 'table' => 'appwrite_usage_users_sessions_create', ], @@ -378,4 +390,4 @@ class Usage } } } -} +} \ No newline at end of file diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Usage/Stats.php similarity index 70% rename from src/Appwrite/Stats/Stats.php rename to src/Appwrite/Usage/Stats.php index ebd785545..68f637d05 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -1,6 +1,6 @@ statsd->setNamespace($this->namespace); if ($httpRequest >= 1) { - $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); + $this->statsd->increment('network.requests' . $tags . ',method=' . \strtolower($httpMethod)); + } + $this->statsd->count('network.inbound' . $tags, $networkRequestSize); + $this->statsd->count('network.outbound' . $tags, $networkResponseSize); + $this->statsd->count('network.bandwidth' . $tags, $networkRequestSize + $networkResponseSize); + + $usersMetrics = [ + 'users.requests.create', + 'users.requests.read', + 'users.requests.update', + 'users.requests.delete', + ]; + + foreach ($usersMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $this->statsd->increment($metric . $tags); + } + } + + $dbMetrics = [ + 'databases.requests.create', + 'databases.requests.read', + 'databases.requests.update', + 'databases.requests.delete', + 'collections.requests.create', + 'collections.requests.read', + 'collections.requests.update', + 'collections.requests.delete', + 'documents.requests.create', + 'documents.requests.read', + 'documents.requests.update', + 'documents.requests.delete', + ]; + + foreach ($dbMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? ''); + $this->statsd->increment($metric . $dbTags); + } + } + + $storageMertics = [ + 'buckets.requests.create', + 'buckets.requests.read', + 'buckets.requests.update', + 'buckets.requests.delete', + 'files.requests.create', + 'files.requests.read', + 'files.requests.update', + 'files.requests.delete', + ]; + + foreach ($storageMertics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->increment($metric . $storageTags); + } + } + + $sessionsMetrics = [ + 'users.sessions.create', + 'users.sessions.delete', + ]; + + foreach ($sessionsMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $sessionTags = $tags . ",provider=" . ($this->params['provider'] ?? ''); + $this->statsd->count($metric . $sessionTags, $value); + } + } + + if ($storage >= 1) { + $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->count('storage.all' . $storageTags, $storage); } if ($functionExecution >= 1) { @@ -119,85 +196,6 @@ class Stats $this->statsd->count('functions.compute.time' . $tags . ',functionId=' . $functionId, $functionCompute); } - $this->statsd->count('network.inbound' . $tags, $networkRequestSize); - $this->statsd->count('network.outbound' . $tags, $networkResponseSize); - $this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize); - - $dbMetrics = [ - 'databases.create', - 'databases.read', - 'databases.update', - 'databases.delete', - 'databases.collections.create', - 'databases.collections.read', - 'databases.collections.update', - 'databases.collections.delete', - 'databases.documents.create', - 'databases.documents.read', - 'databases.documents.update', - 'databases.documents.delete', - ]; - - foreach ($dbMetrics as $metric) { - $value = $this->params[$metric] ?? 0; - if ($value >= 1) { - $tags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? ''); - $this->statsd->increment($metric . $tags); - } - } - - $storageMertics = [ - 'storage.buckets.create', - 'storage.buckets.read', - 'storage.buckets.update', - 'storage.buckets.delete', - 'storage.files.create', - 'storage.files.read', - 'storage.files.update', - 'storage.files.delete', - ]; - - foreach ($storageMertics as $metric) { - $value = $this->params[$metric] ?? 0; - if ($value >= 1) { - $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); - $this->statsd->increment($metric . $tags); - } - } - - $usersMetrics = [ - 'users.create', - 'users.read', - 'users.update', - 'users.delete', - ]; - - foreach ($usersMetrics as $metric) { - $value = $this->params[$metric] ?? 0; - if ($value >= 1) { - $tags = ",projectId={$projectId}"; - $this->statsd->increment($metric . $tags); - } - } - - $sessionsMetrics = [ - 'users.sessions.create', - 'users.sessions.delete', - ]; - - foreach ($sessionsMetrics as $metric) { - $value = $this->params[$metric] ?? 0; - if ($value >= 1) { - $tags = ",projectId={$projectId},provider=" . ($this->params['provider'] ?? ''); - $this->statsd->count($metric . $tags, $value); - } - } - - if ($storage >= 1) { - $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); - $this->statsd->count('storage.all' . $tags, $storage); - } - $this->reset(); }