From 29541617de5aad20b2a37e07592d468044412267 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 16 Jul 2024 13:14:55 +0900 Subject: [PATCH] Start implementing database metrics in usage dump worker --- CONTRIBUTING.md | 2 + app/controllers/shared/api.php | 8 ++- app/init.php | 3 ++ src/Appwrite/Platform/Workers/UsageDump.php | 56 +++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9e3e6fcd81..6395aa656b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -321,8 +321,10 @@ These are the current metrics we collect usage stats for: | databases | Total number of databases per project | | collections | Total number of collections per project | | {databaseInternalId}.collections | Total number of collections per database| +| {databaseInternalId}.storage | Sum of database storage (in bytes) | | documents | Total number of documents per project | | {databaseInternalId}.{collectionInternalId}.documents | Total number of documents per collection | +| {databaseInternalId}.{collectionInternalId}.storage | Sum of database storage used by the collection (in bytes) | | buckets | Total number of buckets per project | | files | Total number of files per project | | {bucketInternalId}.files.storage | Sum of files.storage per bucket (in bytes) | diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 2b0013db29..4074c81932 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -81,6 +81,7 @@ $databaseListener = function (string $event, Document $document, Document $proje break; case $document->getCollection() === 'databases': // databases $queueForUsage + ->addMetric(METRIC_DATABASES_STORAGE, 1) ->addMetric(METRIC_DATABASES, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { @@ -93,7 +94,9 @@ $databaseListener = function (string $event, Document $document, Document $proje $databaseInternalId = $parts[1] ?? 0; $queueForUsage ->addMetric(METRIC_COLLECTIONS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database + ->addMetric(METRIC_DATABASES_STORAGE, 1) + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_STORAGE), 1); // per database ; if ($event === Database::EVENT_DOCUMENT_DELETE) { @@ -108,7 +111,8 @@ $databaseListener = function (string $event, Document $document, Document $proje $queueForUsage ->addMetric(METRIC_DOCUMENTS, $value) // per project ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value) // per collection + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection break; case $document->getCollection() === 'buckets': //buckets $queueForUsage diff --git a/app/init.php b/app/init.php index 325c45a7e8..67b4949c7c 100644 --- a/app/init.php +++ b/app/init.php @@ -216,10 +216,13 @@ const METRIC_MESSAGES_COUNTRY_CODE = '{countryCode}.messages'; const METRIC_SESSIONS = 'sessions'; const METRIC_DATABASES = 'databases'; const METRIC_COLLECTIONS = 'collections'; +const METRIC_DATABASES_STORAGE = 'db_storage'; const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections'; +const METRIC_DATABASE_ID_STORAGE = '{databaseInternalId}.db_storage'; const METRIC_DOCUMENTS = 'documents'; const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.db_storage'; const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index b7097dbb04..7cf4ac1c7d 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -4,9 +4,11 @@ namespace Appwrite\Platform\Workers; use Appwrite\Extend\Exception; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate; +use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; @@ -70,6 +72,11 @@ class UsageDump extends Action continue; } + if (str_ends_with($key, '.db_storage')) { + $this->handleDBStorageCalculation($key, $dbForProject); + return; + } + foreach ($this->periods as $period => $format) { $time = 'inf' === $period ? null : date($format, time()); $id = \md5("{$time}_{$period}_{$key}"); @@ -107,4 +114,53 @@ class UsageDump extends Action } } } + + private function handleDBStorageCalculation(string $key, Database $dbForProject): void + { + $data = explode('.', $key); + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : date($format, time()); + $id = \md5("{$time}_{$period}_{$key}"); + + $value = 0; + $previousValue = 0; + try { + $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value'); + } catch (\Exception $e) { + // No previous value + } + + switch (count($data)) { + // Collection Level + case 3: + $databaseInternalId = $data[0]; + $collectionInternalId = $data[1]; + + $value = $dbForProject->getSizeOfCollection('database_'.$databaseInternalId.'_collection_'.$collectionInternalId); + + // Compare with previous value + $value = $value - $previousValue; + + // Update Collection + + // Update Database + + // Update Project + break; + // Database Level + case 2: + $databaseInternalId = $data[0]; + $collections = $dbForProject->find('database_' . $databaseInternalId); + + foreach ($collections as $collection) { + $value += $dbForProject->getSizeOfCollection($collection->getInternalId()); + } + break; + // Project Level + case 1: + break; + } + } + } }