CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-09-15 00:48:25 +12:00
Code Issues Projects Releases Wiki Activity

Start implementing database metrics in usage dump worker

Browse source
This commit is contained in:
Bradley Schofield 2024-07-16 13:14:55 +09:00
parent d98e04cf63
commit 29541617de
4 changed files with 67 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CONTRIBUTING.md
View file

@ -321,8 +321,10 @@ These are the current metrics we collect usage stats for:
| databases | Total number of databases per project |
| collections | Total number of collections per project |
| {databaseInternalId}.collections | Total number of collections per database|
| {databaseInternalId}.storage | Sum of database storage (in bytes) |
| documents | Total number of documents per project |
| {databaseInternalId}.{collectionInternalId}.documents | Total number of documents per collection |
| {databaseInternalId}.{collectionInternalId}.storage | Sum of database storage used by the collection (in bytes) |
| buckets | Total number of buckets per project |
| files | Total number of files per project |
| {bucketInternalId}.files.storage | Sum of files.storage per bucket (in bytes) |

8
app/controllers/shared/api.php
View file

@ -81,6 +81,7 @@ $databaseListener = function (string $event, Document $document, Document $proje
break;
case $document->getCollection() === 'databases': // databases
$queueForUsage
->addMetric(METRIC_DATABASES_STORAGE, 1)
->addMetric(METRIC_DATABASES, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
@ -93,7 +94,9 @@ $databaseListener = function (string $event, Document $document, Document $proje
$databaseInternalId = $parts[1] ?? 0;
$queueForUsage
->addMetric(METRIC_COLLECTIONS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
->addMetric(METRIC_DATABASES_STORAGE, 1)
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value)
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_STORAGE), 1); // per database
;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
@ -108,7 +111,8 @@ $databaseListener = function (string $event, Document $document, Document $proje
$queueForUsage
->addMetric(METRIC_DOCUMENTS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value) // per collection
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection
break;
case $document->getCollection() === 'buckets': //buckets
$queueForUsage

3
app/init.php
View file

@ -216,10 +216,13 @@ const METRIC_MESSAGES_COUNTRY_CODE = '{countryCode}.messages';
const METRIC_SESSIONS = 'sessions';
const METRIC_DATABASES = 'databases';
const METRIC_COLLECTIONS = 'collections';
const METRIC_DATABASES_STORAGE = 'db_storage';
const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections';
const METRIC_DATABASE_ID_STORAGE = '{databaseInternalId}.db_storage';
const METRIC_DOCUMENTS = 'documents';
const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents';
const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents';
const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.db_storage';
const METRIC_BUCKETS = 'buckets';
const METRIC_FILES = 'files';
const METRIC_FILES_STORAGE = 'files.storage';

56
src/Appwrite/Platform/Workers/UsageDump.php
View file

@ -4,9 +4,11 @@ namespace Appwrite\Platform\Workers;
use Appwrite\Extend\Exception;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\System\System;
@ -70,6 +72,11 @@ class UsageDump extends Action
continue;
}
if (str_ends_with($key, '.db_storage')) {
$this->handleDBStorageCalculation($key, $dbForProject);
return;
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
@ -107,4 +114,53 @@ class UsageDump extends Action
}
}
}
private function handleDBStorageCalculation(string $key, Database $dbForProject): void
{
$data = explode('.', $key);
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
$value = 0;
$previousValue = 0;
try {
$previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value');
} catch (\Exception $e) {
// No previous value
}
switch (count($data)) {
// Collection Level
case 3:
$databaseInternalId = $data[0];
$collectionInternalId = $data[1];
$value = $dbForProject->getSizeOfCollection('database_'.$databaseInternalId.'_collection_'.$collectionInternalId);
// Compare with previous value
$value = $value - $previousValue;
// Update Collection
// Update Database
// Update Project
break;
// Database Level
case 2:
$databaseInternalId = $data[0];
$collections = $dbForProject->find('database_' . $databaseInternalId);
foreach ($collections as $collection) {
$value += $dbForProject->getSizeOfCollection($collection->getInternalId());
}
break;
// Project Level
case 1:
break;
}
}
}
}