From 800a76db729a8b85587537860d7366da69d59791 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 13:46:22 +0545 Subject: [PATCH] counter stats update --- app/controllers/api/projects.php | 37 ++++--------- app/tasks/usage.php | 91 ++++++++++++++++++++++++++------ 2 files changed, 84 insertions(+), 44 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f660bf2f6..bea5332f1 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -285,33 +285,16 @@ App::get('/v1/projects/:projectId/usage') } } - $usersCount = Authorization::skip(function () use ($dbForInternal) { - return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); - }); + $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - $collectionsCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['collections.count'])], 0, ['time'], [Database::ORDER_DESC]); - }); + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - // $documents = []; + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - // foreach ($collections as $collection) { - // $result = $projectDB->getCollection([ - // 'limit' => 0, - // 'offset' => 0, - // 'filters' => [ - // '$collection=' . $collection['$id'], - // ], - // ]); - - // $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; - // } - - $filesCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['files.count'])], 0, ['time'], [Database::ORDER_DESC]); - }); + $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; Authorization::reset(); @@ -344,12 +327,10 @@ App::get('/v1/projects/:projectId/usage') 'data' => [], 'total' => $filesTotal, ], - // 'documents' => [ - // 'data' => $documents, - // 'total' => \array_sum(\array_map(function ($item) { - // return $item['total']; - // }, $documents)), - // ], + 'documents' => [ + 'data' => [], + 'total' => $documentsTotal, + ], 'users' => [ 'data' => [], 'total' => $usersTotal, diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 216949f6d..ffa2df490 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -16,7 +16,7 @@ use Utopia\Database\Validator\Authorization; /** * Metrics We collect - * + * * requests * network * executions @@ -42,13 +42,15 @@ use Utopia\Database\Validator\Authorization; * users.delete * users.sessions.create * users.sessions.delete - * + * * Counters - * + * * users.count - * files.count - * collections.count - * + * storage.files.count + * database.collections.count + * database.documents.count + * database.collections.{collectionId}.documents.count + * */ $cli @@ -249,8 +251,10 @@ $cli } } - if ($iterations % 30 == 0) { - //aggregate number of objects in database + if ($iterations % 30 == 0) { //every 15 minutes + // aggregate number of objects in database + // get count of all the documents per collection - + // buckets will have the same $latestProject = null; do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); @@ -259,25 +263,80 @@ $cli foreach ($projects as $project) { $id = $project->getId(); - $collections = ['users' => [ - 'namespace' => 'internal', - ], 'collections' => [ - 'namespace' => 'external', - ], 'files' => [ - 'namespace' => 'internal', - ]]; + $collections = [ + 'users' => [ + 'namespace' => 'internal', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => 'external', //new change will make this internal + 'subCollections' => [ + 'documents' => [ + 'namespace' => 'external', + ], + ], + ], + 'files' => [ + 'metricPrefix' => 'storage', + 'namespace' => 'internal', + ], + ]; foreach ($collections as $collection => $options) { $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); $count = $dbForProject->count($collection); $dbForProject->setNamespace("project_{$id}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), 'time' => time(), 'period' => '15m', - 'metric' => "{$collection}.count", + 'metric' => $metric, 'value' => $count, 'type' => 1, ])); + + $subCollections = $options['subCollections'] ?? []; + if (!empty($subCollections)) { + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + do { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); + if (!empty($parents)) { + $latestParent = $parents[array_key_last($parents)]; + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { + $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; + + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } + } + } while (!empty($parents)); + + foreach ($subCollectionsCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } } } }