From ad6c3648206c9ddc55645dad3ce90e8954c546d7 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 27 Aug 2021 07:37:15 +0300 Subject: [PATCH 1/4] Added missing offset and types --- app/workers/deletes.php | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index e98723a5ea..1894aaf76b 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -18,8 +18,14 @@ Console::success(APP_NAME.' deletes worker v1 has started'."\n"); class DeletesV1 extends Worker { + /** + * @var array + */ public $args = []; + /** + * @var Database + */ protected $consoleDB = null; public function init(): void @@ -33,7 +39,7 @@ class DeletesV1 extends Worker switch (strval($type)) { case DELETE_TYPE_DOCUMENT: - $document = $this->args['document'] ?? ''; + $document = $this->args['document'] ?? []; $document = new Document($document); switch ($document->getCollection()) { @@ -87,7 +93,8 @@ class DeletesV1 extends Worker * @param Document $document teams document * @param string $projectId */ - protected function deleteMemberships(Document $document, $projectId) { + protected function deleteMemberships(Document $document, string $projectId): void + { $teamId = $document->getAttribute('teamId', ''); // Delete Memberships @@ -99,7 +106,7 @@ class DeletesV1 extends Worker /** * @param Document $document project document */ - protected function deleteProject(Document $document) + protected function deleteProject(Document $document): void { $projectId = $document->getId(); // Delete all DBs @@ -118,7 +125,7 @@ class DeletesV1 extends Worker * @param Document $document user document * @param string $projectId */ - protected function deleteUser(Document $document, $projectId) + protected function deleteUser(Document $document, string $projectId): void { $userId = $document->getId(); @@ -143,9 +150,9 @@ class DeletesV1 extends Worker /** * @param int $timestamp */ - protected function deleteExecutionLogs($timestamp) + protected function deleteExecutionLogs(int $timestamp): void { - $this->deleteForProjectIds(function($projectId) use ($timestamp) { + $this->deleteForProjectIds(function(string $projectId) use ($timestamp) { if (!($dbForInternal = $this->getInternalDB($projectId))) { throw new Exception('Failed to get projectDB for project '.$projectId); } @@ -160,7 +167,7 @@ class DeletesV1 extends Worker /** * @param int $timestamp */ - protected function deleteAbuseLogs($timestamp) + protected function deleteAbuseLogs(int $timestamp): void { if($timestamp == 0) { throw new Exception('Failed to delete audit logs. No timestamp provided'); @@ -180,7 +187,7 @@ class DeletesV1 extends Worker /** * @param int $timestamp */ - protected function deleteAuditLogs($timestamp) + protected function deleteAuditLogs(int $timestamp): void { if($timestamp == 0) { throw new Exception('Failed to delete audit logs. No timestamp provided'); @@ -198,7 +205,7 @@ class DeletesV1 extends Worker * @param Document $document function document * @param string $projectId */ - protected function deleteFunction(Document $document, $projectId) + protected function deleteFunction(Document $document, string $projectId): void { $dbForInternal = $this->getInternalDB($projectId); $device = new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId); @@ -255,7 +262,7 @@ class DeletesV1 extends Worker /** * @param callable $callback */ - protected function deleteForProjectIds(callable $callback) + protected function deleteForProjectIds(callable $callback): void { $count = 0; $chunk = 0; @@ -266,12 +273,12 @@ class DeletesV1 extends Worker $executionStart = \microtime(true); while($sum === $limit) { - $chunk++; - Authorization::disable(); - $projects = $this->getConsoleDB()->find('projects', [], $limit); + $projects = $this->getConsoleDB()->find('projects', [], $limit, ($chunk * $limit)); Authorization::reset(); + $chunk++; + $projectIds = array_map (function ($project) { return $project->getId(); }, $projects); @@ -295,7 +302,7 @@ class DeletesV1 extends Worker * @param Database $database * @param callable $callback */ - protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null) + protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void { $count = 0; $chunk = 0; @@ -331,9 +338,8 @@ class DeletesV1 extends Worker /** * @param Document $document certificates document - * @return Database */ - protected function deleteCertificates(Document $document) + protected function deleteCertificates(Document $document): void { $domain = $document->getAttribute('domain'); $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; From c84dc0fa92c774cd4d6e94d34721a53003edf47b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 30 Aug 2021 10:19:29 +0300 Subject: [PATCH 2/4] init --- app/tasks/usage.php | 222 ++++++++++++++++++++++++++------------------ 1 file changed, 130 insertions(+), 92 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index fdbf1d3eb2..2de5c09377 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -11,15 +11,19 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; /** * Metrics We collect * + * General + * * requests * network * executions + * + * Database + * * database.collections.create * database.collections.read * database.collections.update @@ -32,10 +36,16 @@ use Utopia\Database\Validator\Authorization; * database.collections.{collectionId}.documents.read * database.collections.{collectionId}.documents.update * database.collections.{collectionId}.documents.delete + * + * Storage + * * storage.buckets.{bucketId}.files.create * storage.buckets.{bucketId}.files.read * storage.buckets.{bucketId}.files.update * storage.buckets.{bucketId}.files.delete + * + * Users + * * users.create * users.read * users.update @@ -71,7 +81,7 @@ $cli Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) $periods = [ [ 'key' => '30m', @@ -189,16 +199,18 @@ $cli ], ]; + // TODO Maybe move this to the setResource method, and reuse in the http.php file $attempts = 0; $max = 10; $sleep = 1; + do { // connect to db try { $attempts++; $db = $register->get('db'); $redis = $register->get('cache'); break; // leave the do-while if successful - } catch (\Exception$e) { + } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); if ($attempts >= $max) { throw new \Exception('Failed to connect to database: ' . $e->getMessage()); @@ -207,6 +219,7 @@ $cli } } while ($attempts < $max); + // TODO use inject $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); @@ -223,9 +236,13 @@ $cli $loopStart = microtime(true); + /** + * Aggregate InfluxDB every 30 seconds + */ $client = $register->get('influxdb'); if ($client) { $database = $client->selectDB('telegraf'); + // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics foreach ($periods as $period) { // aggregate data for each period @@ -250,12 +267,11 @@ $cli $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; + if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); - if($metric == 'functions.functionId.executions') { - var_dump($points); - } $metricUpdated = $metric; + if (!empty($groupBy)) { $groupedBy = $point[$options['groupBy']] ?? ''; if (empty($groupedBy)) { @@ -263,9 +279,11 @@ $cli } $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); } + $time = \strtotime($point['time']); $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; + try { $document = $dbForProject->getDocument('stats', $id); if ($document->isEmpty()) { @@ -279,12 +297,11 @@ $cli ])); } else { $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); + $document->setAttribute('value', $value)); } $latestTime[$metric][$period['key']] = $time; - } catch (\Exception$e) { - // if projects are deleted this might fail - Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}"); + } catch (\Exception $e) { // if projects are deleted this might fail + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); } } } @@ -292,119 +309,140 @@ $cli } } - if ($iterations % 30 == 0) { //every 15 minutes - // aggregate number of objects in database - // get count of all the documents per collection - - // buckets will have the same + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + if ($iterations % 30 == 0) { // Every 15 minutes + // Aggregate number of objects in database + // Get count of all the documents per collection - + // Buckets will have the same + $latestProject = null; + do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; + + if (empty($projects)) { + continue; + } - foreach ($projects as $project) { - $id = $project->getId(); + $latestProject = $projects[array_key_last($projects)]; - // get total storage - $dbForProject->setNamespace('project_' . $id . '_internal'); - $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); + foreach ($projects as $project) { + $id = $project->getId(); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'period' => '15m', - 'time' => time(), - 'metric' => 'storage.total', - 'value' => $storageTotal, - 'type' => 1, - ])); + // Get total storage + $dbForProject->setNamespace('project_' . $id . '_internal'); + $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); - $collections = [ - 'users' => [ - 'namespace' => 'internal', - ], - 'collections' => [ - 'metricPrefix' => 'database', - 'namespace' => 'internal', - 'subCollections' => [ - 'documents' => [ - 'namespace' => 'external', - ], + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'period' => '15m', + 'time' => time(), + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + + $collections = [ + 'users' => [ + 'namespace' => 'internal', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => 'internal', + 'subCollections' => [ // TODO better document this key + 'documents' => [ + 'namespace' => 'external', ], ], - 'files' => [ - 'metricPrefix' => 'storage', - 'namespace' => 'internal', - ], - ]; - foreach ($collections as $collection => $options) { - try { + ], + 'files' => [ + 'metricPrefix' => 'storage', + 'namespace' => 'internal', + ], + ]; + + foreach ($collections as $collection => $options) { + try { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$id}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + + $subCollections = $options['subCollections'] ?? []; + + if (empty($subCollections)) { + continue; + } + + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + + do { $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $count = $dbForProject->count($collection); - $dbForProject->setNamespace("project_{$id}_internal"); - $metricPrefix = $options['metricPrefix'] ?? ''; - $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - $subCollections = $options['subCollections'] ?? []; - if (!empty($subCollections)) { - $latestParent = null; - $subCollectionCounts = []; //total project level count of sub collections - do { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - if (!empty($parents)) { - $latestParent = $parents[array_key_last($parents)]; - foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { - $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); - $count = $dbForProject->count($parent->getId()); - $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; + if (empty($parents)) { + continue; + } - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); - } - } - } - } while (!empty($parents)); + $latestParent = $parents[array_key_last($parents)]; + + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { + $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - foreach ($subCollectionsCounts as $subCollection => $count) { $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), 'time' => time(), 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", 'value' => $count, 'type' => 1, ])); } } - } catch (\Exception$e) { - Console::warning("Failed to save database counters data for project {$collection}"); + } while (!empty($parents)); + + foreach ($subCollectionsCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); } + } catch (\Exception $e) { + Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); } } } - } while (!empty($projects)); } + $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - }); + }); \ No newline at end of file From 2477b80978534a797d5a69118b47a0537c805856 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 30 Aug 2021 15:20:32 +0545 Subject: [PATCH 3/4] update counters also as time series data --- app/tasks/usage.php | 250 +++++++++++++++++++++++++++++++------------- 1 file changed, 179 insertions(+), 71 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 2de5c09377..3fa90a3f43 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -17,13 +17,13 @@ use Utopia\Database\Validator\Authorization; * Metrics We collect * * General - * + * * requests * network * executions - * + * * Database - * + * * database.collections.create * database.collections.read * database.collections.update @@ -36,16 +36,16 @@ use Utopia\Database\Validator\Authorization; * database.collections.{collectionId}.documents.read * database.collections.{collectionId}.documents.update * database.collections.{collectionId}.documents.delete - * + * * Storage - * + * * storage.buckets.{bucketId}.files.create * storage.buckets.{bucketId}.files.read * storage.buckets.{bucketId}.files.update * storage.buckets.{bucketId}.files.delete - * + * * Users - * + * * users.create * users.read * users.update @@ -203,7 +203,7 @@ $cli $attempts = 0; $max = 10; $sleep = 1; - + do { // connect to db try { $attempts++; @@ -219,7 +219,7 @@ $cli } } while ($attempts < $max); - // TODO use inject + // TODO use inject $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); @@ -242,7 +242,7 @@ $cli $client = $register->get('influxdb'); if ($client) { $database = $client->selectDB('telegraf'); - + // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics foreach ($periods as $period) { // aggregate data for each period @@ -252,10 +252,10 @@ $cli } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $table = $options['table']; //which influxdb table to query for this metric - $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //some sub level metrics may be grouped by other tags like collectionId, bucketId, etc + $table = $options['table']; //Which influxdb table to query for this metric + $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - $filters = $options['filters'] ?? []; + $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status if (!empty($filters)) { $filters = ' AND ' . implode(' AND ', array_map(function ($filter, $value) { return '"' . $filter . '"=\'' . $value . '\''; @@ -263,7 +263,7 @@ $cli } $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\'' . (empty($filters) ? '' : $filters) . ' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); - + $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; @@ -271,7 +271,7 @@ $cli if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); $metricUpdated = $metric; - + if (!empty($groupBy)) { $groupedBy = $point[$options['groupBy']] ?? ''; if (empty($groupedBy)) { @@ -281,9 +281,9 @@ $cli } $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric + $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; - + try { $document = $dbForProject->getDocument('stats', $id); if ($document->isEmpty()) { @@ -310,19 +310,19 @@ $cli } /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ if ($iterations % 30 == 0) { // Every 15 minutes // Aggregate number of objects in database // Get count of all the documents per collection - // Buckets will have the same $latestProject = null; - + do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - + if (empty($projects)) { continue; } @@ -330,20 +330,45 @@ $cli $latestProject = $projects[array_key_last($projects)]; foreach ($projects as $project) { - $id = $project->getId(); + $projectId = $project->getId(); // Get total storage - $dbForProject->setNamespace('project_' . $id . '_internal'); + $dbForProject->setNamespace('project_' . $projectId . '_internal'); $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'period' => '15m', - 'time' => time(), - 'metric' => 'storage.total', - 'value' => $storageTotal, - 'type' => 1, - ])); + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '1d', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } $collections = [ 'users' => [ @@ -352,7 +377,7 @@ $cli 'collections' => [ 'metricPrefix' => 'database', 'namespace' => 'internal', - 'subCollections' => [ // TODO better document this key + 'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting 'documents' => [ 'namespace' => 'external', ], @@ -366,20 +391,45 @@ $cli foreach ($collections as $collection => $options) { try { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); $count = $dbForProject->count($collection); - $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->setNamespace("project_{$projectId}_internal"); $metricPrefix = $options['metricPrefix'] ?? ''; $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; - - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } $subCollections = $options['subCollections'] ?? []; @@ -391,8 +441,8 @@ $cli $subCollectionCounts = []; //total project level count of sub collections do { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections if (empty($parents)) { continue; @@ -401,37 +451,95 @@ $cli $latestParent = $parents[array_key_last($parents)]; foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { - $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count + $dbForProject->setNamespace("project_{$projectId}_{$subOptions['namespace']}"); $count = $dbForProject->count($parent->getId()); - $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - $dbForProject->setNamespace("project_{$id}_internal"); + $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count"; + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } } } } while (!empty($parents)); - foreach ($subCollectionsCounts as $subCollection => $count) { - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); + /** + * Inserting project level counts for sub collections like database.documents.count + */ + foreach ($subCollectionCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count"; + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } } - } catch (\Exception $e) { + } catch (\Exception$e) { Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); } } @@ -442,7 +550,7 @@ $cli $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); - + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - }); \ No newline at end of file + }); From 87311797f2ece141467966599175cc3ef5313e58 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 30 Aug 2021 15:23:11 +0545 Subject: [PATCH 4/4] more comments --- app/tasks/usage.php | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 3fa90a3f43..e66bec113f 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -310,17 +310,14 @@ $cli } /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ - if ($iterations % 30 == 0) { // Every 15 minutes - // Aggregate number of objects in database - // Get count of all the documents per collection - - // Buckets will have the same + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + if ($iterations % 30 == 0) { // Every 15 minutes aggregate number of objects in database $latestProject = null; - do { + do { // Loop over all the projects $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); if (empty($projects)) { @@ -440,7 +437,7 @@ $cli $latestParent = null; $subCollectionCounts = []; //total project level count of sub collections - do { + do { // Loop over all the parent collection document for each sub collection $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections