From c84dc0fa92c774cd4d6e94d34721a53003edf47b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 30 Aug 2021 10:19:29 +0300 Subject: [PATCH] init --- app/tasks/usage.php | 222 ++++++++++++++++++++++++++------------------ 1 file changed, 130 insertions(+), 92 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index fdbf1d3eb..2de5c0937 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -11,15 +11,19 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; /** * Metrics We collect * + * General + * * requests * network * executions + * + * Database + * * database.collections.create * database.collections.read * database.collections.update @@ -32,10 +36,16 @@ use Utopia\Database\Validator\Authorization; * database.collections.{collectionId}.documents.read * database.collections.{collectionId}.documents.update * database.collections.{collectionId}.documents.delete + * + * Storage + * * storage.buckets.{bucketId}.files.create * storage.buckets.{bucketId}.files.read * storage.buckets.{bucketId}.files.update * storage.buckets.{bucketId}.files.delete + * + * Users + * * users.create * users.read * users.update @@ -71,7 +81,7 @@ $cli Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) $periods = [ [ 'key' => '30m', @@ -189,16 +199,18 @@ $cli ], ]; + // TODO Maybe move this to the setResource method, and reuse in the http.php file $attempts = 0; $max = 10; $sleep = 1; + do { // connect to db try { $attempts++; $db = $register->get('db'); $redis = $register->get('cache'); break; // leave the do-while if successful - } catch (\Exception$e) { + } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); if ($attempts >= $max) { throw new \Exception('Failed to connect to database: ' . $e->getMessage()); @@ -207,6 +219,7 @@ $cli } } while ($attempts < $max); + // TODO use inject $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); @@ -223,9 +236,13 @@ $cli $loopStart = microtime(true); + /** + * Aggregate InfluxDB every 30 seconds + */ $client = $register->get('influxdb'); if ($client) { $database = $client->selectDB('telegraf'); + // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics foreach ($periods as $period) { // aggregate data for each period @@ -250,12 +267,11 @@ $cli $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; + if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); - if($metric == 'functions.functionId.executions') { - var_dump($points); - } $metricUpdated = $metric; + if (!empty($groupBy)) { $groupedBy = $point[$options['groupBy']] ?? ''; if (empty($groupedBy)) { @@ -263,9 +279,11 @@ $cli } $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); } + $time = \strtotime($point['time']); $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; + try { $document = $dbForProject->getDocument('stats', $id); if ($document->isEmpty()) { @@ -279,12 +297,11 @@ $cli ])); } else { $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); + $document->setAttribute('value', $value)); } $latestTime[$metric][$period['key']] = $time; - } catch (\Exception$e) { - // if projects are deleted this might fail - Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}"); + } catch (\Exception $e) { // if projects are deleted this might fail + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); } } } @@ -292,119 +309,140 @@ $cli } } - if ($iterations % 30 == 0) { //every 15 minutes - // aggregate number of objects in database - // get count of all the documents per collection - - // buckets will have the same + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + if ($iterations % 30 == 0) { // Every 15 minutes + // Aggregate number of objects in database + // Get count of all the documents per collection - + // Buckets will have the same + $latestProject = null; + do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; + + if (empty($projects)) { + continue; + } - foreach ($projects as $project) { - $id = $project->getId(); + $latestProject = $projects[array_key_last($projects)]; - // get total storage - $dbForProject->setNamespace('project_' . $id . '_internal'); - $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); + foreach ($projects as $project) { + $id = $project->getId(); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'period' => '15m', - 'time' => time(), - 'metric' => 'storage.total', - 'value' => $storageTotal, - 'type' => 1, - ])); + // Get total storage + $dbForProject->setNamespace('project_' . $id . '_internal'); + $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); - $collections = [ - 'users' => [ - 'namespace' => 'internal', - ], - 'collections' => [ - 'metricPrefix' => 'database', - 'namespace' => 'internal', - 'subCollections' => [ - 'documents' => [ - 'namespace' => 'external', - ], + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'period' => '15m', + 'time' => time(), + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + + $collections = [ + 'users' => [ + 'namespace' => 'internal', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => 'internal', + 'subCollections' => [ // TODO better document this key + 'documents' => [ + 'namespace' => 'external', ], ], - 'files' => [ - 'metricPrefix' => 'storage', - 'namespace' => 'internal', - ], - ]; - foreach ($collections as $collection => $options) { - try { + ], + 'files' => [ + 'metricPrefix' => 'storage', + 'namespace' => 'internal', + ], + ]; + + foreach ($collections as $collection => $options) { + try { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$id}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + + $subCollections = $options['subCollections'] ?? []; + + if (empty($subCollections)) { + continue; + } + + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + + do { $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $count = $dbForProject->count($collection); - $dbForProject->setNamespace("project_{$id}_internal"); - $metricPrefix = $options['metricPrefix'] ?? ''; - $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - $subCollections = $options['subCollections'] ?? []; - if (!empty($subCollections)) { - $latestParent = null; - $subCollectionCounts = []; //total project level count of sub collections - do { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - if (!empty($parents)) { - $latestParent = $parents[array_key_last($parents)]; - foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { - $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); - $count = $dbForProject->count($parent->getId()); - $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; + if (empty($parents)) { + continue; + } - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); - } - } - } - } while (!empty($parents)); + $latestParent = $parents[array_key_last($parents)]; + + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { + $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - foreach ($subCollectionsCounts as $subCollection => $count) { $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), 'time' => time(), 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", 'value' => $count, 'type' => 1, ])); } } - } catch (\Exception$e) { - Console::warning("Failed to save database counters data for project {$collection}"); + } while (!empty($parents)); + + foreach ($subCollectionsCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); } + } catch (\Exception $e) { + Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); } } } - } while (!empty($projects)); } + $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - }); + }); \ No newline at end of file