From 656471ced2b6af92aa0fb980e077f23d49e70725 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 27 Oct 2021 15:57:20 -0400 Subject: [PATCH 1/4] Backfill usage stats with empty records when required --- app/controllers/api/database.php | 44 ++++++++++++++++++++++++++++--- app/controllers/api/functions.php | 22 ++++++++++++++-- app/controllers/api/projects.php | 22 ++++++++++++++-- app/controllers/api/storage.php | 44 ++++++++++++++++++++++++++++--- app/controllers/api/users.php | 22 ++++++++++++++-- 5 files changed, 140 insertions(+), 14 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 65af782193..1c1749a043 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -323,10 +323,13 @@ App::get('/v1/database/usage') Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { foreach ($metrics as $metric) { + $limit = $period[$range]['limit']; + $period = $period[$range]['period']; + $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('period', Query::TYPE_EQUAL, [$period]), new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + ], $limit, 0, ['time'], [Database::ORDER_DESC]); $stats[$metric] = []; foreach ($requestDocs as $requestDoc) { @@ -335,6 +338,21 @@ App::get('/v1/database/usage') 'date' => $requestDoc->getAttribute('time'), ]; } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match($period) { // convert period to seconds for unix timestamp math + '30m' => 1800, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + ]; + $backfill--; + } $stats[$metric] = array_reverse($stats[$metric]); } }); @@ -417,10 +435,13 @@ App::get('/v1/database/:collectionId/usage') Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { foreach ($metrics as $metric) { + $limit = $period[$range]['limit']; + $period = $period[$range]['period']; + $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('period', Query::TYPE_EQUAL, [$period]), new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + ], $limit, 0, ['time'], [Database::ORDER_DESC]); $stats[$metric] = []; foreach ($requestDocs as $requestDoc) { @@ -429,6 +450,21 @@ App::get('/v1/database/:collectionId/usage') 'date' => $requestDoc->getAttribute('time'), ]; } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match($period) { // convert period to seconds for unix timestamp math + '30m' => 1800, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + ]; + $backfill--; + } $stats[$metric] = array_reverse($stats[$metric]); } }); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 4c6749d5d3..679f4c8b1e 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -205,10 +205,13 @@ App::get('/v1/functions/:functionId/usage') Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { foreach ($metrics as $metric) { + $limit = $period[$range]['limit']; + $period = $period[$range]['period']; + $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('period', Query::TYPE_EQUAL, [$period]), new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + ], $limit, 0, ['time'], [Database::ORDER_DESC]); $stats[$metric] = []; foreach ($requestDocs as $requestDoc) { @@ -217,6 +220,21 @@ App::get('/v1/functions/:functionId/usage') 'date' => $requestDoc->getAttribute('time'), ]; } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match($period) { // convert period to seconds for unix timestamp math + '30m' => 1800, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + ]; + $backfill--; + } $stats[$metric] = array_reverse($stats[$metric]); } }); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index c4d8c1053a..69b44ddc93 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -290,10 +290,13 @@ App::get('/v1/projects/:projectId/usage') Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { foreach ($metrics as $metric) { + $limit = $period[$range]['limit']; + $period = $period[$range]['period']; + $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('period', Query::TYPE_EQUAL, [$period]), new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + ], $limit, 0, ['time'], [Database::ORDER_DESC]); $stats[$metric] = []; foreach ($requestDocs as $requestDoc) { @@ -302,6 +305,21 @@ App::get('/v1/projects/:projectId/usage') 'date' => $requestDoc->getAttribute('time'), ]; } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match($period) { // convert period to seconds for unix timestamp math + '30m' => 1800, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + ]; + $backfill--; + } $stats[$metric] = array_reverse($stats[$metric]); } }); diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index e209e500a1..d8abf494d3 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -698,10 +698,13 @@ App::get('/v1/storage/usage') Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { foreach ($metrics as $metric) { + $limit = $period[$range]['limit']; + $period = $period[$range]['period']; + $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('period', Query::TYPE_EQUAL, [$period]), new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + ], $limit, 0, ['time'], [Database::ORDER_DESC]); $stats[$metric] = []; foreach ($requestDocs as $requestDoc) { @@ -710,6 +713,21 @@ App::get('/v1/storage/usage') 'date' => $requestDoc->getAttribute('time'), ]; } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match($period) { // convert period to seconds for unix timestamp math + '30m' => 1800, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + ]; + $backfill--; + } $stats[$metric] = array_reverse($stats[$metric]); } }); @@ -777,10 +795,13 @@ App::get('/v1/storage/:bucketId/usage') Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { foreach ($metrics as $metric) { + $limit = $period[$range]['limit']; + $period = $period[$range]['period']; + $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('period', Query::TYPE_EQUAL, [$period]), new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + ], $limit, 0, ['time'], [Database::ORDER_DESC]); $stats[$metric] = []; foreach ($requestDocs as $requestDoc) { @@ -789,6 +810,21 @@ App::get('/v1/storage/:bucketId/usage') 'date' => $requestDoc->getAttribute('time'), ]; } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match($period) { // convert period to seconds for unix timestamp math + '30m' => 1800, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + ]; + $backfill--; + } $stats[$metric] = array_reverse($stats[$metric]); } }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 050ef34e89..dec653dd4c 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -814,10 +814,13 @@ App::get('/v1/users/usage') Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { foreach ($metrics as $metric) { + $limit = $period[$range]['limit']; + $period = $period[$range]['period']; + $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('period', Query::TYPE_EQUAL, [$period]), new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + ], $limit, 0, ['time'], [Database::ORDER_DESC]); $stats[$metric] = []; foreach ($requestDocs as $requestDoc) { @@ -826,6 +829,21 @@ App::get('/v1/users/usage') 'date' => $requestDoc->getAttribute('time'), ]; } + + // backfill metrics with empty values for graphs + $backfill = $limit - \count($requestDocs); + while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric + $diff = match($period) { // convert period to seconds for unix timestamp math + '30m' => 1800, + '1d' => 86400, + }; + $stats[$metric][] = [ + 'value' => 0, + 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + ]; + $backfill--; + } $stats[$metric] = array_reverse($stats[$metric]); } }); From 79b1fceaf3f3e0d11071ce998b3727bfcd7d46e9 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 27 Oct 2021 18:17:15 -0400 Subject: [PATCH 2/4] Rename to prevent collisions --- app/controllers/api/database.php | 16 ++++++++-------- app/controllers/api/functions.php | 8 ++++---- app/controllers/api/projects.php | 8 ++++---- app/controllers/api/storage.php | 16 ++++++++-------- app/controllers/api/users.php | 8 ++++---- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 1c1749a043..1c84c4d746 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -287,7 +287,7 @@ App::get('/v1/database/usage') $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ + $periods = [ '24h' => [ 'period' => '30m', 'limit' => 48, @@ -321,10 +321,10 @@ App::get('/v1/database/usage') $stats = []; - Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) { foreach ($metrics as $metric) { - $limit = $period[$range]['limit']; - $period = $period[$range]['period']; + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period]), @@ -404,7 +404,7 @@ App::get('/v1/database/:collectionId/usage') $usage = []; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ + $periods = [ '24h' => [ 'period' => '30m', 'limit' => 48, @@ -433,10 +433,10 @@ App::get('/v1/database/:collectionId/usage') $stats = []; - Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) { foreach ($metrics as $metric) { - $limit = $period[$range]['limit']; - $period = $period[$range]['period']; + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period]), diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 679f4c8b1e..5857c3ccc7 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -176,7 +176,7 @@ App::get('/v1/functions/:functionId/usage') $usage = []; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ + $periods = [ '24h' => [ 'period' => '30m', 'limit' => 48, @@ -203,10 +203,10 @@ App::get('/v1/functions/:functionId/usage') $stats = []; - Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) { foreach ($metrics as $metric) { - $limit = $period[$range]['limit']; - $period = $period[$range]['period']; + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period]), diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 69b44ddc93..853e22ffd8 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -255,7 +255,7 @@ App::get('/v1/projects/:projectId/usage') $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ + $periods = [ '24h' => [ 'period' => '30m', 'limit' => 48, @@ -288,10 +288,10 @@ App::get('/v1/projects/:projectId/usage') $stats = []; - Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) { foreach ($metrics as $metric) { - $limit = $period[$range]['limit']; - $period = $period[$range]['period']; + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period]), diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index d8abf494d3..b5151353e8 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -670,7 +670,7 @@ App::get('/v1/storage/usage') $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ + $periods = [ '24h' => [ 'period' => '30m', 'limit' => 48, @@ -696,10 +696,10 @@ App::get('/v1/storage/usage') $stats = []; - Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) { foreach ($metrics as $metric) { - $limit = $period[$range]['limit']; - $period = $period[$range]['period']; + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period]), @@ -764,7 +764,7 @@ App::get('/v1/storage/:bucketId/usage') $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ + $periods = [ '24h' => [ 'period' => '30m', 'limit' => 48, @@ -793,10 +793,10 @@ App::get('/v1/storage/:bucketId/usage') $stats = []; - Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) { foreach ($metrics as $metric) { - $limit = $period[$range]['limit']; - $period = $period[$range]['period']; + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period]), diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index dec653dd4c..6ea6bca091 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -780,7 +780,7 @@ App::get('/v1/users/usage') $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ + $periods = [ '24h' => [ 'period' => '30m', 'limit' => 48, @@ -812,10 +812,10 @@ App::get('/v1/users/usage') $stats = []; - Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) { foreach ($metrics as $metric) { - $limit = $period[$range]['limit']; - $period = $period[$range]['period']; + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period]), From 6e46e2fb0f49b15f695cee23800d90e545b1695c Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 27 Oct 2021 18:17:51 -0400 Subject: [PATCH 3/4] Use correct key in metrics array --- app/controllers/api/database.php | 4 ++-- app/controllers/api/functions.php | 2 +- app/controllers/api/projects.php | 2 +- app/controllers/api/storage.php | 4 ++-- app/controllers/api/users.php | 3 ++- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 1c84c4d746..6d696cf233 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -349,7 +349,7 @@ App::get('/v1/database/usage') }; $stats[$metric][] = [ 'value' => 0, - 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + 'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period ]; $backfill--; } @@ -461,7 +461,7 @@ App::get('/v1/database/:collectionId/usage') }; $stats[$metric][] = [ 'value' => 0, - 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + 'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period ]; $backfill--; } diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 5857c3ccc7..c44294599a 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -231,7 +231,7 @@ App::get('/v1/functions/:functionId/usage') }; $stats[$metric][] = [ 'value' => 0, - 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + 'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period ]; $backfill--; } diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 853e22ffd8..66ac25d8a9 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -316,7 +316,7 @@ App::get('/v1/projects/:projectId/usage') }; $stats[$metric][] = [ 'value' => 0, - 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + 'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period ]; $backfill--; } diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index b5151353e8..7ab1bc8153 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -724,7 +724,7 @@ App::get('/v1/storage/usage') }; $stats[$metric][] = [ 'value' => 0, - 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + 'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period ]; $backfill--; } @@ -821,7 +821,7 @@ App::get('/v1/storage/:bucketId/usage') }; $stats[$metric][] = [ 'value' => 0, - 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + 'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period ]; $backfill--; } diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 6ea6bca091..2df8a96105 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -833,6 +833,7 @@ App::get('/v1/users/usage') // backfill metrics with empty values for graphs $backfill = $limit - \count($requestDocs); while ($backfill > 0) { + $last = $limit - $backfill - 1; // array index of last added metric $diff = match($period) { // convert period to seconds for unix timestamp math '30m' => 1800, @@ -840,7 +841,7 @@ App::get('/v1/users/usage') }; $stats[$metric][] = [ 'value' => 0, - 'date' => $stats[$metric][$last]['time'] - $diff, // time of last metric minus period + 'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period ]; $backfill--; } From 92e48e273d3923b65d278e0d11bf9fc0b26fb7b3 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 27 Oct 2021 18:31:25 -0400 Subject: [PATCH 4/4] Add reminder to check query order --- app/controllers/api/database.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 6d696cf233..e0126248ea 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -353,6 +353,7 @@ App::get('/v1/database/usage') ]; $backfill--; } + // TODO@kodumbeats explore performance if query is ordered by time ASC $stats[$metric] = array_reverse($stats[$metric]); } });