Merge pull request #2360 from appwrite/feat-db-refactor-backfill-usage-stats
Backfill usage stats with empty records when required
This commit is contained in:
commit
0dd32a0a0c
5 changed files with 156 additions and 28 deletions
|
@ -287,7 +287,7 @@ App::get('/v1/database/usage')
|
||||||
|
|
||||||
$usage = [];
|
$usage = [];
|
||||||
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
||||||
$period = [
|
$periods = [
|
||||||
'24h' => [
|
'24h' => [
|
||||||
'period' => '30m',
|
'period' => '30m',
|
||||||
'limit' => 48,
|
'limit' => 48,
|
||||||
|
@ -321,12 +321,15 @@ App::get('/v1/database/usage')
|
||||||
|
|
||||||
$stats = [];
|
$stats = [];
|
||||||
|
|
||||||
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
|
Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) {
|
||||||
foreach ($metrics as $metric) {
|
foreach ($metrics as $metric) {
|
||||||
|
$limit = $periods[$range]['limit'];
|
||||||
|
$period = $periods[$range]['period'];
|
||||||
|
|
||||||
$requestDocs = $dbForInternal->find('stats', [
|
$requestDocs = $dbForInternal->find('stats', [
|
||||||
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
|
new Query('period', Query::TYPE_EQUAL, [$period]),
|
||||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||||
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
|
], $limit, 0, ['time'], [Database::ORDER_DESC]);
|
||||||
|
|
||||||
$stats[$metric] = [];
|
$stats[$metric] = [];
|
||||||
foreach ($requestDocs as $requestDoc) {
|
foreach ($requestDocs as $requestDoc) {
|
||||||
|
@ -335,6 +338,22 @@ App::get('/v1/database/usage')
|
||||||
'date' => $requestDoc->getAttribute('time'),
|
'date' => $requestDoc->getAttribute('time'),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backfill metrics with empty values for graphs
|
||||||
|
$backfill = $limit - \count($requestDocs);
|
||||||
|
while ($backfill > 0) {
|
||||||
|
$last = $limit - $backfill - 1; // array index of last added metric
|
||||||
|
$diff = match($period) { // convert period to seconds for unix timestamp math
|
||||||
|
'30m' => 1800,
|
||||||
|
'1d' => 86400,
|
||||||
|
};
|
||||||
|
$stats[$metric][] = [
|
||||||
|
'value' => 0,
|
||||||
|
'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period
|
||||||
|
];
|
||||||
|
$backfill--;
|
||||||
|
}
|
||||||
|
// TODO@kodumbeats explore performance if query is ordered by time ASC
|
||||||
$stats[$metric] = array_reverse($stats[$metric]);
|
$stats[$metric] = array_reverse($stats[$metric]);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -386,7 +405,7 @@ App::get('/v1/database/:collectionId/usage')
|
||||||
|
|
||||||
$usage = [];
|
$usage = [];
|
||||||
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
||||||
$period = [
|
$periods = [
|
||||||
'24h' => [
|
'24h' => [
|
||||||
'period' => '30m',
|
'period' => '30m',
|
||||||
'limit' => 48,
|
'limit' => 48,
|
||||||
|
@ -415,12 +434,15 @@ App::get('/v1/database/:collectionId/usage')
|
||||||
|
|
||||||
$stats = [];
|
$stats = [];
|
||||||
|
|
||||||
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
|
Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) {
|
||||||
foreach ($metrics as $metric) {
|
foreach ($metrics as $metric) {
|
||||||
|
$limit = $periods[$range]['limit'];
|
||||||
|
$period = $periods[$range]['period'];
|
||||||
|
|
||||||
$requestDocs = $dbForInternal->find('stats', [
|
$requestDocs = $dbForInternal->find('stats', [
|
||||||
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
|
new Query('period', Query::TYPE_EQUAL, [$period]),
|
||||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||||
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
|
], $limit, 0, ['time'], [Database::ORDER_DESC]);
|
||||||
|
|
||||||
$stats[$metric] = [];
|
$stats[$metric] = [];
|
||||||
foreach ($requestDocs as $requestDoc) {
|
foreach ($requestDocs as $requestDoc) {
|
||||||
|
@ -429,6 +451,21 @@ App::get('/v1/database/:collectionId/usage')
|
||||||
'date' => $requestDoc->getAttribute('time'),
|
'date' => $requestDoc->getAttribute('time'),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backfill metrics with empty values for graphs
|
||||||
|
$backfill = $limit - \count($requestDocs);
|
||||||
|
while ($backfill > 0) {
|
||||||
|
$last = $limit - $backfill - 1; // array index of last added metric
|
||||||
|
$diff = match($period) { // convert period to seconds for unix timestamp math
|
||||||
|
'30m' => 1800,
|
||||||
|
'1d' => 86400,
|
||||||
|
};
|
||||||
|
$stats[$metric][] = [
|
||||||
|
'value' => 0,
|
||||||
|
'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period
|
||||||
|
];
|
||||||
|
$backfill--;
|
||||||
|
}
|
||||||
$stats[$metric] = array_reverse($stats[$metric]);
|
$stats[$metric] = array_reverse($stats[$metric]);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
|
@ -176,7 +176,7 @@ App::get('/v1/functions/:functionId/usage')
|
||||||
|
|
||||||
$usage = [];
|
$usage = [];
|
||||||
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
||||||
$period = [
|
$periods = [
|
||||||
'24h' => [
|
'24h' => [
|
||||||
'period' => '30m',
|
'period' => '30m',
|
||||||
'limit' => 48,
|
'limit' => 48,
|
||||||
|
@ -203,12 +203,15 @@ App::get('/v1/functions/:functionId/usage')
|
||||||
|
|
||||||
$stats = [];
|
$stats = [];
|
||||||
|
|
||||||
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
|
Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) {
|
||||||
foreach ($metrics as $metric) {
|
foreach ($metrics as $metric) {
|
||||||
|
$limit = $periods[$range]['limit'];
|
||||||
|
$period = $periods[$range]['period'];
|
||||||
|
|
||||||
$requestDocs = $dbForInternal->find('stats', [
|
$requestDocs = $dbForInternal->find('stats', [
|
||||||
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
|
new Query('period', Query::TYPE_EQUAL, [$period]),
|
||||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||||
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
|
], $limit, 0, ['time'], [Database::ORDER_DESC]);
|
||||||
|
|
||||||
$stats[$metric] = [];
|
$stats[$metric] = [];
|
||||||
foreach ($requestDocs as $requestDoc) {
|
foreach ($requestDocs as $requestDoc) {
|
||||||
|
@ -217,6 +220,21 @@ App::get('/v1/functions/:functionId/usage')
|
||||||
'date' => $requestDoc->getAttribute('time'),
|
'date' => $requestDoc->getAttribute('time'),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backfill metrics with empty values for graphs
|
||||||
|
$backfill = $limit - \count($requestDocs);
|
||||||
|
while ($backfill > 0) {
|
||||||
|
$last = $limit - $backfill - 1; // array index of last added metric
|
||||||
|
$diff = match($period) { // convert period to seconds for unix timestamp math
|
||||||
|
'30m' => 1800,
|
||||||
|
'1d' => 86400,
|
||||||
|
};
|
||||||
|
$stats[$metric][] = [
|
||||||
|
'value' => 0,
|
||||||
|
'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period
|
||||||
|
];
|
||||||
|
$backfill--;
|
||||||
|
}
|
||||||
$stats[$metric] = array_reverse($stats[$metric]);
|
$stats[$metric] = array_reverse($stats[$metric]);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
|
@ -255,7 +255,7 @@ App::get('/v1/projects/:projectId/usage')
|
||||||
|
|
||||||
$usage = [];
|
$usage = [];
|
||||||
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
||||||
$period = [
|
$periods = [
|
||||||
'24h' => [
|
'24h' => [
|
||||||
'period' => '30m',
|
'period' => '30m',
|
||||||
'limit' => 48,
|
'limit' => 48,
|
||||||
|
@ -288,12 +288,15 @@ App::get('/v1/projects/:projectId/usage')
|
||||||
|
|
||||||
$stats = [];
|
$stats = [];
|
||||||
|
|
||||||
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
|
Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) {
|
||||||
foreach ($metrics as $metric) {
|
foreach ($metrics as $metric) {
|
||||||
|
$limit = $periods[$range]['limit'];
|
||||||
|
$period = $periods[$range]['period'];
|
||||||
|
|
||||||
$requestDocs = $dbForInternal->find('stats', [
|
$requestDocs = $dbForInternal->find('stats', [
|
||||||
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
|
new Query('period', Query::TYPE_EQUAL, [$period]),
|
||||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||||
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
|
], $limit, 0, ['time'], [Database::ORDER_DESC]);
|
||||||
|
|
||||||
$stats[$metric] = [];
|
$stats[$metric] = [];
|
||||||
foreach ($requestDocs as $requestDoc) {
|
foreach ($requestDocs as $requestDoc) {
|
||||||
|
@ -302,6 +305,21 @@ App::get('/v1/projects/:projectId/usage')
|
||||||
'date' => $requestDoc->getAttribute('time'),
|
'date' => $requestDoc->getAttribute('time'),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backfill metrics with empty values for graphs
|
||||||
|
$backfill = $limit - \count($requestDocs);
|
||||||
|
while ($backfill > 0) {
|
||||||
|
$last = $limit - $backfill - 1; // array index of last added metric
|
||||||
|
$diff = match($period) { // convert period to seconds for unix timestamp math
|
||||||
|
'30m' => 1800,
|
||||||
|
'1d' => 86400,
|
||||||
|
};
|
||||||
|
$stats[$metric][] = [
|
||||||
|
'value' => 0,
|
||||||
|
'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period
|
||||||
|
];
|
||||||
|
$backfill--;
|
||||||
|
}
|
||||||
$stats[$metric] = array_reverse($stats[$metric]);
|
$stats[$metric] = array_reverse($stats[$metric]);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
|
@ -670,7 +670,7 @@ App::get('/v1/storage/usage')
|
||||||
|
|
||||||
$usage = [];
|
$usage = [];
|
||||||
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
||||||
$period = [
|
$periods = [
|
||||||
'24h' => [
|
'24h' => [
|
||||||
'period' => '30m',
|
'period' => '30m',
|
||||||
'limit' => 48,
|
'limit' => 48,
|
||||||
|
@ -696,12 +696,15 @@ App::get('/v1/storage/usage')
|
||||||
|
|
||||||
$stats = [];
|
$stats = [];
|
||||||
|
|
||||||
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
|
Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) {
|
||||||
foreach ($metrics as $metric) {
|
foreach ($metrics as $metric) {
|
||||||
|
$limit = $periods[$range]['limit'];
|
||||||
|
$period = $periods[$range]['period'];
|
||||||
|
|
||||||
$requestDocs = $dbForInternal->find('stats', [
|
$requestDocs = $dbForInternal->find('stats', [
|
||||||
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
|
new Query('period', Query::TYPE_EQUAL, [$period]),
|
||||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||||
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
|
], $limit, 0, ['time'], [Database::ORDER_DESC]);
|
||||||
|
|
||||||
$stats[$metric] = [];
|
$stats[$metric] = [];
|
||||||
foreach ($requestDocs as $requestDoc) {
|
foreach ($requestDocs as $requestDoc) {
|
||||||
|
@ -710,6 +713,21 @@ App::get('/v1/storage/usage')
|
||||||
'date' => $requestDoc->getAttribute('time'),
|
'date' => $requestDoc->getAttribute('time'),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backfill metrics with empty values for graphs
|
||||||
|
$backfill = $limit - \count($requestDocs);
|
||||||
|
while ($backfill > 0) {
|
||||||
|
$last = $limit - $backfill - 1; // array index of last added metric
|
||||||
|
$diff = match($period) { // convert period to seconds for unix timestamp math
|
||||||
|
'30m' => 1800,
|
||||||
|
'1d' => 86400,
|
||||||
|
};
|
||||||
|
$stats[$metric][] = [
|
||||||
|
'value' => 0,
|
||||||
|
'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period
|
||||||
|
];
|
||||||
|
$backfill--;
|
||||||
|
}
|
||||||
$stats[$metric] = array_reverse($stats[$metric]);
|
$stats[$metric] = array_reverse($stats[$metric]);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -746,7 +764,7 @@ App::get('/v1/storage/:bucketId/usage')
|
||||||
|
|
||||||
$usage = [];
|
$usage = [];
|
||||||
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
||||||
$period = [
|
$periods = [
|
||||||
'24h' => [
|
'24h' => [
|
||||||
'period' => '30m',
|
'period' => '30m',
|
||||||
'limit' => 48,
|
'limit' => 48,
|
||||||
|
@ -775,12 +793,15 @@ App::get('/v1/storage/:bucketId/usage')
|
||||||
|
|
||||||
$stats = [];
|
$stats = [];
|
||||||
|
|
||||||
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
|
Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) {
|
||||||
foreach ($metrics as $metric) {
|
foreach ($metrics as $metric) {
|
||||||
|
$limit = $periods[$range]['limit'];
|
||||||
|
$period = $periods[$range]['period'];
|
||||||
|
|
||||||
$requestDocs = $dbForInternal->find('stats', [
|
$requestDocs = $dbForInternal->find('stats', [
|
||||||
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
|
new Query('period', Query::TYPE_EQUAL, [$period]),
|
||||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||||
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
|
], $limit, 0, ['time'], [Database::ORDER_DESC]);
|
||||||
|
|
||||||
$stats[$metric] = [];
|
$stats[$metric] = [];
|
||||||
foreach ($requestDocs as $requestDoc) {
|
foreach ($requestDocs as $requestDoc) {
|
||||||
|
@ -789,6 +810,21 @@ App::get('/v1/storage/:bucketId/usage')
|
||||||
'date' => $requestDoc->getAttribute('time'),
|
'date' => $requestDoc->getAttribute('time'),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backfill metrics with empty values for graphs
|
||||||
|
$backfill = $limit - \count($requestDocs);
|
||||||
|
while ($backfill > 0) {
|
||||||
|
$last = $limit - $backfill - 1; // array index of last added metric
|
||||||
|
$diff = match($period) { // convert period to seconds for unix timestamp math
|
||||||
|
'30m' => 1800,
|
||||||
|
'1d' => 86400,
|
||||||
|
};
|
||||||
|
$stats[$metric][] = [
|
||||||
|
'value' => 0,
|
||||||
|
'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period
|
||||||
|
];
|
||||||
|
$backfill--;
|
||||||
|
}
|
||||||
$stats[$metric] = array_reverse($stats[$metric]);
|
$stats[$metric] = array_reverse($stats[$metric]);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
|
@ -780,7 +780,7 @@ App::get('/v1/users/usage')
|
||||||
|
|
||||||
$usage = [];
|
$usage = [];
|
||||||
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
|
||||||
$period = [
|
$periods = [
|
||||||
'24h' => [
|
'24h' => [
|
||||||
'period' => '30m',
|
'period' => '30m',
|
||||||
'limit' => 48,
|
'limit' => 48,
|
||||||
|
@ -812,12 +812,15 @@ App::get('/v1/users/usage')
|
||||||
|
|
||||||
$stats = [];
|
$stats = [];
|
||||||
|
|
||||||
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
|
Authorization::skip(function() use ($dbForInternal, $periods, $range, $metrics, &$stats) {
|
||||||
foreach ($metrics as $metric) {
|
foreach ($metrics as $metric) {
|
||||||
|
$limit = $periods[$range]['limit'];
|
||||||
|
$period = $periods[$range]['period'];
|
||||||
|
|
||||||
$requestDocs = $dbForInternal->find('stats', [
|
$requestDocs = $dbForInternal->find('stats', [
|
||||||
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
|
new Query('period', Query::TYPE_EQUAL, [$period]),
|
||||||
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
new Query('metric', Query::TYPE_EQUAL, [$metric]),
|
||||||
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
|
], $limit, 0, ['time'], [Database::ORDER_DESC]);
|
||||||
|
|
||||||
$stats[$metric] = [];
|
$stats[$metric] = [];
|
||||||
foreach ($requestDocs as $requestDoc) {
|
foreach ($requestDocs as $requestDoc) {
|
||||||
|
@ -826,6 +829,22 @@ App::get('/v1/users/usage')
|
||||||
'date' => $requestDoc->getAttribute('time'),
|
'date' => $requestDoc->getAttribute('time'),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backfill metrics with empty values for graphs
|
||||||
|
$backfill = $limit - \count($requestDocs);
|
||||||
|
while ($backfill > 0) {
|
||||||
|
|
||||||
|
$last = $limit - $backfill - 1; // array index of last added metric
|
||||||
|
$diff = match($period) { // convert period to seconds for unix timestamp math
|
||||||
|
'30m' => 1800,
|
||||||
|
'1d' => 86400,
|
||||||
|
};
|
||||||
|
$stats[$metric][] = [
|
||||||
|
'value' => 0,
|
||||||
|
'date' => ($stats[$metric][$last]['date'] ?? \time()) - $diff, // time of last metric minus period
|
||||||
|
];
|
||||||
|
$backfill--;
|
||||||
|
}
|
||||||
$stats[$metric] = array_reverse($stats[$metric]);
|
$stats[$metric] = array_reverse($stats[$metric]);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
Loading…
Reference in a new issue