CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-20 12:42:39 +12:00
Code Issues Projects Releases Wiki Activity

counter stats update

Browse source
This commit is contained in:
Damodar Lohani 2021-08-19 13:46:22 +05:45
parent 91dc62d733
commit 800a76db72
2 changed files with 84 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
app/controllers/api/projects.php
View file

@ -285,33 +285,16 @@ App::get('/v1/projects/:projectId/usage')
}
}
$usersCount = Authorization::skip(function () use ($dbForInternal) {
return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]);
});
$usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]);
$usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0;
$collectionsCount = Authorization::skip(function () use ($dbForInternal, $period, $range) {
return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['collections.count'])], 0, ['time'], [Database::ORDER_DESC]);
});
$collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]);
$collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0;
// $documents = [];
$documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]);
$documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0;
// foreach ($collections as $collection) {
// $result = $projectDB->getCollection([
// 'limit' => 0,
// 'offset' => 0,
// 'filters' => [
// '$collection=' . $collection['$id'],
// ],
// ]);
// $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()];
// }
$filesCount = Authorization::skip(function () use ($dbForInternal, $period, $range) {
return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['files.count'])], 0, ['time'], [Database::ORDER_DESC]);
});
$filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]);
$filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0;
Authorization::reset();
@ -344,12 +327,10 @@ App::get('/v1/projects/:projectId/usage')
'data' => [],
'total' => $filesTotal,
],
// 'documents' => [
// 'data' => $documents,
// 'total' => \array_sum(\array_map(function ($item) {
// return $item['total'];
// }, $documents)),
// ],
'documents' => [
'data' => [],
'total' => $documentsTotal,
],
'users' => [
'data' => [],
'total' => $usersTotal,

91
app/tasks/usage.php
View file

@ -16,7 +16,7 @@ use Utopia\Database\Validator\Authorization;
/**
* Metrics We collect
*
*
* requests
* network
* executions
@ -42,13 +42,15 @@ use Utopia\Database\Validator\Authorization;
* users.delete
* users.sessions.create
* users.sessions.delete
*
*
* Counters
*
*
* users.count
* files.count
* collections.count
*
* storage.files.count
* database.collections.count
* database.documents.count
* database.collections.{collectionId}.documents.count
*
*/
$cli
@ -249,8 +251,10 @@ $cli
}
}
if ($iterations % 30 == 0) {
//aggregate number of objects in database
if ($iterations % 30 == 0) { //every 15 minutes
// aggregate number of objects in database
// get count of all the documents per collection -
// buckets will have the same
$latestProject = null;
do {
$projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject);
@ -259,25 +263,80 @@ $cli
foreach ($projects as $project) {
$id = $project->getId();
$collections = ['users' => [
'namespace' => 'internal',
], 'collections' => [
'namespace' => 'external',
], 'files' => [
'namespace' => 'internal',
]];
$collections = [
'users' => [
'namespace' => 'internal',
],
'collections' => [
'metricPrefix' => 'database',
'namespace' => 'external', //new change will make this internal
'subCollections' => [
'documents' => [
'namespace' => 'external',
],
],
],
'files' => [
'metricPrefix' => 'storage',
'namespace' => 'internal',
],
];
foreach ($collections as $collection => $options) {
$dbForProject->setNamespace("project_{$id}_{$options['namespace']}");
$count = $dbForProject->count($collection);
$dbForProject->setNamespace("project_{$id}_internal");
$metricPrefix = $options['metricPrefix'] ?? '';
$metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
$dbForProject->createDocument('stats', new Document([
'$id' => $dbForProject->getId(),
'time' => time(),
'period' => '15m',
'metric' => "{$collection}.count",
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
$subCollections = $options['subCollections'] ?? [];
if (!empty($subCollections)) {
$latestParent = null;
$subCollectionCounts = []; //total project level count of sub collections
do {
$dbForProject->setNamespace("project_{$id}_{$options['namespace']}");
$parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent);
if (!empty($parents)) {
$latestParent = $parents[array_key_last($parents)];
foreach ($parents as $parent) {
foreach ($subCollections as $subCollection => $subOptions) {
$dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}");
$count = $dbForProject->count($parent->getId());
$subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count;
$dbForProject->setNamespace("project_{$id}_internal");
$dbForProject->createDocument('stats', new Document([
'$id' => $dbForProject->getId(),
'time' => time(),
'period' => '15m',
'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count",
'value' => $count,
'type' => 1,
]));
}
}
}
} while (!empty($parents));
foreach ($subCollectionsCounts as $subCollection => $count) {
$dbForProject->setNamespace("project_{$id}_internal");
$dbForProject->createDocument('stats', new Document([
'$id' => $dbForProject->getId(),
'time' => time(),
'period' => '15m',
'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count",
'value' => $count,
'type' => 1,
]));
}
}
}
}
}