CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-20 12:42:39 +12:00
Code Issues Projects Releases Wiki Activity

refactor db object counts

Browse source
This commit is contained in:
Damodar Lohani 2021-08-17 11:48:27 +05:45
parent 583c447b4e
commit eeee0f712b
1 changed files with 41 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
app/tasks/usage.php
View file

@ -141,12 +141,15 @@ $cli
$cacheAdapter = new Cache(new Redis($redis));
$dbForProject = new Database(new MariaDB($db), $cacheAdapter);
$dbForConsole = new Database(new MariaDB($db), $cacheAdapter);
$dbForConsole->setNamespace('project_console_internal');
$latestTime = [];
Authorization::disable();
Console::loop(function () use ($interval, $register, $dbForProject, $globalMetrics, $periods, &$latestTime) {
$iterations = 0;
Console::loop(function () use ($interval, $register, $dbForProject, $dbForConsole, $globalMetrics, $periods, &$latestTime, &$iterations) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating usage data every {$interval} seconds");
@ -174,7 +177,7 @@ $cli
if (!empty($projectId) && $projectId != 'console') {
$dbForProject->setNamespace('project_' . $projectId . '_internal');
if (!empty($groupBy)) {
$groupedBy = $point[$groupBy];
$groupedBy = $point[$groupBy] ?? '';
if (empty($groupedBy)) {
continue;
}
@ -209,50 +212,44 @@ $cli
}
}
if ($iterations % 30 == 0) {
//aggregate number of objects in database
$latestProject = null;
do {
$projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject);
if (!empty($projects)) {
$latestProject = $projects[array_key_last($projects)];
foreach ($projects as $project) {
$id = $project->getId();
$collections = ['users' => [
'namespace' => 'internal',
], 'collections' => [
'namespace' => 'external',
], 'files' => [
'namespace' => 'internal',
]];
foreach ($collections as $collection => $options) {
$dbForProject->setNamespace("project_{$id}_{$options['namespace']}");
$count = $dbForProject->count($collection);
$dbForProject->setNamespace("project_{$id}_internal");
$dbForProject->createDocument('stats', new Document([
'$id' => $dbForProject->getId(),
'time' => time(),
'period' => '15m',
'metric' => "{$collection}.count",
'value' => $count,
'type' => 1,
]));
}
}
}
} while (!empty($projects));
}
$iterations++;
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
//aggregate number of objects in database
$dbForConsole = new Database(new MariaDB($db), $cacheAdapter);
$dbForConsole->setNamespace('project_console_internal');
$dbCountInterval = 15*60; //aggregate data every 15 minutes
Console::loop(function () use ($dbForConsole, $dbForProject, $dbCountInterval) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating Database object count every {$dbCountInterval} seconds");
$loopStart = microtime(true);
$latestProject = null;
do {
$projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject);
if (!empty($projects)) {
$latestProject = $projects[array_key_last($projects)];
foreach ($projects as $project) {
$id = $project->getId();
$dbForProject->setNamespace("project_{$id}_internal");
$collections = ['users', 'collections', 'files'];
foreach ($collections as $collection) {
$count = $dbForProject->count($collection);
$dbForProject->createDocument('stats', new Document([
'$id' => $dbForProject->getId(),
'time' => time(),
'period' => '15m',
'metric' => "{$collection}.count",
'value' => $count,
'type' => 1,
]));
}
}
}
} while (!empty($projects));
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] DB objects count aggregation took {$loopTook} seconds");
}, $dbCountInterval);
});