CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-04 14:10:33 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #5084 from appwrite/automate-hamster-script

Browse source 
feat: automate hamster
This commit is contained in:
Christy Jacob 2023-02-06 16:02:33 +05:30 committed by GitHub
parent 28e6de81d4 4672b31685
commit ef20d5f007
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 110 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.env
View file

@ -46,6 +46,7 @@ _APP_SMTP_SECURE=
_APP_SMTP_USERNAME=
_APP_SMTP_PASSWORD=
_APP_HAMSTER_RECIPIENTS=
_APP_HAMSTER_INTERVAL=86400
_APP_SMS_PROVIDER=sms://username:password@mock
_APP_SMS_FROM=+123456789
_APP_STORAGE_LIMIT=30000000

2
app/console

@ -1 +1 @@
Subproject commit aea8c5f2bbe0836fc9fb4720b21d44fe01ac93d1
Subproject commit 43891a526e061454617cbb13def3c4901d99a7f1

33
docker-compose.yml
View file

@ -544,6 +544,39 @@ services:
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-hamster:
entrypoint: hamster
<<: *x-logging
container_name: appwrite-hamster
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_HAMSTER_RECIPIENTS
- _APP_HAMSTER_INTERVAL
appwrite-maintenance:
entrypoint: maintenance
<<: *x-logging

135
src/Appwrite/Platform/Tasks/Hamster.php
View file

@ -145,79 +145,94 @@ class Hamster extends Action
public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void
{
Console::info('Getting stats...');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
Console::title('Cloud Hamster V1');
Console::success(APP_NAME . ' cloud hamster process v1 has started');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
/** Database connections */
$totalProjects = $dbForConsole->count('projects') + 1;
Console::success("Found a total of: {$totalProjects} projects");
Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $interval) {
$projects = [$console];
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Getting Cloud Usage Stats every {$interval} seconds");
$loopStart = microtime(true);
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects') + 1;
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project);
$csv->insertOne(array_values($statsPerProject));
} catch (\Throwable $th) {
throw $th;
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
Console::info("Getting stats for {$project->getId()}");
$sum = \count($projects);
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$offset = $offset + $limit;
$count = $count + $sum;
$statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project);
$csv->insertOne(array_values($statsPerProject));
} catch (\Throwable $th) {
throw $th;
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...');
}
$sum = \count($projects);
$this->sendEmail($register);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$pools
->get('console')
->reclaim();
$offset = $offset + $limit;
$count = $count + $sum;
Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...');
}
$this->sendEmail($register);
$pools
->get('console')
->reclaim();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
}, $interval);
}
private function sendEmail(Registry $register)