2021-08-10 20:44:31 +12:00
|
|
|
<?php
|
|
|
|
|
2021-08-11 22:15:00 +12:00
|
|
|
global $cli, $register;
|
2021-08-10 20:44:31 +12:00
|
|
|
|
2022-06-13 22:56:24 +12:00
|
|
|
use Appwrite\Stats\Usage;
|
|
|
|
use Appwrite\Stats\UsageDB;
|
2022-08-09 17:52:53 +12:00
|
|
|
use Appwrite\Usage\Calculators\Aggregator;
|
|
|
|
use Appwrite\Usage\Calculators\Database;
|
|
|
|
use Appwrite\Usage\Calculators\TimeSeries;
|
2022-06-13 22:56:24 +12:00
|
|
|
use InfluxDB\Database as InfluxDatabase;
|
2021-08-10 20:44:31 +12:00
|
|
|
use Utopia\App;
|
2022-06-13 22:56:24 +12:00
|
|
|
use Utopia\Cache\Adapter\Redis as RedisCache;
|
2021-08-11 22:15:00 +12:00
|
|
|
use Utopia\Cache\Cache;
|
2021-08-10 20:44:31 +12:00
|
|
|
use Utopia\CLI\Console;
|
2021-08-11 22:15:00 +12:00
|
|
|
use Utopia\Database\Adapter\MariaDB;
|
2022-08-09 17:52:53 +12:00
|
|
|
use Utopia\Database\Database as UtopiaDatabase;
|
2021-08-11 22:15:00 +12:00
|
|
|
use Utopia\Database\Validator\Authorization;
|
2022-06-13 22:56:24 +12:00
|
|
|
use Utopia\Registry\Registry;
|
2022-06-15 11:40:56 +12:00
|
|
|
use Utopia\Logger\Log;
|
2022-08-09 17:29:24 +12:00
|
|
|
use Utopia\Validator\WhiteList;
|
2022-06-13 22:56:24 +12:00
|
|
|
|
2022-06-20 23:21:36 +12:00
|
|
|
Authorization::disable();
|
|
|
|
Authorization::setDefaultStatus(false);
|
|
|
|
|
2022-08-09 17:52:53 +12:00
|
|
|
function getDatabase(Registry &$register, string $namespace): UtopiaDatabase
|
2022-06-13 22:56:24 +12:00
|
|
|
{
|
|
|
|
$attempts = 0;
|
|
|
|
|
|
|
|
do {
|
|
|
|
try {
|
|
|
|
$attempts++;
|
|
|
|
|
|
|
|
$db = $register->get('db');
|
|
|
|
$redis = $register->get('cache');
|
|
|
|
|
|
|
|
$cache = new Cache(new RedisCache($redis));
|
2022-08-09 17:52:53 +12:00
|
|
|
$database = new UtopiaDatabase(new MariaDB($db), $cache);
|
2022-06-13 22:56:24 +12:00
|
|
|
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
|
|
|
|
$database->setNamespace($namespace);
|
|
|
|
|
2022-06-13 23:11:26 +12:00
|
|
|
if (!$database->exists($database->getDefaultDatabase(), 'projects')) {
|
|
|
|
throw new Exception('Projects collection not ready');
|
2022-06-13 22:56:24 +12:00
|
|
|
}
|
|
|
|
break; // leave loop if successful
|
2022-08-09 17:52:53 +12:00
|
|
|
} catch (\Exception $e) {
|
2022-06-13 22:56:24 +12:00
|
|
|
Console::warning("Database not ready. Retrying connection ({$attempts})...");
|
|
|
|
if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
|
|
|
|
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
|
|
|
|
}
|
|
|
|
sleep(DATABASE_RECONNECT_SLEEP);
|
|
|
|
}
|
|
|
|
} while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
|
|
|
|
|
|
|
|
return $database;
|
|
|
|
}
|
|
|
|
|
2022-06-14 03:49:27 +12:00
|
|
|
function getInfluxDB(Registry &$register): InfluxDatabase
|
2022-06-13 22:56:24 +12:00
|
|
|
{
|
|
|
|
/** @var InfluxDB\Client $client */
|
|
|
|
$client = $register->get('influxdb');
|
|
|
|
$attempts = 0;
|
|
|
|
$max = 10;
|
|
|
|
$sleep = 1;
|
|
|
|
|
|
|
|
do { // check if telegraf database is ready
|
|
|
|
try {
|
|
|
|
$attempts++;
|
|
|
|
$database = $client->selectDB('telegraf');
|
|
|
|
if (in_array('telegraf', $client->listDatabases())) {
|
|
|
|
break; // leave the do-while if successful
|
|
|
|
}
|
2022-08-09 17:52:53 +12:00
|
|
|
} catch (\Throwable $th) {
|
2022-06-13 22:56:24 +12:00
|
|
|
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
|
|
|
|
if ($attempts >= $max) {
|
|
|
|
throw new \Exception('InfluxDB database not ready yet');
|
|
|
|
}
|
|
|
|
sleep($sleep);
|
|
|
|
}
|
|
|
|
} while ($attempts < $max);
|
|
|
|
return $database;
|
|
|
|
}
|
2021-08-10 20:44:31 +12:00
|
|
|
|
2022-06-15 11:40:56 +12:00
|
|
|
$logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) {
|
|
|
|
$logger = $register->get('logger');
|
|
|
|
|
|
|
|
if ($logger) {
|
|
|
|
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
|
|
|
|
|
|
|
|
$log = new Log();
|
2022-07-07 19:52:29 +12:00
|
|
|
$log->setNamespace("usage");
|
2022-06-15 11:40:56 +12:00
|
|
|
$log->setServer(\gethostname());
|
|
|
|
$log->setVersion($version);
|
|
|
|
$log->setType(Log::TYPE_ERROR);
|
|
|
|
$log->setMessage($error->getMessage());
|
|
|
|
|
|
|
|
$log->addTag('code', $error->getCode());
|
|
|
|
$log->addTag('verboseType', get_class($error));
|
|
|
|
|
|
|
|
$log->addExtra('file', $error->getFile());
|
|
|
|
$log->addExtra('line', $error->getLine());
|
|
|
|
$log->addExtra('trace', $error->getTraceAsString());
|
|
|
|
$log->addExtra('detailedTrace', $error->getTrace());
|
|
|
|
|
|
|
|
$log->setAction($action);
|
|
|
|
|
|
|
|
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
|
|
|
|
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
|
|
|
|
|
|
|
|
$responseCode = $logger->addLog($log);
|
|
|
|
Console::info('Usage stats log pushed with status code: ' . $responseCode);
|
|
|
|
}
|
|
|
|
|
|
|
|
Console::warning("Failed: {$error->getMessage()}");
|
|
|
|
Console::warning($error->getTraceAsString());
|
2022-06-13 23:11:26 +12:00
|
|
|
};
|
2021-08-19 18:54:32 +12:00
|
|
|
|
2022-08-13 16:51:52 +12:00
|
|
|
|
|
|
|
function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
|
|
|
|
{
|
|
|
|
$interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
|
2022-09-25 18:41:13 +13:00
|
|
|
$region = App::getEnv('region', 'default');
|
|
|
|
$usage = new TimeSeries($region, $database, $influxDB, $logError);
|
2022-08-13 16:51:52 +12:00
|
|
|
|
|
|
|
Console::loop(function () use ($interval, $usage) {
|
|
|
|
$now = date('d-m-Y H:i:s', time());
|
|
|
|
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
|
|
|
|
$loopStart = microtime(true);
|
|
|
|
|
|
|
|
$usage->collect();
|
|
|
|
|
|
|
|
$loopTook = microtime(true) - $loopStart;
|
|
|
|
$now = date('d-m-Y H:i:s', time());
|
|
|
|
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
|
|
|
|
}, $interval);
|
|
|
|
}
|
|
|
|
|
|
|
|
function aggregateDatabase(UtopiaDatabase $database, callable $logError): void
|
|
|
|
{
|
|
|
|
$interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default)
|
2022-09-25 18:41:13 +13:00
|
|
|
$region = App::getEnv('region', 'default');
|
|
|
|
$usage = new Database($region, $database, $logError);
|
|
|
|
$aggregrator = new Aggregator($region, $database, $logError);
|
2022-08-13 16:51:52 +12:00
|
|
|
|
|
|
|
Console::loop(function () use ($interval, $usage, $aggregrator) {
|
|
|
|
$now = date('d-m-Y H:i:s', time());
|
|
|
|
Console::info("[{$now}] Aggregating database usage every {$interval} seconds.");
|
|
|
|
$loopStart = microtime(true);
|
|
|
|
$usage->collect();
|
|
|
|
$aggregrator->collect();
|
|
|
|
$loopTook = microtime(true) - $loopStart;
|
|
|
|
$now = date('d-m-Y H:i:s', time());
|
|
|
|
|
|
|
|
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
|
|
|
|
}, $interval);
|
|
|
|
}
|
|
|
|
|
2021-08-10 20:44:31 +12:00
|
|
|
$cli
|
|
|
|
->task('usage')
|
2022-08-09 17:29:24 +12:00
|
|
|
->param('type', 'timeseries', new WhiteList(['timeseries', 'database']))
|
2021-08-10 20:44:31 +12:00
|
|
|
->desc('Schedules syncing data from influxdb to Appwrite console db')
|
2022-08-09 17:29:24 +12:00
|
|
|
->action(function (string $type) use ($register, $logError) {
|
2021-08-17 00:31:49 +12:00
|
|
|
Console::title('Usage Aggregation V1');
|
|
|
|
Console::success(APP_NAME . ' usage aggregation process v1 has started');
|
2021-08-10 20:44:31 +12:00
|
|
|
|
2022-06-13 22:56:24 +12:00
|
|
|
$database = getDatabase($register, '_console');
|
|
|
|
$influxDB = getInfluxDB($register);
|
|
|
|
|
2022-08-13 17:10:34 +12:00
|
|
|
switch ($type) {
|
2022-08-13 16:51:52 +12:00
|
|
|
case 'timeseries':
|
|
|
|
aggregateTimeseries($database, $influxDB, $logError);
|
|
|
|
break;
|
|
|
|
case 'database':
|
|
|
|
aggregateDatabase($database, $logError);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
Console::error("Unsupported usage aggregation type");
|
2022-08-09 17:52:53 +12:00
|
|
|
}
|
2022-06-13 22:56:24 +12:00
|
|
|
});
|