basic setup for extendable tasks

app/cli.php

@@ -3,30 +3,11 @@
 require_once __DIR__ . '/init.php';
 require_once __DIR__ . '/controllers/general.php';
 
-use Utopia\App;
-use Utopia\CLI\CLI;
-use Utopia\CLI\Console;
+use Appwrite\Task\Tasks;
 use Utopia\Database\Validator\Authorization;
 
 Authorization::disable();
 
-$cli = new CLI();
+Tasks::init();
+Tasks::getCli()->run();
 
-include 'tasks/doctor.php';
-include 'tasks/maintenance.php';
-include 'tasks/install.php';
-include 'tasks/migrate.php';
-include 'tasks/sdks.php';
-include 'tasks/specs.php';
-include 'tasks/ssl.php';
-include 'tasks/vars.php';
-include 'tasks/usage.php';
-
-$cli
-    ->task('version')
-    ->desc('Get the server version')
-    ->action(function () {
-        Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN'));
-    });
-
-$cli->run();

composer.json

@@ -43,7 +43,7 @@
         "utopia-php/analytics": "0.2.*",
         "utopia-php/audit": "0.8.*",
         "utopia-php/cache": "0.6.*",
-        "utopia-php/cli": "0.12.*",
+        "utopia-php/cli": "dev-feat-allow-adding-task as 0.12.0",
         "utopia-php/config": "0.2.*",
         "utopia-php/database": "0.18.*",
         "utopia-php/locale": "0.4.*",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "dedc6a6328b4fdc5dfbd556a08534403",
+    "content-hash": "4bd2cfb69f671c01e68b67221f8664fd",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -1947,16 +1947,16 @@
         },
         {
             "name": "utopia-php/cli",
-            "version": "0.12.0",
+            "version": "dev-feat-allow-adding-task",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/cli.git",
-                "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b"
+                "reference": "1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/cli/zipball/6d164b752efeb1ca089e3a517bc274d8b383474b",
-                "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b",
+                "url": "https://api.github.com/repos/utopia-php/cli/zipball/1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2",
+                "reference": "1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2",
                 "shasum": ""
             },
             "require": {
@@ -1994,9 +1994,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/cli/issues",
-                "source": "https://github.com/utopia-php/cli/tree/0.12.0"
+                "source": "https://github.com/utopia-php/cli/tree/feat-allow-adding-task"
             },
-            "time": "2022-02-18T22:10:41+00:00"
+            "time": "2022-07-08T01:44:31+00:00"
         },
         {
             "name": "utopia-php/config",
@@ -5346,9 +5346,18 @@
             "time": "2022-05-17T05:48:52+00:00"
         }
     ],
-    "aliases": [],
+    "aliases": [
+        {
+            "package": "utopia-php/cli",
+            "version": "dev-feat-allow-adding-task",
+            "alias": "0.12.0",
+            "alias_normalized": "0.12.0.0"
+        }
+    ],
     "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "utopia-php/cli": 20
+    },
     "prefer-stable": false,
     "prefer-lowest": false,
     "platform": {
@@ -5370,5 +5379,5 @@
     "platform-overrides": {
         "php": "8.0"
     },
-    "plugin-api-version": "2.3.0"
+    "plugin-api-version": "2.2.0"
 }

src/Appwrite/Task/Task.php

@@ -0,0 +1,9 @@
+<?php
+
+namespace Appwrite\Task;
+
+use Utopia\CLI\Task as CLITask;
+
+interface Task {
+    public static function getTask(): CLITask;
+}

src/Appwrite/Task/Tasks.php

@@ -0,0 +1,24 @@
+<?php
+namespace Appwrite\Task;
+
+use Utopia\CLI\CLI;
+use Appwrite\Task\Usage;
+use Appwrite\Task\Version;
+
+class Tasks {
+    protected static CLI $cli;
+
+
+    public static function init(): void
+    {
+        self::$cli = new CLI();
+        self::$cli->addTask(Vars::getTask());
+        self::$cli->addTask(Usage::getTask());
+        self::$cli->addTask(Version::getTask());
+    }
+
+    public static function getCli(): CLI
+    {
+        return self::$cli;
+    }
+}

src/Appwrite/Task/Usage.php

@@ -0,0 +1,178 @@
+<?php
+namespace Appwrite\Task;
+
+use Throwable;
+use Exception;
+use Appwrite\Task\Task;
+use Appwrite\Stats\Usage as InfluxUsage;
+use Appwrite\Stats\UsageDB;
+use InfluxDB\Database as InfluxDatabase;
+use Utopia\App;
+use Utopia\Cache\Adapter\Redis as RedisCache;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Adapter\MariaDB;
+use Utopia\Database\Database;
+use Utopia\Database\Validator\Authorization;
+use Utopia\Registry\Registry;
+use Utopia\Logger\Log;
+use Utopia\CLI\Task as CLITask;
+
+class Usage implements Task{
+    private static CLITask $task;
+    protected static function getDatabase(Registry &$register, string $namespace): Database
+    {
+        $attempts = 0;
+    
+        do {
+            try {
+                $attempts++;
+    
+                $db = $register->get('db');
+                $redis = $register->get('cache');
+    
+                $cache = new Cache(new RedisCache($redis));
+                $database = new Database(new MariaDB($db), $cache);
+                $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
+                $database->setNamespace($namespace);
+    
+                if (!$database->exists($database->getDefaultDatabase(), 'projects')) {
+                    throw new Exception('Projects collection not ready');
+                }
+                break; // leave loop if successful
+            } catch (\Exception$e) {
+                Console::warning("Database not ready. Retrying connection ({$attempts})...");
+                if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
+                    throw new \Exception('Failed to connect to database: ' . $e->getMessage());
+                }
+                sleep(DATABASE_RECONNECT_SLEEP);
+            }
+        } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
+    
+        return $database;
+    }
+    
+    protected static function getInfluxDB(Registry &$register): InfluxDatabase
+    {
+        /** @var InfluxDB\Client $client */
+        $client = $register->get('influxdb');
+        $attempts = 0;
+        $max = 10;
+        $sleep = 1;
+    
+        do { // check if telegraf database is ready
+            try {
+                $attempts++;
+                $database = $client->selectDB('telegraf');
+                if (in_array('telegraf', $client->listDatabases())) {
+                    break; // leave the do-while if successful
+                }
+            } catch (\Throwable$th) {
+                Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
+                if ($attempts >= $max) {
+                    throw new \Exception('InfluxDB database not ready yet');
+                }
+                sleep($sleep);
+            }
+        } while ($attempts < $max);
+        return $database;
+    }
+    
+    public static function getTask(): CLITask
+    {
+        global $register;
+
+        Authorization::disable();
+        Authorization::setDefaultStatus(false);
+
+        $logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) {
+            $logger = $register->get('logger');
+        
+            if ($logger) {
+                $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
+        
+                $log = new Log();
+                $log->setNamespace("usage");
+                $log->setServer(\gethostname());
+                $log->setVersion($version);
+                $log->setType(Log::TYPE_ERROR);
+                $log->setMessage($error->getMessage());
+        
+                $log->addTag('code', $error->getCode());
+                $log->addTag('verboseType', get_class($error));
+        
+                $log->addExtra('file', $error->getFile());
+                $log->addExtra('line', $error->getLine());
+                $log->addExtra('trace', $error->getTraceAsString());
+                $log->addExtra('detailedTrace', $error->getTrace());
+        
+                $log->setAction($action);
+        
+                $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
+                $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
+        
+                $responseCode = $logger->addLog($log);
+                Console::info('Usage stats log pushed with status code: ' . $responseCode);
+            }
+        
+            Console::warning("Failed: {$error->getMessage()}");
+            Console::warning($error->getTraceAsString());
+        };
+
+        $usage = new CLITask('usage');
+        
+        $usage
+            ->desc('Schedules syncing data from influxdb to Appwrite console db')
+            ->action(function () use ($register, $logError) {
+                Console::title('Usage Aggregation V1');
+                Console::success(APP_NAME . ' usage aggregation process v1 has started');
+        
+                $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
+        
+                $database = self::getDatabase($register, '_console');
+                $influxDB = self::getInfluxDB($register);
+        
+                $usage = new InfluxUsage($database, $influxDB, $logError);
+                $usageDB = new UsageDB($database, $logError);
+        
+                $iterations = 0;
+                Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) {
+                    $now = date('d-m-Y H:i:s', time());
+                    Console::info("[{$now}] Aggregating usage data every {$interval} seconds");
+        
+                    $loopStart = microtime(true);
+        
+                    /**
+                     * Aggregate InfluxDB every 30 seconds
+                     */
+                    $usage->collect();
+        
+                    if ($iterations % 30 != 0) { // return if 30 iterations has not passed
+                        $iterations++;
+                        $loopTook = microtime(true) - $loopStart;
+                        $now = date('d-m-Y H:i:s', time());
+                        Console::info("[{$now}] Aggregation took {$loopTook} seconds");
+                        return;
+                    }
+        
+                    $iterations = 0; // Reset iterations to prevent overflow when running for long time
+                    /**
+                     * Aggregate MariaDB every 15 minutes
+                     * Some of the queries here might contain full-table scans.
+                     */
+                    $now = date('d-m-Y H:i:s', time());
+                    Console::info("[{$now}] Aggregating database counters.");
+        
+                    $usageDB->collect();
+        
+                    $iterations++;
+                    $loopTook = microtime(true) - $loopStart;
+                    $now = date('d-m-Y H:i:s', time());
+        
+                    Console::info("[{$now}] Aggregation took {$loopTook} seconds");
+                }, $interval);
+            });
+        self::$task = $usage;
+        return self::$task;
+    }
+}

src/Appwrite/Task/Vars.php

@@ -0,0 +1,34 @@
+<?php
+namespace Appwrite\Task;
+use Utopia\App;
+use Utopia\CLI\Task as CLITask;
+use Utopia\Config\Config;
+use Utopia\CLI\Console;
+
+
+class Vars implements Task{
+    private static CLITask $task;
+    
+    public static function getTask(): CLITask
+    {
+        $vars = new CLITask('vars');
+        $vars
+            ->desc('List all the server environment variables')
+            ->action(function () {
+                $config = Config::getParam('variables', []);
+                $vars = [];
+        
+                foreach ($config as $category) {
+                    foreach ($category['variables'] ?? [] as $var) {
+                        $vars[] = $var;
+                    }
+                }
+        
+                foreach ($vars as $key => $value) {
+                    Console::log('- ' . $value['name'] . '=' . App::getEnv($value['name'], ''));
+                }
+            });
+        self::$task = $vars;
+        return self::$task;
+    }
+}

src/Appwrite/Task/Version.php

@@ -0,0 +1,22 @@
+<?php
+namespace Appwrite\Task;
+
+use Utopia\App;
+use Appwrite\Task\Task;
+use Utopia\CLI\Task as CLITask;
+use Utopia\CLI\Console;
+
+class Version implements Task {
+    protected static CLITask $task;
+
+    public static function getTask(): CLITask {
+        $version = new CLITask('version');
+        $version
+            ->desc('Get the server version')
+            ->action(function () {
+                Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN'));
+            });
+        self::$task = $version;
+        return self::$task;
+    }
+}