From cb2fc93f78bf0ae71da94934f79607cc0827721b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 8 Jul 2022 02:27:06 +0000 Subject: [PATCH 01/50] basic setup for extendable tasks --- app/cli.php | 25 +---- composer.json | 2 +- composer.lock | 29 ++++-- src/Appwrite/Task/Task.php | 9 ++ src/Appwrite/Task/Tasks.php | 24 +++++ src/Appwrite/Task/Usage.php | 178 ++++++++++++++++++++++++++++++++++ src/Appwrite/Task/Vars.php | 34 +++++++ src/Appwrite/Task/Version.php | 22 +++++ 8 files changed, 290 insertions(+), 33 deletions(-) create mode 100644 src/Appwrite/Task/Task.php create mode 100644 src/Appwrite/Task/Tasks.php create mode 100644 src/Appwrite/Task/Usage.php create mode 100644 src/Appwrite/Task/Vars.php create mode 100644 src/Appwrite/Task/Version.php diff --git a/app/cli.php b/app/cli.php index 09b3dc9413..0539c98313 100644 --- a/app/cli.php +++ b/app/cli.php @@ -3,30 +3,11 @@ require_once __DIR__ . '/init.php'; require_once __DIR__ . '/controllers/general.php'; -use Utopia\App; -use Utopia\CLI\CLI; -use Utopia\CLI\Console; +use Appwrite\Task\Tasks; use Utopia\Database\Validator\Authorization; Authorization::disable(); -$cli = new CLI(); +Tasks::init(); +Tasks::getCli()->run(); -include 'tasks/doctor.php'; -include 'tasks/maintenance.php'; -include 'tasks/install.php'; -include 'tasks/migrate.php'; -include 'tasks/sdks.php'; -include 'tasks/specs.php'; -include 'tasks/ssl.php'; -include 'tasks/vars.php'; -include 'tasks/usage.php'; - -$cli - ->task('version') - ->desc('Get the server version') - ->action(function () { - Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN')); - }); - -$cli->run(); diff --git a/composer.json b/composer.json index ded0834bb3..2cf3e44004 100644 --- a/composer.json +++ b/composer.json @@ -43,7 +43,7 @@ "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.8.*", "utopia-php/cache": "0.6.*", - "utopia-php/cli": "0.12.*", + "utopia-php/cli": "dev-feat-allow-adding-task as 0.12.0", "utopia-php/config": "0.2.*", "utopia-php/database": "0.18.*", "utopia-php/locale": "0.4.*", diff --git a/composer.lock b/composer.lock index 9d29bc8ac7..3460506d35 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "dedc6a6328b4fdc5dfbd556a08534403", + "content-hash": "4bd2cfb69f671c01e68b67221f8664fd", "packages": [ { "name": "adhocore/jwt", @@ -1947,16 +1947,16 @@ }, { "name": "utopia-php/cli", - "version": "0.12.0", + "version": "dev-feat-allow-adding-task", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b" + "reference": "1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/6d164b752efeb1ca089e3a517bc274d8b383474b", - "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2", + "reference": "1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2", "shasum": "" }, "require": { @@ -1994,9 +1994,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.12.0" + "source": "https://github.com/utopia-php/cli/tree/feat-allow-adding-task" }, - "time": "2022-02-18T22:10:41+00:00" + "time": "2022-07-08T01:44:31+00:00" }, { "name": "utopia-php/config", @@ -5346,9 +5346,18 @@ "time": "2022-05-17T05:48:52+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/cli", + "version": "dev-feat-allow-adding-task", + "alias": "0.12.0", + "alias_normalized": "0.12.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/cli": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -5370,5 +5379,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Task/Task.php b/src/Appwrite/Task/Task.php new file mode 100644 index 0000000000..a10049b407 --- /dev/null +++ b/src/Appwrite/Task/Task.php @@ -0,0 +1,9 @@ +addTask(Vars::getTask()); + self::$cli->addTask(Usage::getTask()); + self::$cli->addTask(Version::getTask()); + } + + public static function getCli(): CLI + { + return self::$cli; + } +} diff --git a/src/Appwrite/Task/Usage.php b/src/Appwrite/Task/Usage.php new file mode 100644 index 0000000000..777dc303f6 --- /dev/null +++ b/src/Appwrite/Task/Usage.php @@ -0,0 +1,178 @@ +get('db'); + $redis = $register->get('cache'); + + $cache = new Cache(new RedisCache($redis)); + $database = new Database(new MariaDB($db), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace($namespace); + + if (!$database->exists($database->getDefaultDatabase(), 'projects')) { + throw new Exception('Projects collection not ready'); + } + break; // leave loop if successful + } catch (\Exception$e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep(DATABASE_RECONNECT_SLEEP); + } + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + + return $database; + } + + protected static function getInfluxDB(Registry &$register): InfluxDatabase + { + /** @var InfluxDB\Client $client */ + $client = $register->get('influxdb'); + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if (in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful + } + } catch (\Throwable$th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } + } while ($attempts < $max); + return $database; + } + + public static function getTask(): CLITask + { + global $register; + + Authorization::disable(); + Authorization::setDefaultStatus(false); + + $logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) { + $logger = $register->get('logger'); + + if ($logger) { + $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace("usage"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + + $log->setAction($action); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $responseCode = $logger->addLog($log); + Console::info('Usage stats log pushed with status code: ' . $responseCode); + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); + }; + + $usage = new CLITask('usage'); + + $usage + ->desc('Schedules syncing data from influxdb to Appwrite console db') + ->action(function () use ($register, $logError) { + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); + + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) + + $database = self::getDatabase($register, '_console'); + $influxDB = self::getInfluxDB($register); + + $usage = new InfluxUsage($database, $influxDB, $logError); + $usageDB = new UsageDB($database, $logError); + + $iterations = 0; + Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); + + $loopStart = microtime(true); + + /** + * Aggregate InfluxDB every 30 seconds + */ + $usage->collect(); + + if ($iterations % 30 != 0) { // return if 30 iterations has not passed + $iterations++; + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + return; + } + + $iterations = 0; // Reset iterations to prevent overflow when running for long time + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating database counters."); + + $usageDB->collect(); + + $iterations++; + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); + }); + self::$task = $usage; + return self::$task; + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Vars.php b/src/Appwrite/Task/Vars.php new file mode 100644 index 0000000000..7cfb1904ac --- /dev/null +++ b/src/Appwrite/Task/Vars.php @@ -0,0 +1,34 @@ +desc('List all the server environment variables') + ->action(function () { + $config = Config::getParam('variables', []); + $vars = []; + + foreach ($config as $category) { + foreach ($category['variables'] ?? [] as $var) { + $vars[] = $var; + } + } + + foreach ($vars as $key => $value) { + Console::log('- ' . $value['name'] . '=' . App::getEnv($value['name'], '')); + } + }); + self::$task = $vars; + return self::$task; + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Version.php b/src/Appwrite/Task/Version.php new file mode 100644 index 0000000000..d63e1306c5 --- /dev/null +++ b/src/Appwrite/Task/Version.php @@ -0,0 +1,22 @@ +desc('Get the server version') + ->action(function () { + Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN')); + }); + self::$task = $version; + return self::$task; + } +} From a17496d2915afc0b3158042e051a9f7990d2bcc7 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 8 Jul 2022 02:36:54 +0000 Subject: [PATCH 02/50] update tasks --- app/cli.php | 6 ++++-- src/Appwrite/Task/Tasks.php | 18 +++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/app/cli.php b/app/cli.php index 0539c98313..9b4bfdd85c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -8,6 +8,8 @@ use Utopia\Database\Validator\Authorization; Authorization::disable(); -Tasks::init(); -Tasks::getCli()->run(); +$tasks = new Tasks(); +$tasks + ->init() + ->run(); diff --git a/src/Appwrite/Task/Tasks.php b/src/Appwrite/Task/Tasks.php index 6eae8e97a1..ca50c1caa5 100644 --- a/src/Appwrite/Task/Tasks.php +++ b/src/Appwrite/Task/Tasks.php @@ -6,19 +6,19 @@ use Appwrite\Task\Usage; use Appwrite\Task\Version; class Tasks { - protected static CLI $cli; + protected CLI $cli; - - public static function init(): void + public function init(): Tasks { - self::$cli = new CLI(); - self::$cli->addTask(Vars::getTask()); - self::$cli->addTask(Usage::getTask()); - self::$cli->addTask(Version::getTask()); + $this->cli = new CLI(); + $this->cli->addTask(Vars::getTask()); + $this->cli->addTask(Usage::getTask()); + $this->cli->addTask(Version::getTask()); + return $this; } - public static function getCli(): CLI + public function run(): CLI { - return self::$cli; + return $this->cli->run(); } } From 752117e07a4ebb4e133ca86b5fc0522c0f4e2b42 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 13 Jul 2022 06:26:22 +0000 Subject: [PATCH 03/50] using platform to implement CLI --- app/cli.php | 11 +-- composer.json | 9 ++- composer.lock | 94 ++++++++++++++++++----- src/Appwrite/Task/CLIPlatform.php | 12 +++ src/Appwrite/Task/Task.php | 9 --- src/Appwrite/Task/Tasks.php | 24 ++---- src/Appwrite/Task/Usage.php | 123 +++++++++++++++--------------- src/Appwrite/Task/Vars.php | 44 +++++------ src/Appwrite/Task/Version.php | 17 ++--- 9 files changed, 196 insertions(+), 147 deletions(-) create mode 100644 src/Appwrite/Task/CLIPlatform.php delete mode 100644 src/Appwrite/Task/Task.php diff --git a/app/cli.php b/app/cli.php index 9b4bfdd85c..413fa234ea 100644 --- a/app/cli.php +++ b/app/cli.php @@ -3,13 +3,14 @@ require_once __DIR__ . '/init.php'; require_once __DIR__ . '/controllers/general.php'; -use Appwrite\Task\Tasks; +use Appwrite\Task\CLIPlatform; use Utopia\Database\Validator\Authorization; Authorization::disable(); -$tasks = new Tasks(); -$tasks - ->init() - ->run(); +$cliPlatform = new CLIPlatform(); +$cliPlatform->init('CLI'); + +$cli = $cliPlatform->getCli(); +$cli->run(); diff --git a/composer.json b/composer.json index 8cc38115b5..d29a6d4e8f 100644 --- a/composer.json +++ b/composer.json @@ -42,13 +42,14 @@ "ext-sockets": "*", "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.10.*", - "utopia-php/framework": "0.19.*", + "utopia-php/framework": "dev-feat-public-add-route as 0.19.21", "utopia-php/logger": "0.3.*", "utopia-php/abuse": "0.7.*", "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.8.*", "utopia-php/cache": "0.6.*", - "utopia-php/cli": "dev-feat-allow-adding-task as 0.12.0", + "utopia-php/cli": "0.12.0", + "utopia-php/platform": "dev-dev", "utopia-php/config": "0.2.*", "utopia-php/database": "0.18.*", "utopia-php/locale": "0.4.*", @@ -73,6 +74,10 @@ { "url": "https://github.com/appwrite/runtimes.git", "type": "git" + }, + { + "url": "https://github.com/utopia-php/platform.git", + "type": "git" } ], "require-dev": { diff --git a/composer.lock b/composer.lock index eefbe33a8c..1994e95967 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4bd2cfb69f671c01e68b67221f8664fd", + "content-hash": "3775608dcbc4262c4edd8710aa94ce4d", "packages": [ { "name": "adhocore/jwt", @@ -1947,16 +1947,16 @@ }, { "name": "utopia-php/cli", - "version": "dev-feat-allow-adding-task", + "version": "0.12.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2" + "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2", - "reference": "1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/6d164b752efeb1ca089e3a517bc274d8b383474b", + "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b", "shasum": "" }, "require": { @@ -1994,9 +1994,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/feat-allow-adding-task" + "source": "https://github.com/utopia-php/cli/tree/0.12.0" }, - "time": "2022-07-08T01:44:31+00:00" + "time": "2022-02-18T22:10:41+00:00" }, { "name": "utopia-php/config", @@ -2169,16 +2169,16 @@ }, { "name": "utopia-php/framework", - "version": "0.19.21", + "version": "dev-feat-public-add-route", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "3b7bd8e4acf84fd7d560ced8e0142221d302575d" + "reference": "d16f3d44375beb1e880600957ac65e97699d6e23" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/3b7bd8e4acf84fd7d560ced8e0142221d302575d", - "reference": "3b7bd8e4acf84fd7d560ced8e0142221d302575d", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/d16f3d44375beb1e880600957ac65e97699d6e23", + "reference": "d16f3d44375beb1e880600957ac65e97699d6e23", "shasum": "" }, "require": { @@ -2212,9 +2212,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.19.21" + "source": "https://github.com/utopia-php/framework/tree/feat-public-add-route" }, - "time": "2022-05-12T18:42:28+00:00" + "time": "2022-07-10T08:39:49+00:00" }, { "name": "utopia-php/image", @@ -2440,6 +2440,63 @@ }, "time": "2022-02-20T09:23:06+00:00" }, + { + "name": "utopia-php/platform", + "version": "dev-dev", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/platform.git", + "reference": "370565fcc39996506357338e28cad97cb9e3db0f" + }, + "require": { + "ext-json": "*", + "ext-redis": "*", + "php": ">=8.0", + "utopia-php/cli": "0.12.*", + "utopia-php/framework": "dev-feat-public-add-route as 0.19.21" + }, + "require-dev": { + "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Platform\\": "src/Platform" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Platform" + } + }, + "scripts": { + "format": [ + "vendor/bin/phpcbf" + ], + "test": [ + "docker-compose up -d && sleep 10 && docker-compose exec web vendor/bin/phpunit --configuration phpunit.xml" + ] + }, + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + } + ], + "description": "Light and Fast Platform Library", + "keywords": [ + "cache", + "framework", + "php", + "upf", + "utopia" + ], + "time": "2022-07-13T05:59:26+00:00" + }, { "name": "utopia-php/preloader", "version": "0.2.4", @@ -5348,15 +5405,16 @@ ], "aliases": [ { - "package": "utopia-php/cli", - "version": "dev-feat-allow-adding-task", - "alias": "0.12.0", - "alias_normalized": "0.12.0.0" + "package": "utopia-php/framework", + "version": "dev-feat-public-add-route", + "alias": "0.19.21", + "alias_normalized": "0.19.21.0" } ], "minimum-stability": "stable", "stability-flags": { - "utopia-php/cli": 20 + "utopia-php/framework": 20, + "utopia-php/platform": 20 }, "prefer-stable": false, "prefer-lowest": false, diff --git a/src/Appwrite/Task/CLIPlatform.php b/src/Appwrite/Task/CLIPlatform.php new file mode 100644 index 0000000000..847071fd63 --- /dev/null +++ b/src/Appwrite/Task/CLIPlatform.php @@ -0,0 +1,12 @@ +addService('cliTasks', new Tasks()); + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Task.php b/src/Appwrite/Task/Task.php deleted file mode 100644 index a10049b407..0000000000 --- a/src/Appwrite/Task/Task.php +++ /dev/null @@ -1,9 +0,0 @@ -cli = new CLI(); - $this->cli->addTask(Vars::getTask()); - $this->cli->addTask(Usage::getTask()); - $this->cli->addTask(Version::getTask()); - return $this; - } - - public function run(): CLI - { - return $this->cli->run(); + $this->type = self::TYPE_CLI; + $this->addAction('version', new Version()); + $this->addAction('usage', new Usage()); + $this->addAction('vars', new Vars()); } } diff --git a/src/Appwrite/Task/Usage.php b/src/Appwrite/Task/Usage.php index 777dc303f6..9dd855e2f3 100644 --- a/src/Appwrite/Task/Usage.php +++ b/src/Appwrite/Task/Usage.php @@ -3,7 +3,6 @@ namespace Appwrite\Task; use Throwable; use Exception; -use Appwrite\Task\Task; use Appwrite\Stats\Usage as InfluxUsage; use Appwrite\Stats\UsageDB; use InfluxDB\Database as InfluxDatabase; @@ -16,11 +15,10 @@ use Utopia\Database\Database; use Utopia\Database\Validator\Authorization; use Utopia\Registry\Registry; use Utopia\Logger\Log; -use Utopia\CLI\Task as CLITask; +use Utopia\Platform\Action; -class Usage implements Task{ - private static CLITask $task; - protected static function getDatabase(Registry &$register, string $namespace): Database +class Usage extends Action{ + protected function getDatabase(Registry &$register, string $namespace): Database { $attempts = 0; @@ -52,7 +50,7 @@ class Usage implements Task{ return $database; } - protected static function getInfluxDB(Registry &$register): InfluxDatabase + protected function getInfluxDB(Registry &$register): InfluxDatabase { /** @var InfluxDB\Client $client */ $client = $register->get('influxdb'); @@ -78,8 +76,17 @@ class Usage implements Task{ return $database; } - public static function getTask(): CLITask + public function __construct() { + + + $this + ->desc('Schedules syncing data from influxdb to Appwrite console db') + ->callback(fn () => $this->action()); + } + + public function action() { + global $register; Authorization::disable(); @@ -119,60 +126,52 @@ class Usage implements Task{ Console::warning($error->getTraceAsString()); }; - $usage = new CLITask('usage'); - - $usage - ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function () use ($register, $logError) { - Console::title('Usage Aggregation V1'); - Console::success(APP_NAME . ' usage aggregation process v1 has started'); - - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) - - $database = self::getDatabase($register, '_console'); - $influxDB = self::getInfluxDB($register); - - $usage = new InfluxUsage($database, $influxDB, $logError); - $usageDB = new UsageDB($database, $logError); - - $iterations = 0; - Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); - - $loopStart = microtime(true); - - /** - * Aggregate InfluxDB every 30 seconds - */ - $usage->collect(); - - if ($iterations % 30 != 0) { // return if 30 iterations has not passed - $iterations++; - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - return; - } - - $iterations = 0; // Reset iterations to prevent overflow when running for long time - /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating database counters."); - - $usageDB->collect(); - - $iterations++; - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - }, $interval); - }); - self::$task = $usage; - return self::$task; + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); + + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) + + $database = self::getDatabase($register, '_console'); + $influxDB = self::getInfluxDB($register); + + $usage = new InfluxUsage($database, $influxDB, $logError); + $usageDB = new UsageDB($database, $logError); + + $iterations = 0; + Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); + + $loopStart = microtime(true); + + /** + * Aggregate InfluxDB every 30 seconds + */ + $usage->collect(); + + if ($iterations % 30 != 0) { // return if 30 iterations has not passed + $iterations++; + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + return; + } + + $iterations = 0; // Reset iterations to prevent overflow when running for long time + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating database counters."); + + $usageDB->collect(); + + $iterations++; + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); } } \ No newline at end of file diff --git a/src/Appwrite/Task/Vars.php b/src/Appwrite/Task/Vars.php index 7cfb1904ac..fc50e2b114 100644 --- a/src/Appwrite/Task/Vars.php +++ b/src/Appwrite/Task/Vars.php @@ -1,34 +1,32 @@ desc('List all the server environment variables') - ->action(function () { - $config = Config::getParam('variables', []); - $vars = []; - - foreach ($config as $category) { - foreach ($category['variables'] ?? [] as $var) { - $vars[] = $var; - } - } - - foreach ($vars as $key => $value) { - Console::log('- ' . $value['name'] . '=' . App::getEnv($value['name'], '')); - } - }); - self::$task = $vars; - return self::$task; + ->callback(fn () => $this->action()); + } + + public function action(): void + { + $config = Config::getParam('variables', []); + $vars = []; + + foreach ($config as $category) { + foreach ($category['variables'] ?? [] as $var) { + $vars[] = $var; + } + } + + foreach ($vars as $key => $value) { + Console::log('- ' . $value['name'] . '=' . App::getEnv($value['name'], '')); + } } } \ No newline at end of file diff --git a/src/Appwrite/Task/Version.php b/src/Appwrite/Task/Version.php index d63e1306c5..07ef47f292 100644 --- a/src/Appwrite/Task/Version.php +++ b/src/Appwrite/Task/Version.php @@ -2,21 +2,16 @@ namespace Appwrite\Task; use Utopia\App; -use Appwrite\Task\Task; -use Utopia\CLI\Task as CLITask; use Utopia\CLI\Console; +use Utopia\Platform\Action; -class Version implements Task { - protected static CLITask $task; - - public static function getTask(): CLITask { - $version = new CLITask('version'); - $version +class Version extends Action { + public function __construct() + { + $this ->desc('Get the server version') - ->action(function () { + ->callback(function () { Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN')); }); - self::$task = $version; - return self::$task; } } From 5793a9c3761b3a705296105943b4ff83b6fad457 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 13 Jul 2022 07:02:55 +0000 Subject: [PATCH 04/50] refactor all the tasks using action --- src/Appwrite/Task/Doctor.php | 255 ++++++++++++++++++++++++++++ src/Appwrite/Task/Install.php | 241 ++++++++++++++++++++++++++ src/Appwrite/Task/Maintenance.php | 159 ++++++++++++++++++ src/Appwrite/Task/Migrate.php | 95 +++++++++++ src/Appwrite/Task/SDKs.php | 271 ++++++++++++++++++++++++++++++ src/Appwrite/Task/SSL.php | 34 ++++ src/Appwrite/Task/Specs.php | 269 +++++++++++++++++++++++++++++ src/Appwrite/Task/Tasks.php | 14 +- src/Appwrite/Task/Usage.php | 2 + src/Appwrite/Task/Vars.php | 2 + src/Appwrite/Task/Version.php | 2 + 11 files changed, 1341 insertions(+), 3 deletions(-) create mode 100644 src/Appwrite/Task/Doctor.php create mode 100644 src/Appwrite/Task/Install.php create mode 100644 src/Appwrite/Task/Maintenance.php create mode 100644 src/Appwrite/Task/Migrate.php create mode 100644 src/Appwrite/Task/SDKs.php create mode 100644 src/Appwrite/Task/SSL.php create mode 100644 src/Appwrite/Task/Specs.php diff --git a/src/Appwrite/Task/Doctor.php b/src/Appwrite/Task/Doctor.php new file mode 100644 index 0000000000..529116813b --- /dev/null +++ b/src/Appwrite/Task/Doctor.php @@ -0,0 +1,255 @@ +desc('Validate server health') + ->callback(fn () => $this->action()); + } + + public function action(): void + { + global $register; + + Console::log(" __ ____ ____ _ _ ____ __ ____ ____ __ __ + / _\ ( _ \( _ \/ )( \( _ \( )(_ _)( __) ( )/ \ +/ \ ) __/ ) __/\ /\ / ) / )( )( ) _) _ )(( O ) +\_/\_/(__) (__) (_/\_)(__\_)(__) (__) (____)(_)(__)\__/ "); + + Console::log("\n" . '👩‍⚕️ Running ' . APP_NAME . ' Doctor for version ' . App::getEnv('_APP_VERSION', 'UNKNOWN') . ' ...' . "\n"); + + Console::log('Checking for production best practices...'); + + $domain = new Domain(App::getEnv('_APP_DOMAIN')); + + if (!$domain->isKnown() || $domain->isTest()) { + Console::log('🔴 Hostname has no public suffix (' . $domain->get() . ')'); + } else { + Console::log('🟢 Hostname has a public suffix (' . $domain->get() . ')'); + } + + $domain = new Domain(App::getEnv('_APP_DOMAIN_TARGET')); + + if (!$domain->isKnown() || $domain->isTest()) { + Console::log('🔴 CNAME target has no public suffix (' . $domain->get() . ')'); + } else { + Console::log('🟢 CNAME target has a public suffix (' . $domain->get() . ')'); + } + + if (App::getEnv('_APP_OPENSSL_KEY_V1') === 'your-secret-key' || empty(App::getEnv('_APP_OPENSSL_KEY_V1'))) { + Console::log('🔴 Not using a unique secret key for encryption'); + } else { + Console::log('🟢 Using a unique secret key for encryption'); + } + + if (App::getEnv('_APP_ENV', 'development') !== 'production') { + Console::log('🔴 App environment is set for development'); + } else { + Console::log('🟢 App environment is set for production'); + } + + if ('enabled' !== App::getEnv('_APP_OPTIONS_ABUSE', 'disabled')) { + Console::log('🔴 Abuse protection is disabled'); + } else { + Console::log('🟢 Abuse protection is enabled'); + } + + $authWhitelistRoot = App::getEnv('_APP_CONSOLE_WHITELIST_ROOT', null); + $authWhitelistEmails = App::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null); + $authWhitelistIPs = App::getEnv('_APP_CONSOLE_WHITELIST_IPS', null); + + if ( + empty($authWhitelistRoot) + && empty($authWhitelistEmails) + && empty($authWhitelistIPs) + ) { + Console::log('🔴 Console access limits are disabled'); + } else { + Console::log('🟢 Console access limits are enabled'); + } + + if ('enabled' !== App::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled')) { + Console::log('🔴 HTTPS force option is disabled'); + } else { + Console::log('🟢 HTTPS force option is enabled'); + } + + + $providerName = App::getEnv('_APP_LOGGING_PROVIDER', ''); + $providerConfig = App::getEnv('_APP_LOGGING_CONFIG', ''); + + if (empty($providerName) || empty($providerConfig) || !Logger::hasProvider($providerName)) { + Console::log('🔴 Logging adapter is disabled'); + } else { + Console::log('🟢 Logging adapter is enabled (' . $providerName . ')'); + } + + \sleep(0.2); + + try { + Console::log("\n" . 'Checking connectivity...'); + } catch (\Throwable $th) { + //throw $th; + } + + try { + $register->get('db'); /* @var $db PDO */ + Console::success('Database............connected 👍'); + } catch (\Throwable $th) { + Console::error('Database.........disconnected 👎'); + } + + try { + $register->get('cache'); + Console::success('Queue...............connected 👍'); + } catch (\Throwable $th) { + Console::error('Queue............disconnected 👎'); + } + + try { + $register->get('cache'); + Console::success('Cache...............connected 👍'); + } catch (\Throwable $th) { + Console::error('Cache............disconnected 👎'); + } + + if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled') { // Check if scans are enabled + try { + $antivirus = new Network( + App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), + (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) + ); + + if ((@$antivirus->ping())) { + Console::success('Antivirus...........connected 👍'); + } else { + Console::error('Antivirus........disconnected 👎'); + } + } catch (\Throwable $th) { + Console::error('Antivirus........disconnected 👎'); + } + } + + try { + $mail = $register->get('smtp'); /* @var $mail \PHPMailer\PHPMailer\PHPMailer */ + + $mail->addAddress('demo@example.com', 'Example.com'); + $mail->Subject = 'Test SMTP Connection'; + $mail->Body = 'Hello World'; + $mail->AltBody = 'Hello World'; + + $mail->send(); + Console::success('SMTP................connected 👍'); + } catch (\Throwable $th) { + Console::error('SMTP.............disconnected 👎'); + } + + $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); + $port = App::getEnv('_APP_STATSD_PORT', 8125); + + if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) { + Console::success('StatsD..............connected 👍'); + \fclose($fp); + } else { + Console::error('StatsD...........disconnected 👎'); + } + + $host = App::getEnv('_APP_INFLUXDB_HOST', ''); + $port = App::getEnv('_APP_INFLUXDB_PORT', ''); + + if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) { + Console::success('InfluxDB............connected 👍'); + \fclose($fp); + } else { + Console::error('InfluxDB.........disconnected 👎'); + } + + \sleep(0.2); + + Console::log(''); + Console::log('Checking volumes...'); + + foreach ( + [ + 'Uploads' => APP_STORAGE_UPLOADS, + 'Cache' => APP_STORAGE_CACHE, + 'Config' => APP_STORAGE_CONFIG, + 'Certs' => APP_STORAGE_CERTIFICATES + ] as $key => $volume + ) { + $device = new Local($volume); + + if (\is_readable($device->getRoot())) { + Console::success('🟢 ' . $key . ' Volume is readable'); + } else { + Console::error('🔴 ' . $key . ' Volume is unreadable'); + } + + if (\is_writable($device->getRoot())) { + Console::success('🟢 ' . $key . ' Volume is writeable'); + } else { + Console::error('🔴 ' . $key . ' Volume is unwriteable'); + } + } + + \sleep(0.2); + + Console::log(''); + Console::log('Checking disk space usage...'); + + foreach ( + [ + 'Uploads' => APP_STORAGE_UPLOADS, + 'Cache' => APP_STORAGE_CACHE, + 'Config' => APP_STORAGE_CONFIG, + 'Certs' => APP_STORAGE_CERTIFICATES + ] as $key => $volume + ) { + $device = new Local($volume); + + $percentage = (($device->getPartitionTotalSpace() - $device->getPartitionFreeSpace()) + / $device->getPartitionTotalSpace()) * 100; + + $message = $key . ' Volume has ' . Storage::human($device->getPartitionFreeSpace()) . ' free space (' . \round($percentage, 2) . '% used)'; + + if ($percentage < 80) { + Console::success('🟢 ' . $message); + } else { + Console::error('🔴 ' . $message); + } + } + + try { + if (App::isProduction()) { + Console::log(''); + $version = \json_decode(@\file_get_contents(App::getEnv('_APP_HOME', 'http://localhost') . '/v1/health/version'), true); + + if ($version && isset($version['version'])) { + if (\version_compare($version['version'], App::getEnv('_APP_VERSION', 'UNKNOWN')) === 0) { + Console::info('You are running the latest version of ' . APP_NAME . '! 🥳'); + } else { + Console::info('A new version (' . $version['version'] . ') is available! 🥳' . "\n"); + } + } else { + Console::error('Failed to check for a newer version' . "\n"); + } + } + } catch (\Throwable $th) { + Console::error('Failed to check for a newer version' . "\n"); + } + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Install.php b/src/Appwrite/Task/Install.php new file mode 100644 index 0000000000..8d6a7d4294 --- /dev/null +++ b/src/Appwrite/Task/Install.php @@ -0,0 +1,241 @@ +desc('Install Appwrite') + ->param('httpPort', '', new Text(4), 'Server HTTP port', true) + ->param('httpsPort', '', new Text(4), 'Server HTTPS port', true) + ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true) + ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true) + ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true) + ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive)); + } + + public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void + { + /** + * 1. Start - DONE + * 2. Check for older setup and get older version - DONE + * 2.1 If older version is equal or bigger(?) than current version, **stop setup** + * 2.2. Get ENV vars - DONE + * 2.2.1 Fetch from older docker-compose.yml file + * 2.2.2 Fetch from older .env file (manually parse) + * 2.3 Use old ENV vars as default values + * 2.4 Ask for all required vars not given as CLI args and if in interactive mode + * Otherwise, just use default vars. - DONE + * 3. Ask user to backup important volumes, env vars, and SQL tables + * In th future we can try and automate this for smaller/medium size setups + * 4. Drop new docker-compose.yml setup (located inside the container, no network dependencies with appwrite.io) - DONE + * 5. Run docker compose up -d - DONE + * 6. Run data migration + */ + $config = Config::getParam('variables'); + $path = '/usr/src/code/appwrite'; + $defaultHTTPPort = '80'; + $defaultHTTPSPort = '443'; + $vars = []; + + /** + * We are using a random value every execution for identification. + * This allows us to collect information without invading the privacy of our users. + */ + $analytics = new GoogleAnalytics('UA-26264668-9', uniqid('server.', true)); + + foreach ($config as $category) { + foreach ($category['variables'] ?? [] as $var) { + $vars[] = $var; + } + } + + Console::success('Starting Appwrite installation...'); + + // Create directory with write permissions + if (null !== $path && !\file_exists(\dirname($path))) { + if (!@\mkdir(\dirname($path), 0755, true)) { + Console::error('Can\'t create directory ' . \dirname($path)); + Console::exit(1); + } + } + + $data = @file_get_contents($path . '/docker-compose.yml'); + + if ($data !== false) { + $time = \time(); + Console::info('Compose file found, creating backup: docker-compose.yml.' . $time . '.backup'); + file_put_contents($path . '/docker-compose.yml.' . $time . '.backup', $data); + $compose = new Compose($data); + $appwrite = $compose->getService('appwrite'); + $oldVersion = ($appwrite) ? $appwrite->getImageVersion() : null; + try { + $ports = $compose->getService('traefik')->getPorts(); + } catch (\Throwable $th) { + $ports = [ + $defaultHTTPPort => $defaultHTTPPort, + $defaultHTTPSPort => $defaultHTTPSPort + ]; + Console::warning('Traefik not found. Falling back to default ports.'); + } + + if ($oldVersion) { + foreach ($compose->getServices() as $service) { // Fetch all env vars from previous compose file + if (!$service) { + continue; + } + + $env = $service->getEnvironment()->list(); + + foreach ($env as $key => $value) { + if (is_null($value)) { + continue; + } + foreach ($vars as &$var) { + if ($var['name'] === $key) { + $var['default'] = $value; + } + } + } + } + + $data = @file_get_contents($path . '/.env'); + + if ($data !== false) { // Fetch all env vars from previous .env file + Console::info('Env file found, creating backup: .env.' . $time . '.backup'); + file_put_contents($path . '/.env.' . $time . '.backup', $data); + $env = new Env($data); + + foreach ($env->list() as $key => $value) { + if (is_null($value)) { + continue; + } + foreach ($vars as &$var) { + if ($var['name'] === $key) { + $var['default'] = $value; + } + } + } + } + + foreach ($ports as $key => $value) { + if ($value === $defaultHTTPPort) { + $defaultHTTPPort = $key; + } + + if ($value === $defaultHTTPSPort) { + $defaultHTTPSPort = $key; + } + } + } + } + + if (empty($httpPort)) { + $httpPort = Console::confirm('Choose your server HTTP port: (default: ' . $defaultHTTPPort . ')'); + $httpPort = ($httpPort) ? $httpPort : $defaultHTTPPort; + } + + if (empty($httpsPort)) { + $httpsPort = Console::confirm('Choose your server HTTPS port: (default: ' . $defaultHTTPSPort . ')'); + $httpsPort = ($httpsPort) ? $httpsPort : $defaultHTTPSPort; + } + + $input = []; + + foreach ($vars as $key => $var) { + if (!empty($var['filter']) && ($interactive !== 'Y' || !Console::isInteractive())) { + if ($data && $var['default'] !== null) { + $input[$var['name']] = $var['default']; + continue; + } + + if ($var['filter'] === 'token') { + $input[$var['name']] = Auth::tokenGenerator(); + continue; + } + + if ($var['filter'] === 'password') { + $input[$var['name']] = Auth::passwordGenerator(); + continue; + } + } + if (!$var['required'] || !Console::isInteractive() || $interactive !== 'Y') { + $input[$var['name']] = $var['default']; + continue; + } + + $input[$var['name']] = Console::confirm($var['question'] . ' (default: \'' . $var['default'] . '\')'); + + if (empty($input[$var['name']])) { + $input[$var['name']] = $var['default']; + } + } + + $templateForCompose = new View(__DIR__ . '/../views/install/compose.phtml'); + $templateForEnv = new View(__DIR__ . '/../views/install/env.phtml'); + + $templateForCompose + ->setParam('httpPort', $httpPort) + ->setParam('httpsPort', $httpsPort) + ->setParam('version', APP_VERSION_STABLE) + ->setParam('organization', $organization) + ->setParam('image', $image) + ; + + $templateForEnv + ->setParam('vars', $input) + ; + + if (!file_put_contents($path . '/docker-compose.yml', $templateForCompose->render(false))) { + $message = 'Failed to save Docker Compose file'; + $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message); + Console::error($message); + Console::exit(1); + } + + if (!file_put_contents($path . '/.env', $templateForEnv->render(false))) { + $message = 'Failed to save environment variables file'; + $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message); + Console::error($message); + Console::exit(1); + } + + $env = ''; + $stdout = ''; + $stderr = ''; + + foreach ($input as $key => $value) { + if ($value) { + $env .= $key . '=' . \escapeshellarg($value) . ' '; + } + } + + Console::log("Running \"docker compose -f {$path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes\""); + + $exit = Console::execute("${env} docker compose -f {$path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr); + + if ($exit !== 0) { + $message = 'Failed to install Appwrite dockers'; + $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message); + Console::error($message); + Console::error($stderr); + Console::exit($exit); + } else { + $message = 'Appwrite installed successfully'; + $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message); + Console::success($message); + } + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Maintenance.php b/src/Appwrite/Task/Maintenance.php new file mode 100644 index 0000000000..3339438cfe --- /dev/null +++ b/src/Appwrite/Task/Maintenance.php @@ -0,0 +1,159 @@ +get('cache'))); + $database = new Database(new MariaDB($register->get('db')), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace('_console'); // Main DB + + if (!$database->exists($database->getDefaultDatabase(), 'certificates')) { + throw new \Exception('Console project not ready'); + } + + break; // leave loop if successful + } catch (\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep(DATABASE_RECONNECT_SLEEP); + } + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + + return $database; + } + + public function __construct() + { + $this + ->desc('Schedules maintenance tasks and publishes them to resque') + ->callback(fn () => $this->action()); + } + + public function action(): void + { + Console::title('Maintenance V1'); + Console::success(APP_NAME . ' maintenance process v1 has started'); + + function notifyDeleteExecutionLogs(int $interval) + { + (new Delete()) + ->setType(DELETE_TYPE_EXECUTIONS) + ->setTimestamp(time() - $interval) + ->trigger(); + } + + function notifyDeleteAbuseLogs(int $interval) + { + (new Delete()) + ->setType(DELETE_TYPE_ABUSE) + ->setTimestamp(time() - $interval) + ->trigger(); + } + + function notifyDeleteAuditLogs(int $interval) + { + (new Delete()) + ->setType(DELETE_TYPE_AUDIT) + ->setTimestamp(time() - $interval) + ->trigger(); + } + + function notifyDeleteUsageStats(int $interval30m, int $interval1d) + { + (new Delete()) + ->setType(DELETE_TYPE_USAGE) + ->setTimestamp1d(time() - $interval1d) + ->setTimestamp30m(time() - $interval30m) + ->trigger(); + } + + function notifyDeleteConnections() + { + (new Delete()) + ->setType(DELETE_TYPE_REALTIME) + ->setTimestamp(time() - 60) + ->trigger(); + } + + function notifyDeleteExpiredSessions() + { + (new Delete()) + ->setType(DELETE_TYPE_SESSIONS) + ->setTimestamp(time() - Auth::TOKEN_EXPIRATION_LOGIN_LONG) + ->trigger(); + } + + function renewCertificates($dbForConsole) + { + $time = date('d-m-Y H:i:s', time()); + $certificates = $dbForConsole->find('certificates', [ + new Query('attempts', Query::TYPE_LESSEREQUAL, [5]), // Maximum 5 attempts + new Query('renewDate', Query::TYPE_LESSEREQUAL, [\time()]) // includes 60 days cooldown (we have 30 days to renew) + ], 200); // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) + + + if (\count($certificates) > 0) { + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + + $event = new Certificate(); + foreach ($certificates as $certificate) { + $event + ->setDomain(new Document([ + 'domain' => $certificate->getAttribute('domain') + ])) + ->trigger(); + } + } else { + Console::info("[{$time}] No certificates for renewal."); + } + } + + // # of days in seconds (1 day = 86400s) + $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); + $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); + $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); + $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); + $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours + $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days + + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { + $database = $this->getConsoleDB(); + + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); + notifyDeleteExecutionLogs($executionLogsRetention); + notifyDeleteAbuseLogs($abuseLogsRetention); + notifyDeleteAuditLogs($auditLogRetention); + notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); + notifyDeleteConnections(); + notifyDeleteExpiredSessions(); + renewCertificates($database); + }, $interval); + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Migrate.php b/src/Appwrite/Task/Migrate.php new file mode 100644 index 0000000000..61cd757ca7 --- /dev/null +++ b/src/Appwrite/Task/Migrate.php @@ -0,0 +1,95 @@ +param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) + ->callback(fn ($version) => $this->action($version)); + } + + public function action($version) { + global $register; + Authorization::disable(); + if (!array_key_exists($version, Migration::$versions)) { + Console::error("Version {$version} not found."); + Console::exit(1); + return; + } + + $app = new App('UTC'); + + Console::success('Starting Data Migration to version ' . $version); + + $db = $register->get('db', true); + $redis = $register->get('cache', true); + $redis->flushAll(); + $cache = new Cache(new RedisCache($redis)); + + $projectDB = new Database(new MariaDB($db), $cache); + $projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + + $consoleDB = new Database(new MariaDB($db), $cache); + $consoleDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $consoleDB->setNamespace('_project_console'); + + $console = $app->getResource('console'); + + $limit = 30; + $sum = 30; + $offset = 0; + $projects = [$console]; + $count = 0; + + try { + $totalProjects = $consoleDB->count('projects') + 1; + } catch (\Throwable $th) { + $consoleDB->setNamespace('_console'); + $totalProjects = $consoleDB->count('projects') + 1; + } + + $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + $migration = new $class(); + + while (!empty($projects)) { + foreach ($projects as $project) { + try { + $migration + ->setProject($project, $projectDB, $consoleDB) + ->execute(); + } catch (\Throwable $th) { + throw $th; + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } + } + + $sum = \count($projects); + $projects = $consoleDB->find('projects', limit: $limit, offset: $offset); + + $offset = $offset + $limit; + $count = $count + $sum; + + Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); + } + + Event::wait(); // Wait for Coroutines to finish + $redis->flushAll(); + Console::success('Data Migration Completed'); + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/SDKs.php b/src/Appwrite/Task/SDKs.php new file mode 100644 index 0000000000..b5f1506919 --- /dev/null +++ b/src/Appwrite/Task/SDKs.php @@ -0,0 +1,271 @@ +callback(fn () => $this->action()); + } + + public function action(): void + { + $platforms = Config::getParam('platforms'); + $selected = \strtolower(Console::confirm('Choose SDK ("*" for all):')); + $version = Console::confirm('Choose an Appwrite version'); + $git = (Console::confirm('Should we use git push? (yes/no)') == 'yes'); + $production = ($git) ? (Console::confirm('Type "Appwrite" to push code to production git repos') == 'Appwrite') : false; + $message = ($git) ? Console::confirm('Please enter your commit message:') : ''; + + if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', 'latest'])) { + throw new Exception('Unknown version given'); + } + + foreach ($platforms as $key => $platform) { + foreach ($platform['languages'] as $language) { + if ($selected !== $language['key'] && $selected !== '*') { + continue; + } + + if (!$language['enabled']) { + Console::warning($language['name'] . ' for ' . $platform['name'] . ' is disabled'); + continue; + } + + Console::info('Fetching API Spec for ' . $language['name'] . ' for ' . $platform['name'] . ' (version: ' . $version . ')'); + + $spec = file_get_contents(__DIR__ . '/../config/specs/swagger2-' . $version . '-' . $language['family'] . '.json'); + + $cover = 'https://appwrite.io/images/github.png'; + $result = \realpath(__DIR__ . '/..') . '/sdks/' . $key . '-' . $language['key']; + $resultExamples = \realpath(__DIR__ . '/../..') . '/docs/examples/' . $version . '/' . $key . '-' . $language['key']; + $target = \realpath(__DIR__ . '/..') . '/sdks/git/' . $language['key'] . '/'; + $readme = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/README.md'); + $readme = ($readme) ? \file_get_contents($readme) : ''; + $gettingStarted = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); + $gettingStarted = ($gettingStarted) ? \file_get_contents($gettingStarted) : ''; + $examples = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/EXAMPLES.md'); + $examples = ($examples) ? \file_get_contents($examples) : ''; + $changelog = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/CHANGELOG.md'); + $changelog = ($changelog) ? \file_get_contents($changelog) : '# Change Log'; + $warning = '**This SDK is compatible with Appwrite server version ' . $version . '. For older versions, please check [previous releases](' . $language['url'] . '/releases).**'; + $license = 'BSD-3-Clause'; + $licenseContent = 'Copyright (c) ' . date('Y') . ' Appwrite (https://appwrite.io) and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + + 3. Neither the name Appwrite nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.'; + + switch ($language['key']) { + case 'web': + $config = new Web(); + $config->setNPMPackage('appwrite'); + $config->setBowerPackage('appwrite'); + break; + case 'cli': + $config = new CLI(); + $config->setNPMPackage('appwrite-cli'); + $config->setExecutableName('appwrite'); + $config->setLogo(json_encode(" + _ _ _ ___ __ _____ + /_\ _ __ _ ____ ___ __(_) |_ ___ / __\ / / \_ \ + //_\\\| '_ \| '_ \ \ /\ / / '__| | __/ _ \ / / / / / /\/ + / _ \ |_) | |_) \ V V /| | | | || __/ / /___/ /___/\/ /_ + \_/ \_/ .__/| .__/ \_/\_/ |_| |_|\__\___| \____/\____/\____/ + |_| |_| + +")); + $config->setLogoUnescaped(" + _ _ _ ___ __ _____ + /_\ _ __ _ ____ ___ __(_) |_ ___ / __\ / / \_ \ + //_\\\| '_ \| '_ \ \ /\ / / '__| | __/ _ \ / / / / / /\/ + / _ \ |_) | |_) \ V V /| | | | || __/ / /___/ /___/\/ /_ + \_/ \_/ .__/| .__/ \_/\_/ |_| |_|\__\___| \____/\____/\____/ + |_| |_| "); + break; + case 'php': + $config = new PHP(); + $config->setComposerVendor('appwrite'); + $config->setComposerPackage('appwrite'); + break; + case 'nodejs': + $config = new Node(); + $config->setNPMPackage('node-appwrite'); + $config->setBowerPackage('appwrite'); + $warning = $warning . "\n\n > This is the Node.js SDK for integrating with Appwrite from your Node.js server-side code. + If you're looking to integrate from the browser, you should check [appwrite/sdk-for-web](https://github.com/appwrite/sdk-for-web)"; + break; + case 'deno': + $config = new Deno(); + break; + case 'python': + $config = new Python(); + $config->setPipPackage('appwrite'); + $license = 'BSD License'; // license edited due to classifiers in pypi + break; + case 'ruby': + $config = new Ruby(); + $config->setGemPackage('appwrite'); + break; + case 'flutter': + $config = new Flutter(); + $config->setPackageName('appwrite'); + break; + case 'flutter-dev': + $config = new Flutter(); + $config->setPackageName('appwrite_dev'); + break; + case 'dart': + $config = new Dart(); + $config->setPackageName('dart_appwrite'); + $warning = $warning . "\n\n > This is the Dart SDK for integrating with Appwrite from your Dart server-side code. If you're looking for the Flutter SDK you should check [appwrite/sdk-for-flutter](https://github.com/appwrite/sdk-for-flutter)"; + break; + case 'go': + $config = new Go(); + break; + case 'swift': + $config = new Swift(); + $warning = $warning . "\n\n > This is the Swift SDK for integrating with Appwrite from your Swift server-side code. If you're looking for the Apple SDK you should check [appwrite/sdk-for-apple](https://github.com/appwrite/sdk-for-apple)"; + break; + case 'apple': + $config = new SwiftClient(); + break; + case 'dotnet': + $cover = ''; + $config = new DotNet(); + break; + case 'android': + $config = new Android(); + break; + case 'kotlin': + $config = new Kotlin(); + $warning = $warning . "\n\n > This is the Kotlin SDK for integrating with Appwrite from your Kotlin server-side code. If you're looking for the Android SDK you should check [appwrite/sdk-for-android](https://github.com/appwrite/sdk-for-android)"; + break; + default: + throw new Exception('Language "' . $language['key'] . '" not supported'); + break; + } + + Console::info("Generating {$language['name']} SDK..."); + + $sdk = new SDK($config, new Swagger2($spec)); + + $sdk + ->setName($language['name']) + ->setNamespace('io appwrite') + ->setDescription("Appwrite is an open-source backend as a service server that abstract and simplify complex and repetitive development tasks behind a very simple to use REST API. Appwrite aims to help you develop your apps faster and in a more secure way. Use the {$language['name']} SDK to integrate your app with the Appwrite server to easily start interacting with all of Appwrite backend APIs and tools. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)") + ->setShortDescription('Appwrite is an open-source self-hosted backend server that abstract and simplify complex and repetitive development tasks behind a very simple REST API') + ->setLicense($license) + ->setLicenseContent($licenseContent) + ->setVersion($language['version']) + ->setGitURL($language['url']) + ->setGitRepo($language['gitUrl']) + ->setGitRepoName($language['gitRepoName']) + ->setGitUserName($language['gitUserName']) + ->setLogo($cover) + ->setURL('https://appwrite.io') + ->setShareText('Appwrite is a backend as a service for building web or mobile apps') + ->setShareURL('http://appwrite.io') + ->setShareTags('JS,javascript,reactjs,angular,ios,android,serverless') + ->setShareVia('appwrite') + ->setWarning($warning) + ->setReadme($readme) + ->setGettingStarted($gettingStarted) + ->setChangelog($changelog) + ->setExamples($examples) + ->setTwitter(APP_SOCIAL_TWITTER_HANDLE) + ->setDiscord(APP_SOCIAL_DISCORD_CHANNEL, APP_SOCIAL_DISCORD) + ->setDefaultHeaders([ + 'X-Appwrite-Response-Format' => '0.15.0', + ]); + + try { + $sdk->generate($result); + } catch (Exception $exception) { + Console::error($exception->getMessage()); + } catch (Throwable $exception) { + Console::error($exception->getMessage()); + } + + $gitUrl = $language['gitUrl']; + $gitBranch = $language['gitBranch']; + + + if (!$production) { + $gitUrl = 'git@github.com:aw-tests/' . $language['gitRepoName'] . '.git'; + } + + if ($git && !empty($gitUrl)) { + \exec('rm -rf ' . $target . ' && \ + mkdir -p ' . $target . ' && \ + cd ' . $target . ' && \ + git init --initial-branch=' . $gitBranch . ' && \ + git remote add origin ' . $gitUrl . ' && \ + git fetch && \ + git pull ' . $gitUrl . ' && \ + rm -rf ' . $target . '/* && \ + cp -r ' . $result . '/* ' . $target . '/ && \ + git add . && \ + git commit -m "' . $message . '" && \ + git push -u origin ' . $gitBranch . ' + '); + + Console::success("Pushed {$language['name']} SDK to {$gitUrl}"); + + \exec('rm -rf ' . $target); + Console::success("Remove temp directory '{$target}' for {$language['name']} SDK"); + } + + $docDirectories = $language['docDirectories'] ?? ['']; + + if ($version === 'latest') { + continue; + } + + foreach ($docDirectories as $languageTitle => $path) { + $languagePath = strtolower($languageTitle !== 0 ? '/' . $languageTitle : ''); + \exec( + 'mkdir -p ' . $resultExamples . $languagePath . ' && \ + cp -r ' . $result . '/docs/examples' . $languagePath . ' ' . $resultExamples + ); + Console::success("Copied code examples for {$language['name']} SDK to: {$resultExamples}"); + } + } + } + + Console::exit(); + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/SSL.php b/src/Appwrite/Task/SSL.php new file mode 100644 index 0000000000..f4688c611e --- /dev/null +++ b/src/Appwrite/Task/SSL.php @@ -0,0 +1,34 @@ +desc('Validate server certificates') + ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) + ->callback(fn ($domain) => $this->action($domain)); + } + + public function action(string $domain): void + { + Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain); + + (new Certificate()) + ->setDomain(new Document([ + 'domain' => $domain + ])) + ->setSkipRenewCheck(true) + ->trigger(); + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Specs.php b/src/Appwrite/Task/Specs.php new file mode 100644 index 0000000000..3812f9302a --- /dev/null +++ b/src/Appwrite/Task/Specs.php @@ -0,0 +1,269 @@ +param('version', 'latest', new Text(8), 'Spec version', true) + ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true) + ->callback(fn ($version, $mode) => $this->action($version, $mode)); + } + + public function action(string $version, string $mode): void + { + global $register; + + $db = $register->get('db'); + $redis = $register->get('cache'); + $appRoutes = App::getRoutes(); + $response = new Response(new HttpResponse()); + $mocks = ($mode === 'mocks'); + + App::setResource('request', fn () => new Request()); + App::setResource('response', fn () => $response); + App::setResource('db', fn () => $db); + App::setResource('cache', fn () => $redis); + + $platforms = [ + 'client' => APP_PLATFORM_CLIENT, + 'server' => APP_PLATFORM_SERVER, + 'console' => APP_PLATFORM_CONSOLE, + ]; + + $authCounts = [ + 'client' => 1, + 'server' => 2, + 'console' => 1, + ]; + + $keys = [ + APP_PLATFORM_CLIENT => [ + 'Project' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-Project', + 'description' => 'Your project ID', + 'in' => 'header', + ], + 'JWT' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-JWT', + 'description' => 'Your secret JSON Web Token', + 'in' => 'header', + ], + 'Locale' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-Locale', + 'description' => '', + 'in' => 'header', + ], + ], + APP_PLATFORM_SERVER => [ + 'Project' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-Project', + 'description' => 'Your project ID', + 'in' => 'header', + ], + 'Key' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-Key', + 'description' => 'Your secret API key', + 'in' => 'header', + ], + 'JWT' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-JWT', + 'description' => 'Your secret JSON Web Token', + 'in' => 'header', + ], + 'Locale' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-Locale', + 'description' => '', + 'in' => 'header', + ], + ], + APP_PLATFORM_CONSOLE => [ + 'Project' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-Project', + 'description' => 'Your project ID', + 'in' => 'header', + ], + 'Key' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-Key', + 'description' => 'Your secret API key', + 'in' => 'header', + ], + 'JWT' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-JWT', + 'description' => 'Your secret JSON Web Token', + 'in' => 'header', + ], + 'Locale' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-Locale', + 'description' => '', + 'in' => 'header', + ], + 'Mode' => [ + 'type' => 'apiKey', + 'name' => 'X-Appwrite-Mode', + 'description' => '', + 'in' => 'header', + ], + ], + ]; + + foreach ($platforms as $platform) { + $routes = []; + $models = []; + $services = []; + + foreach ($appRoutes as $key => $method) { + foreach ($method as $route) { + /** @var \Utopia\Route $route */ + $routeSecurity = $route->getLabel('sdk.auth', []); + $sdkPlaforms = []; + + foreach ($routeSecurity as $value) { + switch ($value) { + case APP_AUTH_TYPE_SESSION: + $sdkPlaforms[] = APP_PLATFORM_CLIENT; + break; + case APP_AUTH_TYPE_KEY: + $sdkPlaforms[] = APP_PLATFORM_SERVER; + break; + case APP_AUTH_TYPE_JWT: + $sdkPlaforms[] = APP_PLATFORM_SERVER; + break; + case APP_AUTH_TYPE_ADMIN: + $sdkPlaforms[] = APP_PLATFORM_CONSOLE; + break; + } + } + + if (empty($routeSecurity)) { + $sdkPlaforms[] = APP_PLATFORM_CLIENT; + } + + if (!$route->getLabel('docs', true)) { + continue; + } + + if ($route->getLabel('sdk.mock', false) && !$mocks) { + continue; + } + + if (!$route->getLabel('sdk.mock', false) && $mocks) { + continue; + } + + if (empty($route->getLabel('sdk.namespace', null))) { + continue; + } + + if ($platform !== APP_PLATFORM_CONSOLE && !\in_array($platforms[$platform], $sdkPlaforms)) { + continue; + } + + $routes[] = $route; + } + } + + foreach (Config::getParam('services', []) as $service) { + if ( + !isset($service['docs']) // Skip service if not part of the public API + || !isset($service['sdk']) + || !$service['docs'] + || !$service['sdk'] + ) { + continue; + } + + $services[] = [ + 'name' => $service['key'] ?? '', + 'description' => $service['subtitle'] ?? '', + 'x-globalAttributes' => $service['globalAttributes'] ?? [], + ]; + } + + $models = $response->getModels(); + + foreach ($models as $key => $value) { + if ($platform !== APP_PLATFORM_CONSOLE && !$value->isPublic()) { + unset($models[$key]); + } + } + // var_dump($models); + $arguments = [new App('UTC'), $services, $routes, $models, $keys[$platform], $authCounts[$platform] ?? 0]; + foreach (['swagger2', 'open-api3'] as $format) { + $formatInstance = match ($format) { + 'swagger2' => new Swagger2(...$arguments), + 'open-api3' => new OpenAPI3(...$arguments), + default => throw new Exception('Format not found: ' . $format) + }; + + $specs = new Specification($formatInstance); + $endpoint = App::getEnv('_APP_HOME', '[HOSTNAME]'); + $email = App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); + + $formatInstance + ->setParam('name', APP_NAME) + ->setParam('description', 'Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)') + ->setParam('endpoint', 'https://HOSTNAME/v1') + ->setParam('version', APP_VERSION_STABLE) + ->setParam('terms', $endpoint . '/policy/terms') + ->setParam('support.email', $email) + ->setParam('support.url', $endpoint . '/support') + ->setParam('contact.name', APP_NAME . ' Team') + ->setParam('contact.email', $email) + ->setParam('contact.url', $endpoint . '/support') + ->setParam('license.name', 'BSD-3-Clause') + ->setParam('license.url', 'https://raw.githubusercontent.com/appwrite/appwrite/master/LICENSE') + ->setParam('docs.description', 'Full API docs, specs and tutorials') + ->setParam('docs.url', $endpoint . '/docs'); + + if ($mocks) { + $path = __DIR__ . '/../config/specs/' . $format . '-mocks-' . $platform . '.json'; + + if (!file_put_contents($path, json_encode($specs->parse()))) { + throw new Exception('Failed to save mocks spec file: ' . $path); + } + + Console::success('Saved mocks spec file: ' . realpath($path)); + + continue; + } + + $path = __DIR__ . '/../config/specs/' . $format . '-' . $version . '-' . $platform . '.json'; + + if (!file_put_contents($path, json_encode($specs->parse()))) { + throw new Exception('Failed to save spec file: ' . $path); + } + + Console::success('Saved spec file: ' . realpath($path)); + } + } + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Tasks.php b/src/Appwrite/Task/Tasks.php index a7d2a7527c..3bacb03857 100644 --- a/src/Appwrite/Task/Tasks.php +++ b/src/Appwrite/Task/Tasks.php @@ -7,8 +7,16 @@ class Tasks extends Service { public function __construct() { $this->type = self::TYPE_CLI; - $this->addAction('version', new Version()); - $this->addAction('usage', new Usage()); - $this->addAction('vars', new Vars()); + $this + ->addAction(Version::NAME, new Version()) + ->addAction(Usage::NAME, new Usage()) + ->addAction(Vars::NAME, new Vars()) + ->addAction(SSL::NAME, new SSL()) + ->addAction(Doctor::NAME, new Doctor()) + ->addAction(Install::NAME, new Install()) + ->addAction(Maintenance::NAME, new Maintenance()) + ->addAction(Migrate::NAME, new Migrate()) + ->addAction(SDKs::NAME, new SDKs()) + ->addAction(Specs::NAME, new Specs()); } } diff --git a/src/Appwrite/Task/Usage.php b/src/Appwrite/Task/Usage.php index 9dd855e2f3..a9f25a701a 100644 --- a/src/Appwrite/Task/Usage.php +++ b/src/Appwrite/Task/Usage.php @@ -18,6 +18,8 @@ use Utopia\Logger\Log; use Utopia\Platform\Action; class Usage extends Action{ + public const NAME = 'usage'; + protected function getDatabase(Registry &$register, string $namespace): Database { $attempts = 0; diff --git a/src/Appwrite/Task/Vars.php b/src/Appwrite/Task/Vars.php index fc50e2b114..818563f362 100644 --- a/src/Appwrite/Task/Vars.php +++ b/src/Appwrite/Task/Vars.php @@ -6,6 +6,8 @@ use Utopia\CLI\Console; use Utopia\Platform\Action; class Vars extends Action{ + + public const NAME = 'vars'; public function __construct() { diff --git a/src/Appwrite/Task/Version.php b/src/Appwrite/Task/Version.php index 07ef47f292..104c675a58 100644 --- a/src/Appwrite/Task/Version.php +++ b/src/Appwrite/Task/Version.php @@ -6,6 +6,8 @@ use Utopia\CLI\Console; use Utopia\Platform\Action; class Version extends Action { + public const NAME = 'version'; + public function __construct() { $this From 031c8a5594e239c467dd6b06b0ae0d5b7e5b8920 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 13 Jul 2022 07:16:28 +0000 Subject: [PATCH 05/50] fix spec and sdks tasks --- src/Appwrite/Task/SDKs.php | 16 ++++++++-------- src/Appwrite/Task/Specs.php | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Task/SDKs.php b/src/Appwrite/Task/SDKs.php index b5f1506919..9b89dfc7eb 100644 --- a/src/Appwrite/Task/SDKs.php +++ b/src/Appwrite/Task/SDKs.php @@ -60,19 +60,19 @@ class SDKs extends Action{ Console::info('Fetching API Spec for ' . $language['name'] . ' for ' . $platform['name'] . ' (version: ' . $version . ')'); - $spec = file_get_contents(__DIR__ . '/../config/specs/swagger2-' . $version . '-' . $language['family'] . '.json'); + $spec = file_get_contents(__DIR__ . '/../../../app/config/specs/swagger2-' . $version . '-' . $language['family'] . '.json'); $cover = 'https://appwrite.io/images/github.png'; - $result = \realpath(__DIR__ . '/..') . '/sdks/' . $key . '-' . $language['key']; - $resultExamples = \realpath(__DIR__ . '/../..') . '/docs/examples/' . $version . '/' . $key . '-' . $language['key']; - $target = \realpath(__DIR__ . '/..') . '/sdks/git/' . $language['key'] . '/'; - $readme = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/README.md'); + $result = \realpath(__DIR__ . '/../../../app') . '/sdks/' . $key . '-' . $language['key']; + $resultExamples = \realpath(__DIR__ . '/../../..') . '/docs/examples/' . $version . '/' . $key . '-' . $language['key']; + $target = \realpath(__DIR__ . '/../../../app') . '/sdks/git/' . $language['key'] . '/'; + $readme = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/README.md'); $readme = ($readme) ? \file_get_contents($readme) : ''; - $gettingStarted = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); + $gettingStarted = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); $gettingStarted = ($gettingStarted) ? \file_get_contents($gettingStarted) : ''; - $examples = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/EXAMPLES.md'); + $examples = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/EXAMPLES.md'); $examples = ($examples) ? \file_get_contents($examples) : ''; - $changelog = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/CHANGELOG.md'); + $changelog = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/CHANGELOG.md'); $changelog = ($changelog) ? \file_get_contents($changelog) : '# Change Log'; $warning = '**This SDK is compatible with Appwrite server version ' . $version . '. For older versions, please check [previous releases](' . $language['url'] . '/releases).**'; $license = 'BSD-3-Clause'; diff --git a/src/Appwrite/Task/Specs.php b/src/Appwrite/Task/Specs.php index 3812f9302a..a79f434320 100644 --- a/src/Appwrite/Task/Specs.php +++ b/src/Appwrite/Task/Specs.php @@ -256,7 +256,7 @@ class Specs extends Action{ continue; } - $path = __DIR__ . '/../config/specs/' . $format . '-' . $version . '-' . $platform . '.json'; + $path = __DIR__ . '/../../../app/config/specs/' . $format . '-' . $version . '-' . $platform . '.json'; if (!file_put_contents($path, json_encode($specs->parse()))) { throw new Exception('Failed to save spec file: ' . $path); From 082967095eee3884ced6c924e5621a8449abb1f8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 14 Jul 2022 02:04:31 +0000 Subject: [PATCH 06/50] linter fixes --- app/cli.php | 1 - src/Appwrite/Task/CLIPlatform.php | 5 ++-- src/Appwrite/Task/Doctor.php | 5 ++-- src/Appwrite/Task/Install.php | 8 +++--- src/Appwrite/Task/Maintenance.php | 18 +++++++------ src/Appwrite/Task/Migrate.php | 12 +++++---- src/Appwrite/Task/SDKs.php | 9 ++++--- src/Appwrite/Task/SSL.php | 9 ++++--- src/Appwrite/Task/Specs.php | 9 ++++--- src/Appwrite/Task/Tasks.php | 4 ++- src/Appwrite/Task/Usage.php | 45 ++++++++++++++++--------------- src/Appwrite/Task/Vars.php | 10 ++++--- src/Appwrite/Task/Version.php | 4 ++- 13 files changed, 79 insertions(+), 60 deletions(-) diff --git a/app/cli.php b/app/cli.php index 413fa234ea..66c4208e14 100644 --- a/app/cli.php +++ b/app/cli.php @@ -13,4 +13,3 @@ $cliPlatform->init('CLI'); $cli = $cliPlatform->getCli(); $cli->run(); - diff --git a/src/Appwrite/Task/CLIPlatform.php b/src/Appwrite/Task/CLIPlatform.php index 847071fd63..063c6b6ca8 100644 --- a/src/Appwrite/Task/CLIPlatform.php +++ b/src/Appwrite/Task/CLIPlatform.php @@ -4,9 +4,10 @@ namespace Appwrite\Task; use Utopia\Platform\Platform; -class CLIPlatform extends Platform { +class CLIPlatform extends Platform +{ public function __construct() { $this->addService('cliTasks', new Tasks()); } -} \ No newline at end of file +} diff --git a/src/Appwrite/Task/Doctor.php b/src/Appwrite/Task/Doctor.php index 529116813b..281efe4d35 100644 --- a/src/Appwrite/Task/Doctor.php +++ b/src/Appwrite/Task/Doctor.php @@ -11,7 +11,8 @@ use Utopia\Storage\Storage; use Utopia\Domains\Domain; use Utopia\Platform\Action; -class Doctor extends Action{ +class Doctor extends Action +{ public const NAME = 'doctor'; public function __construct() @@ -252,4 +253,4 @@ class Doctor extends Action{ Console::error('Failed to check for a newer version' . "\n"); } } -} \ No newline at end of file +} diff --git a/src/Appwrite/Task/Install.php b/src/Appwrite/Task/Install.php index 8d6a7d4294..fe3fb826d4 100644 --- a/src/Appwrite/Task/Install.php +++ b/src/Appwrite/Task/Install.php @@ -1,4 +1,5 @@ get('db')), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $database->setNamespace('_console'); // Main DB - + if (!$database->exists($database->getDefaultDatabase(), 'certificates')) { throw new \Exception('Console project not ready'); } - + break; // leave loop if successful } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); @@ -44,10 +46,10 @@ class Maintenance extends Action { sleep(DATABASE_RECONNECT_SLEEP); } } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - + return $database; } - + public function __construct() { $this @@ -156,4 +158,4 @@ class Maintenance extends Action { renewCertificates($database); }, $interval); } -} \ No newline at end of file +} diff --git a/src/Appwrite/Task/Migrate.php b/src/Appwrite/Task/Migrate.php index 61cd757ca7..d1c154a0df 100644 --- a/src/Appwrite/Task/Migrate.php +++ b/src/Appwrite/Task/Migrate.php @@ -1,4 +1,5 @@ callback(fn ($version) => $this->action($version)); } - public function action($version) { + public function action($version) + { global $register; Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -92,4 +94,4 @@ class Migrate extends Action{ $redis->flushAll(); Console::success('Data Migration Completed'); } -} \ No newline at end of file +} diff --git a/src/Appwrite/Task/SDKs.php b/src/Appwrite/Task/SDKs.php index 9b89dfc7eb..0bbe10bbc9 100644 --- a/src/Appwrite/Task/SDKs.php +++ b/src/Appwrite/Task/SDKs.php @@ -1,4 +1,5 @@ setSkipRenewCheck(true) ->trigger(); } -} \ No newline at end of file +} diff --git a/src/Appwrite/Task/Specs.php b/src/Appwrite/Task/Specs.php index a79f434320..d2e0f73f3c 100644 --- a/src/Appwrite/Task/Specs.php +++ b/src/Appwrite/Task/Specs.php @@ -1,4 +1,5 @@ type = self::TYPE_CLI; diff --git a/src/Appwrite/Task/Usage.php b/src/Appwrite/Task/Usage.php index a9f25a701a..f2f0b73f76 100644 --- a/src/Appwrite/Task/Usage.php +++ b/src/Appwrite/Task/Usage.php @@ -1,4 +1,5 @@ get('db'); $redis = $register->get('cache'); - + $cache = new Cache(new RedisCache($redis)); $database = new Database(new MariaDB($db), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $database->setNamespace($namespace); - + if (!$database->exists($database->getDefaultDatabase(), 'projects')) { throw new Exception('Projects collection not ready'); } @@ -48,10 +50,10 @@ class Usage extends Action{ sleep(DATABASE_RECONNECT_SLEEP); } } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - + return $database; } - + protected function getInfluxDB(Registry &$register): InfluxDatabase { /** @var InfluxDB\Client $client */ @@ -59,7 +61,7 @@ class Usage extends Action{ $attempts = 0; $max = 10; $sleep = 1; - + do { // check if telegraf database is ready try { $attempts++; @@ -77,17 +79,18 @@ class Usage extends Action{ } while ($attempts < $max); return $database; } - + public function __construct() { - - + + $this ->desc('Schedules syncing data from influxdb to Appwrite console db') ->callback(fn () => $this->action()); } - public function action() { + public function action() + { global $register; @@ -96,34 +99,34 @@ class Usage extends Action{ $logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) { $logger = $register->get('logger'); - + if ($logger) { $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); - + $log = new Log(); $log->setNamespace("usage"); $log->setServer(\gethostname()); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); - + $log->addTag('code', $error->getCode()); $log->addTag('verboseType', get_class($error)); - + $log->addExtra('file', $error->getFile()); $log->addExtra('line', $error->getLine()); $log->addExtra('trace', $error->getTraceAsString()); $log->addExtra('detailedTrace', $error->getTrace()); - + $log->setAction($action); - + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - + $responseCode = $logger->addLog($log); Console::info('Usage stats log pushed with status code: ' . $responseCode); } - + Console::warning("Failed: {$error->getMessage()}"); Console::warning($error->getTraceAsString()); }; @@ -176,4 +179,4 @@ class Usage extends Action{ Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); } -} \ No newline at end of file +} diff --git a/src/Appwrite/Task/Vars.php b/src/Appwrite/Task/Vars.php index 818563f362..cd9ff4de6f 100644 --- a/src/Appwrite/Task/Vars.php +++ b/src/Appwrite/Task/Vars.php @@ -1,14 +1,16 @@ Date: Thu, 14 Jul 2022 02:20:29 +0000 Subject: [PATCH 07/50] composer update --- composer.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/composer.lock b/composer.lock index 1994e95967..1854fe86be 100644 --- a/composer.lock +++ b/composer.lock @@ -2446,7 +2446,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "370565fcc39996506357338e28cad97cb9e3db0f" + "reference": "7a4c9b48b3ad06fb850cf8394c671603223bc872" }, "require": { "ext-json": "*", @@ -2495,7 +2495,7 @@ "upf", "utopia" ], - "time": "2022-07-13T05:59:26+00:00" + "time": "2022-07-14T02:00:12+00:00" }, { "name": "utopia-php/preloader", @@ -5437,5 +5437,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } From 57518c69500e43472f1959fb99eb93cd5a41ac53 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 15 Jul 2022 05:02:39 +0000 Subject: [PATCH 08/50] updated implementation --- composer.lock | 6 +- src/Appwrite/Platform/Action.php | 110 +++++++++++++++++++++++++++++++ src/Appwrite/Task/Usage.php | 104 +---------------------------- 3 files changed, 115 insertions(+), 105 deletions(-) create mode 100644 src/Appwrite/Platform/Action.php diff --git a/composer.lock b/composer.lock index 1854fe86be..bf8963e887 100644 --- a/composer.lock +++ b/composer.lock @@ -2446,7 +2446,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "7a4c9b48b3ad06fb850cf8394c671603223bc872" + "reference": "4327c4cbaafc39cd94ec4d977cbdc25c8e9df613" }, "require": { "ext-json": "*", @@ -2495,7 +2495,7 @@ "upf", "utopia" ], - "time": "2022-07-14T02:00:12+00:00" + "time": "2022-07-14T07:30:15+00:00" }, { "name": "utopia-php/preloader", @@ -5437,5 +5437,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Platform/Action.php b/src/Appwrite/Platform/Action.php new file mode 100644 index 0000000000..314d7ede20 --- /dev/null +++ b/src/Appwrite/Platform/Action.php @@ -0,0 +1,110 @@ +get('logger'); + + if ($logger) { + $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace("usage"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + + $log->setAction($action); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $responseCode = $logger->addLog($log); + Console::info('Usage stats log pushed with status code: ' . $responseCode); + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); + } + + protected function getDatabase(Registry &$register, string $namespace): Database + { + $attempts = 0; + + do { + try { + $attempts++; + + $db = $register->get('db'); + $redis = $register->get('cache'); + + $cache = new Cache(new RedisCache($redis)); + $database = new Database(new MariaDB($db), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace($namespace); + + if (!$database->exists($database->getDefaultDatabase(), 'projects')) { + throw new Exception('Projects collection not ready'); + } + break; // leave loop if successful + } catch (\Exception$e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep(DATABASE_RECONNECT_SLEEP); + } + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + + return $database; + } + + protected function getInfluxDB(Registry &$register): InfluxDatabase + { + /** @var InfluxDB\Client $client */ + $client = $register->get('influxdb'); + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if (in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful + } + } catch (\Throwable$th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } + } while ($attempts < $max); + return $database; + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Usage.php b/src/Appwrite/Task/Usage.php index f2f0b73f76..2093f32720 100644 --- a/src/Appwrite/Task/Usage.php +++ b/src/Appwrite/Task/Usage.php @@ -2,88 +2,21 @@ namespace Appwrite\Task; +use Appwrite\Platform\Action; use Throwable; use Exception; use Appwrite\Stats\Usage as InfluxUsage; use Appwrite\Stats\UsageDB; -use InfluxDB\Database as InfluxDatabase; use Utopia\App; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; use Utopia\CLI\Console; -use Utopia\Database\Adapter\MariaDB; -use Utopia\Database\Database; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; -use Utopia\Logger\Log; -use Utopia\Platform\Action; class Usage extends Action { public const NAME = 'usage'; - protected function getDatabase(Registry &$register, string $namespace): Database - { - $attempts = 0; - - do { - try { - $attempts++; - - $db = $register->get('db'); - $redis = $register->get('cache'); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); - - if (!$database->exists($database->getDefaultDatabase(), 'projects')) { - throw new Exception('Projects collection not ready'); - } - break; // leave loop if successful - } catch (\Exception$e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; - } - - protected function getInfluxDB(Registry &$register): InfluxDatabase - { - /** @var InfluxDB\Client $client */ - $client = $register->get('influxdb'); - $attempts = 0; - $max = 10; - $sleep = 1; - - do { // check if telegraf database is ready - try { - $attempts++; - $database = $client->selectDB('telegraf'); - if (in_array('telegraf', $client->listDatabases())) { - break; // leave the do-while if successful - } - } catch (\Throwable$th) { - Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('InfluxDB database not ready yet'); - } - sleep($sleep); - } - } while ($attempts < $max); - return $database; - } - public function __construct() { - - $this ->desc('Schedules syncing data from influxdb to Appwrite console db') ->callback(fn () => $this->action()); @@ -91,45 +24,12 @@ class Usage extends Action public function action() { - global $register; Authorization::disable(); Authorization::setDefaultStatus(false); - $logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) { - $logger = $register->get('logger'); - - if ($logger) { - $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); - - $log = new Log(); - $log->setNamespace("usage"); - $log->setServer(\gethostname()); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - - $log->addTag('code', $error->getCode()); - $log->addTag('verboseType', get_class($error)); - - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('detailedTrace', $error->getTrace()); - - $log->setAction($action); - - $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - $responseCode = $logger->addLog($log); - Console::info('Usage stats log pushed with status code: ' . $responseCode); - } - - Console::warning("Failed: {$error->getMessage()}"); - Console::warning($error->getTraceAsString()); - }; + $logError = fn(Throwable $error, string $action = 'syncUsageStats') => $this->logError($register, $error, $action); Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); From 69feb7138a9f5eca0f9e3441ab15f2f35e6b219e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 17 Jul 2022 01:29:45 +0000 Subject: [PATCH 09/50] update platform fix --- composer.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.lock b/composer.lock index 7f26a6e167..bb06931cd7 100644 --- a/composer.lock +++ b/composer.lock @@ -2446,7 +2446,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "fd133dd0f4d464c3aa6b4e5c338f7fc7201f8e4d" + "reference": "c187abd39d134287c1d71585083e4e99bf63ebb3" }, "require": { "ext-json": "*", @@ -2498,7 +2498,7 @@ "upf", "utopia" ], - "time": "2022-07-17T01:05:38+00:00" + "time": "2022-07-17T01:28:06+00:00" }, { "name": "utopia-php/preloader", From bc4ec9652cb3b3369d00c2d448fdb64b10cdf096 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 17 Jul 2022 01:57:37 +0000 Subject: [PATCH 10/50] lint fixes --- src/Appwrite/Platform/Action.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Action.php b/src/Appwrite/Platform/Action.php index 314d7ede20..3982acfbe4 100644 --- a/src/Appwrite/Platform/Action.php +++ b/src/Appwrite/Platform/Action.php @@ -15,8 +15,10 @@ use Utopia\Registry\Registry; use Utopia\Logger\Log; use Throwable; -abstract class Action extends PlatformAction { - protected function logError(Registry $register, Throwable $error, string $action = 'syncUsageStats') { +abstract class Action extends PlatformAction +{ + protected function logError(Registry $register, Throwable $error, string $action = 'syncUsageStats') + { $logger = $register->get('logger'); if ($logger) { @@ -107,4 +109,4 @@ abstract class Action extends PlatformAction { } while ($attempts < $max); return $database; } -} \ No newline at end of file +} From 2df3bc813a7243dd16ff998c8d5a327fea68b890 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 2 Aug 2022 01:32:46 +0000 Subject: [PATCH 11/50] refactor rename --- app/cli.php | 7 +- app/tasks/doctor.php | 243 ----------------- app/tasks/install.php | 242 ----------------- app/tasks/maintenance.php | 151 ---------- app/tasks/migrate.php | 84 ------ app/tasks/sdks.php | 257 ------------------ app/tasks/specs.php | 256 ----------------- app/tasks/ssl.php | 24 -- app/tasks/usage.php | 164 ----------- app/tasks/vars.php | 25 -- composer.lock | 2 +- src/Appwrite/CLI/Tasks.php | 13 + src/Appwrite/{Task => CLI/Tasks}/Doctor.php | 2 +- src/Appwrite/{Task => CLI/Tasks}/Install.php | 2 +- .../{Task => CLI/Tasks}/Maintenance.php | 2 +- src/Appwrite/{Task => CLI/Tasks}/Migrate.php | 2 +- src/Appwrite/{Task => CLI/Tasks}/SDKs.php | 2 +- src/Appwrite/{Task => CLI/Tasks}/SSL.php | 2 +- src/Appwrite/{Task => CLI/Tasks}/Specs.php | 2 +- src/Appwrite/{Task => CLI/Tasks}/Usage.php | 2 +- src/Appwrite/{Task => CLI/Tasks}/Vars.php | 2 +- src/Appwrite/{Task => CLI/Tasks}/Version.php | 2 +- .../{Task/Tasks.php => CLI/TasksService.php} | 14 +- src/Appwrite/Task/CLIPlatform.php | 13 - 24 files changed, 40 insertions(+), 1475 deletions(-) delete mode 100644 app/tasks/doctor.php delete mode 100644 app/tasks/install.php delete mode 100644 app/tasks/maintenance.php delete mode 100644 app/tasks/migrate.php delete mode 100644 app/tasks/sdks.php delete mode 100644 app/tasks/specs.php delete mode 100644 app/tasks/ssl.php delete mode 100644 app/tasks/usage.php delete mode 100644 app/tasks/vars.php create mode 100644 src/Appwrite/CLI/Tasks.php rename src/Appwrite/{Task => CLI/Tasks}/Doctor.php (99%) rename src/Appwrite/{Task => CLI/Tasks}/Install.php (99%) rename src/Appwrite/{Task => CLI/Tasks}/Maintenance.php (99%) rename src/Appwrite/{Task => CLI/Tasks}/Migrate.php (99%) rename src/Appwrite/{Task => CLI/Tasks}/SDKs.php (99%) rename src/Appwrite/{Task => CLI/Tasks}/SSL.php (96%) rename src/Appwrite/{Task => CLI/Tasks}/Specs.php (99%) rename src/Appwrite/{Task => CLI/Tasks}/Usage.php (98%) rename src/Appwrite/{Task => CLI/Tasks}/Vars.php (96%) rename src/Appwrite/{Task => CLI/Tasks}/Version.php (92%) rename src/Appwrite/{Task/Tasks.php => CLI/TasksService.php} (64%) delete mode 100644 src/Appwrite/Task/CLIPlatform.php diff --git a/app/cli.php b/app/cli.php index 66c4208e14..ef0445e67f 100644 --- a/app/cli.php +++ b/app/cli.php @@ -3,13 +3,14 @@ require_once __DIR__ . '/init.php'; require_once __DIR__ . '/controllers/general.php'; -use Appwrite\Task\CLIPlatform; +use Appwrite\CLI\Tasks; use Utopia\Database\Validator\Authorization; +use Utopia\Platform\Service; Authorization::disable(); -$cliPlatform = new CLIPlatform(); -$cliPlatform->init('CLI'); +$cliPlatform = new Tasks(); +$cliPlatform->init(Service::TYPE_CLI); $cli = $cliPlatform->getCli(); $cli->run(); diff --git a/app/tasks/doctor.php b/app/tasks/doctor.php deleted file mode 100644 index d79e8d530f..0000000000 --- a/app/tasks/doctor.php +++ /dev/null @@ -1,243 +0,0 @@ -task('doctor') - ->desc('Validate server health') - ->action(function () use ($register) { - Console::log(" __ ____ ____ _ _ ____ __ ____ ____ __ __ - / _\ ( _ \( _ \/ )( \( _ \( )(_ _)( __) ( )/ \ -/ \ ) __/ ) __/\ /\ / ) / )( )( ) _) _ )(( O ) -\_/\_/(__) (__) (_/\_)(__\_)(__) (__) (____)(_)(__)\__/ "); - - Console::log("\n" . '👩‍⚕️ Running ' . APP_NAME . ' Doctor for version ' . App::getEnv('_APP_VERSION', 'UNKNOWN') . ' ...' . "\n"); - - Console::log('Checking for production best practices...'); - - $domain = new Domain(App::getEnv('_APP_DOMAIN')); - - if (!$domain->isKnown() || $domain->isTest()) { - Console::log('🔴 Hostname has no public suffix (' . $domain->get() . ')'); - } else { - Console::log('🟢 Hostname has a public suffix (' . $domain->get() . ')'); - } - - $domain = new Domain(App::getEnv('_APP_DOMAIN_TARGET')); - - if (!$domain->isKnown() || $domain->isTest()) { - Console::log('🔴 CNAME target has no public suffix (' . $domain->get() . ')'); - } else { - Console::log('🟢 CNAME target has a public suffix (' . $domain->get() . ')'); - } - - if (App::getEnv('_APP_OPENSSL_KEY_V1') === 'your-secret-key' || empty(App::getEnv('_APP_OPENSSL_KEY_V1'))) { - Console::log('🔴 Not using a unique secret key for encryption'); - } else { - Console::log('🟢 Using a unique secret key for encryption'); - } - - if (App::getEnv('_APP_ENV', 'development') !== 'production') { - Console::log('🔴 App environment is set for development'); - } else { - Console::log('🟢 App environment is set for production'); - } - - if ('enabled' !== App::getEnv('_APP_OPTIONS_ABUSE', 'disabled')) { - Console::log('🔴 Abuse protection is disabled'); - } else { - Console::log('🟢 Abuse protection is enabled'); - } - - $authWhitelistRoot = App::getEnv('_APP_CONSOLE_WHITELIST_ROOT', null); - $authWhitelistEmails = App::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null); - $authWhitelistIPs = App::getEnv('_APP_CONSOLE_WHITELIST_IPS', null); - - if ( - empty($authWhitelistRoot) - && empty($authWhitelistEmails) - && empty($authWhitelistIPs) - ) { - Console::log('🔴 Console access limits are disabled'); - } else { - Console::log('🟢 Console access limits are enabled'); - } - - if ('enabled' !== App::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled')) { - Console::log('🔴 HTTPS force option is disabled'); - } else { - Console::log('🟢 HTTPS force option is enabled'); - } - - - $providerName = App::getEnv('_APP_LOGGING_PROVIDER', ''); - $providerConfig = App::getEnv('_APP_LOGGING_CONFIG', ''); - - if (empty($providerName) || empty($providerConfig) || !Logger::hasProvider($providerName)) { - Console::log('🔴 Logging adapter is disabled'); - } else { - Console::log('🟢 Logging adapter is enabled (' . $providerName . ')'); - } - - \sleep(0.2); - - try { - Console::log("\n" . 'Checking connectivity...'); - } catch (\Throwable $th) { - //throw $th; - } - - try { - $register->get('db'); /* @var $db PDO */ - Console::success('Database............connected 👍'); - } catch (\Throwable $th) { - Console::error('Database.........disconnected 👎'); - } - - try { - $register->get('cache'); - Console::success('Queue...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Queue............disconnected 👎'); - } - - try { - $register->get('cache'); - Console::success('Cache...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Cache............disconnected 👎'); - } - - if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled') { // Check if scans are enabled - try { - $antivirus = new Network( - App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), - (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) - ); - - if ((@$antivirus->ping())) { - Console::success('Antivirus...........connected 👍'); - } else { - Console::error('Antivirus........disconnected 👎'); - } - } catch (\Throwable $th) { - Console::error('Antivirus........disconnected 👎'); - } - } - - try { - $mail = $register->get('smtp'); /* @var $mail \PHPMailer\PHPMailer\PHPMailer */ - - $mail->addAddress('demo@example.com', 'Example.com'); - $mail->Subject = 'Test SMTP Connection'; - $mail->Body = 'Hello World'; - $mail->AltBody = 'Hello World'; - - $mail->send(); - Console::success('SMTP................connected 👍'); - } catch (\Throwable $th) { - Console::error('SMTP.............disconnected 👎'); - } - - $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); - $port = App::getEnv('_APP_STATSD_PORT', 8125); - - if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) { - Console::success('StatsD..............connected 👍'); - \fclose($fp); - } else { - Console::error('StatsD...........disconnected 👎'); - } - - $host = App::getEnv('_APP_INFLUXDB_HOST', ''); - $port = App::getEnv('_APP_INFLUXDB_PORT', ''); - - if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) { - Console::success('InfluxDB............connected 👍'); - \fclose($fp); - } else { - Console::error('InfluxDB.........disconnected 👎'); - } - - \sleep(0.2); - - Console::log(''); - Console::log('Checking volumes...'); - - foreach ( - [ - 'Uploads' => APP_STORAGE_UPLOADS, - 'Cache' => APP_STORAGE_CACHE, - 'Config' => APP_STORAGE_CONFIG, - 'Certs' => APP_STORAGE_CERTIFICATES - ] as $key => $volume - ) { - $device = new Local($volume); - - if (\is_readable($device->getRoot())) { - Console::success('🟢 ' . $key . ' Volume is readable'); - } else { - Console::error('🔴 ' . $key . ' Volume is unreadable'); - } - - if (\is_writable($device->getRoot())) { - Console::success('🟢 ' . $key . ' Volume is writeable'); - } else { - Console::error('🔴 ' . $key . ' Volume is unwriteable'); - } - } - - \sleep(0.2); - - Console::log(''); - Console::log('Checking disk space usage...'); - - foreach ( - [ - 'Uploads' => APP_STORAGE_UPLOADS, - 'Cache' => APP_STORAGE_CACHE, - 'Config' => APP_STORAGE_CONFIG, - 'Certs' => APP_STORAGE_CERTIFICATES - ] as $key => $volume - ) { - $device = new Local($volume); - - $percentage = (($device->getPartitionTotalSpace() - $device->getPartitionFreeSpace()) - / $device->getPartitionTotalSpace()) * 100; - - $message = $key . ' Volume has ' . Storage::human($device->getPartitionFreeSpace()) . ' free space (' . \round($percentage, 2) . '% used)'; - - if ($percentage < 80) { - Console::success('🟢 ' . $message); - } else { - Console::error('🔴 ' . $message); - } - } - - try { - if (App::isProduction()) { - Console::log(''); - $version = \json_decode(@\file_get_contents(App::getEnv('_APP_HOME', 'http://localhost') . '/v1/health/version'), true); - - if ($version && isset($version['version'])) { - if (\version_compare($version['version'], App::getEnv('_APP_VERSION', 'UNKNOWN')) === 0) { - Console::info('You are running the latest version of ' . APP_NAME . '! 🥳'); - } else { - Console::info('A new version (' . $version['version'] . ') is available! 🥳' . "\n"); - } - } else { - Console::error('Failed to check for a newer version' . "\n"); - } - } - } catch (\Throwable $th) { - Console::error('Failed to check for a newer version' . "\n"); - } - }); diff --git a/app/tasks/install.php b/app/tasks/install.php deleted file mode 100644 index 3519b58d0b..0000000000 --- a/app/tasks/install.php +++ /dev/null @@ -1,242 +0,0 @@ -task('install') - ->desc('Install Appwrite') - ->param('httpPort', '', new Text(4), 'Server HTTP port', true) - ->param('httpsPort', '', new Text(4), 'Server HTTPS port', true) - ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true) - ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true) - ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true) - ->action(function ($httpPort, $httpsPort, $organization, $image, $interactive) { - /** - * 1. Start - DONE - * 2. Check for older setup and get older version - DONE - * 2.1 If older version is equal or bigger(?) than current version, **stop setup** - * 2.2. Get ENV vars - DONE - * 2.2.1 Fetch from older docker-compose.yml file - * 2.2.2 Fetch from older .env file (manually parse) - * 2.3 Use old ENV vars as default values - * 2.4 Ask for all required vars not given as CLI args and if in interactive mode - * Otherwise, just use default vars. - DONE - * 3. Ask user to backup important volumes, env vars, and SQL tables - * In th future we can try and automate this for smaller/medium size setups - * 4. Drop new docker-compose.yml setup (located inside the container, no network dependencies with appwrite.io) - DONE - * 5. Run docker compose up -d - DONE - * 6. Run data migration - */ - $config = Config::getParam('variables'); - $path = '/usr/src/code/appwrite'; - $defaultHTTPPort = '80'; - $defaultHTTPSPort = '443'; - $vars = []; - - /** - * We are using a random value every execution for identification. - * This allows us to collect information without invading the privacy of our users. - */ - $analytics = new GoogleAnalytics('UA-26264668-9', uniqid('server.', true)); - - foreach ($config as $category) { - foreach ($category['variables'] ?? [] as $var) { - $vars[] = $var; - } - } - - Console::success('Starting Appwrite installation...'); - - // Create directory with write permissions - if (null !== $path && !\file_exists(\dirname($path))) { - if (!@\mkdir(\dirname($path), 0755, true)) { - Console::error('Can\'t create directory ' . \dirname($path)); - Console::exit(1); - } - } - - $data = @file_get_contents($path . '/docker-compose.yml'); - - if ($data !== false) { - $time = \time(); - Console::info('Compose file found, creating backup: docker-compose.yml.' . $time . '.backup'); - file_put_contents($path . '/docker-compose.yml.' . $time . '.backup', $data); - $compose = new Compose($data); - $appwrite = $compose->getService('appwrite'); - $oldVersion = ($appwrite) ? $appwrite->getImageVersion() : null; - try { - $ports = $compose->getService('traefik')->getPorts(); - } catch (\Throwable $th) { - $ports = [ - $defaultHTTPPort => $defaultHTTPPort, - $defaultHTTPSPort => $defaultHTTPSPort - ]; - Console::warning('Traefik not found. Falling back to default ports.'); - } - - if ($oldVersion) { - foreach ($compose->getServices() as $service) { // Fetch all env vars from previous compose file - if (!$service) { - continue; - } - - $env = $service->getEnvironment()->list(); - - foreach ($env as $key => $value) { - if (is_null($value)) { - continue; - } - foreach ($vars as &$var) { - if ($var['name'] === $key) { - $var['default'] = $value; - } - } - } - } - - $data = @file_get_contents($path . '/.env'); - - if ($data !== false) { // Fetch all env vars from previous .env file - Console::info('Env file found, creating backup: .env.' . $time . '.backup'); - file_put_contents($path . '/.env.' . $time . '.backup', $data); - $env = new Env($data); - - foreach ($env->list() as $key => $value) { - if (is_null($value)) { - continue; - } - foreach ($vars as &$var) { - if ($var['name'] === $key) { - $var['default'] = $value; - } - } - } - } - - foreach ($ports as $key => $value) { - if ($value === $defaultHTTPPort) { - $defaultHTTPPort = $key; - } - - if ($value === $defaultHTTPSPort) { - $defaultHTTPSPort = $key; - } - } - } - } - - if (empty($httpPort)) { - $httpPort = Console::confirm('Choose your server HTTP port: (default: ' . $defaultHTTPPort . ')'); - $httpPort = ($httpPort) ? $httpPort : $defaultHTTPPort; - } - - if (empty($httpsPort)) { - $httpsPort = Console::confirm('Choose your server HTTPS port: (default: ' . $defaultHTTPSPort . ')'); - $httpsPort = ($httpsPort) ? $httpsPort : $defaultHTTPSPort; - } - - $input = []; - - foreach ($vars as $key => $var) { - if (!empty($var['filter']) && ($interactive !== 'Y' || !Console::isInteractive())) { - if ($data && $var['default'] !== null) { - $input[$var['name']] = $var['default']; - continue; - } - - if ($var['filter'] === 'token') { - $input[$var['name']] = Auth::tokenGenerator(); - continue; - } - - if ($var['filter'] === 'password') { - $input[$var['name']] = Auth::passwordGenerator(); - continue; - } - } - if (!$var['required'] || !Console::isInteractive() || $interactive !== 'Y') { - $input[$var['name']] = $var['default']; - continue; - } - - $input[$var['name']] = Console::confirm($var['question'] . ' (default: \'' . $var['default'] . '\')'); - - if (empty($input[$var['name']])) { - $input[$var['name']] = $var['default']; - } - - if ($var['filter'] === 'domainTarget') { - if ($input[$var['name']] !== 'localhost') { - Console::warning("\nIf you haven't already done so, set the following record for {$input[$var['name']]} on your DNS provider:\n"); - $mask = "%-15.15s %-10.10s %-30.30s\n"; - printf($mask, "Type", "Name", "Value"); - printf($mask, "A or AAAA", "@", ""); - Console::warning("\nUse 'AAAA' if you're using an IPv6 address and 'A' if you're using an IPv4 address.\n"); - } - } - } - - $templateForCompose = new View(__DIR__ . '/../views/install/compose.phtml'); - $templateForEnv = new View(__DIR__ . '/../views/install/env.phtml'); - - $templateForCompose - ->setParam('httpPort', $httpPort) - ->setParam('httpsPort', $httpsPort) - ->setParam('version', APP_VERSION_STABLE) - ->setParam('organization', $organization) - ->setParam('image', $image) - ; - - $templateForEnv - ->setParam('vars', $input) - ; - - if (!file_put_contents($path . '/docker-compose.yml', $templateForCompose->render(false))) { - $message = 'Failed to save Docker Compose file'; - $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message); - Console::error($message); - Console::exit(1); - } - - if (!file_put_contents($path . '/.env', $templateForEnv->render(false))) { - $message = 'Failed to save environment variables file'; - $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message); - Console::error($message); - Console::exit(1); - } - - $env = ''; - $stdout = ''; - $stderr = ''; - - foreach ($input as $key => $value) { - if ($value) { - $env .= $key . '=' . \escapeshellarg($value) . ' '; - } - } - - Console::log("Running \"docker compose -f {$path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes\""); - - $exit = Console::execute("${env} docker compose -f {$path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr); - - if ($exit !== 0) { - $message = 'Failed to install Appwrite dockers'; - $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message); - Console::error($message); - Console::error($stderr); - Console::exit($exit); - } else { - $message = 'Appwrite installed successfully'; - $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message); - Console::success($message); - } - }); diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php deleted file mode 100644 index eb50dbc0e2..0000000000 --- a/app/tasks/maintenance.php +++ /dev/null @@ -1,151 +0,0 @@ -get('cache'))); - $database = new Database(new MariaDB($register->get('db')), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); // Main DB - - if (!$database->exists($database->getDefaultDatabase(), 'certificates')) { - throw new \Exception('Console project not ready'); - } - - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; -} - -$cli - ->task('maintenance') - ->desc('Schedules maintenance tasks and publishes them to resque') - ->action(function () { - Console::title('Maintenance V1'); - Console::success(APP_NAME . ' maintenance process v1 has started'); - - function notifyDeleteExecutionLogs(int $interval) - { - (new Delete()) - ->setType(DELETE_TYPE_EXECUTIONS) - ->setTimestamp(time() - $interval) - ->trigger(); - } - - function notifyDeleteAbuseLogs(int $interval) - { - (new Delete()) - ->setType(DELETE_TYPE_ABUSE) - ->setTimestamp(time() - $interval) - ->trigger(); - } - - function notifyDeleteAuditLogs(int $interval) - { - (new Delete()) - ->setType(DELETE_TYPE_AUDIT) - ->setTimestamp(time() - $interval) - ->trigger(); - } - - function notifyDeleteUsageStats(int $interval30m, int $interval1d) - { - (new Delete()) - ->setType(DELETE_TYPE_USAGE) - ->setTimestamp1d(time() - $interval1d) - ->setTimestamp30m(time() - $interval30m) - ->trigger(); - } - - function notifyDeleteConnections() - { - (new Delete()) - ->setType(DELETE_TYPE_REALTIME) - ->setTimestamp(time() - 60) - ->trigger(); - } - - function notifyDeleteExpiredSessions() - { - (new Delete()) - ->setType(DELETE_TYPE_SESSIONS) - ->setTimestamp(time() - Auth::TOKEN_EXPIRATION_LOGIN_LONG) - ->trigger(); - } - - function renewCertificates($dbForConsole) - { - $time = date('d-m-Y H:i:s', time()); - $certificates = $dbForConsole->find('certificates', [ - new Query('attempts', Query::TYPE_LESSEREQUAL, [5]), // Maximum 5 attempts - new Query('renewDate', Query::TYPE_LESSEREQUAL, [\time()]) // includes 60 days cooldown (we have 30 days to renew) - ], 200); // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) - - - if (\count($certificates) > 0) { - Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); - - $event = new Certificate(); - foreach ($certificates as $certificate) { - $event - ->setDomain(new Document([ - 'domain' => $certificate->getAttribute('domain') - ])) - ->trigger(); - } - } else { - Console::info("[{$time}] No certificates for renewal."); - } - } - - // # of days in seconds (1 day = 86400s) - $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); - $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); - $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); - $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); - $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours - $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days - - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { - $database = getConsoleDB(); - - $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); - notifyDeleteExecutionLogs($executionLogsRetention); - notifyDeleteAbuseLogs($abuseLogsRetention); - notifyDeleteAuditLogs($auditLogRetention); - notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); - notifyDeleteConnections(); - notifyDeleteExpiredSessions(); - renewCertificates($database); - }, $interval); - }); diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php deleted file mode 100644 index ff5ec3593f..0000000000 --- a/app/tasks/migrate.php +++ /dev/null @@ -1,84 +0,0 @@ -task('migrate') - ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) - ->action(function ($version) use ($register) { - Authorization::disable(); - if (!array_key_exists($version, Migration::$versions)) { - Console::error("Version {$version} not found."); - Console::exit(1); - return; - } - - $app = new App('UTC'); - - Console::success('Starting Data Migration to version ' . $version); - - $db = $register->get('db', true); - $redis = $register->get('cache', true); - $redis->flushAll(); - $cache = new Cache(new RedisCache($redis)); - - $projectDB = new Database(new MariaDB($db), $cache); - $projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - - $consoleDB = new Database(new MariaDB($db), $cache); - $consoleDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $consoleDB->setNamespace('_project_console'); - - $console = $app->getResource('console'); - - $limit = 30; - $sum = 30; - $offset = 0; - $projects = [$console]; - $count = 0; - - try { - $totalProjects = $consoleDB->count('projects') + 1; - } catch (\Throwable $th) { - $consoleDB->setNamespace('_console'); - $totalProjects = $consoleDB->count('projects') + 1; - } - - $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; - $migration = new $class(); - - while (!empty($projects)) { - foreach ($projects as $project) { - try { - $migration - ->setProject($project, $projectDB, $consoleDB) - ->execute(); - } catch (\Throwable $th) { - throw $th; - Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); - } - } - - $sum = \count($projects); - $projects = $consoleDB->find('projects', limit: $limit, offset: $offset); - - $offset = $offset + $limit; - $count = $count + $sum; - - Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); - } - - Swoole\Event::wait(); // Wait for Coroutines to finish - $redis->flushAll(); - Console::success('Data Migration Completed'); - }); diff --git a/app/tasks/sdks.php b/app/tasks/sdks.php deleted file mode 100644 index d873167743..0000000000 --- a/app/tasks/sdks.php +++ /dev/null @@ -1,257 +0,0 @@ -task('sdks') - ->action(function () { - $platforms = Config::getParam('platforms'); - $selected = \strtolower(Console::confirm('Choose SDK ("*" for all):')); - $version = Console::confirm('Choose an Appwrite version'); - $git = (Console::confirm('Should we use git push? (yes/no)') == 'yes'); - $production = ($git) ? (Console::confirm('Type "Appwrite" to push code to production git repos') == 'Appwrite') : false; - $message = ($git) ? Console::confirm('Please enter your commit message:') : ''; - - if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', 'latest'])) { - throw new Exception('Unknown version given'); - } - - foreach ($platforms as $key => $platform) { - foreach ($platform['languages'] as $language) { - if ($selected !== $language['key'] && $selected !== '*') { - continue; - } - - if (!$language['enabled']) { - Console::warning($language['name'] . ' for ' . $platform['name'] . ' is disabled'); - continue; - } - - Console::info('Fetching API Spec for ' . $language['name'] . ' for ' . $platform['name'] . ' (version: ' . $version . ')'); - - $spec = file_get_contents(__DIR__ . '/../config/specs/swagger2-' . $version . '-' . $language['family'] . '.json'); - - $cover = 'https://appwrite.io/images/github.png'; - $result = \realpath(__DIR__ . '/..') . '/sdks/' . $key . '-' . $language['key']; - $resultExamples = \realpath(__DIR__ . '/../..') . '/docs/examples/' . $version . '/' . $key . '-' . $language['key']; - $target = \realpath(__DIR__ . '/..') . '/sdks/git/' . $language['key'] . '/'; - $readme = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/README.md'); - $readme = ($readme) ? \file_get_contents($readme) : ''; - $gettingStarted = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md'); - $gettingStarted = ($gettingStarted) ? \file_get_contents($gettingStarted) : ''; - $examples = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/EXAMPLES.md'); - $examples = ($examples) ? \file_get_contents($examples) : ''; - $changelog = \realpath(__DIR__ . '/../../docs/sdks/' . $language['key'] . '/CHANGELOG.md'); - $changelog = ($changelog) ? \file_get_contents($changelog) : '# Change Log'; - $warning = '**This SDK is compatible with Appwrite server version ' . $version . '. For older versions, please check [previous releases](' . $language['url'] . '/releases).**'; - $license = 'BSD-3-Clause'; - $licenseContent = 'Copyright (c) ' . date('Y') . ' Appwrite (https://appwrite.io) and individual contributors. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - - 3. Neither the name Appwrite nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.'; - - switch ($language['key']) { - case 'web': - $config = new Web(); - $config->setNPMPackage('appwrite'); - $config->setBowerPackage('appwrite'); - break; - case 'cli': - $config = new CLI(); - $config->setNPMPackage('appwrite-cli'); - $config->setExecutableName('appwrite'); - $config->setLogo(json_encode(" - _ _ _ ___ __ _____ - /_\ _ __ _ ____ ___ __(_) |_ ___ / __\ / / \_ \ - //_\\\| '_ \| '_ \ \ /\ / / '__| | __/ _ \ / / / / / /\/ - / _ \ |_) | |_) \ V V /| | | | || __/ / /___/ /___/\/ /_ - \_/ \_/ .__/| .__/ \_/\_/ |_| |_|\__\___| \____/\____/\____/ - |_| |_| - -")); - $config->setLogoUnescaped(" - _ _ _ ___ __ _____ - /_\ _ __ _ ____ ___ __(_) |_ ___ / __\ / / \_ \ - //_\\\| '_ \| '_ \ \ /\ / / '__| | __/ _ \ / / / / / /\/ - / _ \ |_) | |_) \ V V /| | | | || __/ / /___/ /___/\/ /_ - \_/ \_/ .__/| .__/ \_/\_/ |_| |_|\__\___| \____/\____/\____/ - |_| |_| "); - break; - case 'php': - $config = new PHP(); - $config->setComposerVendor('appwrite'); - $config->setComposerPackage('appwrite'); - break; - case 'nodejs': - $config = new Node(); - $config->setNPMPackage('node-appwrite'); - $config->setBowerPackage('appwrite'); - $warning = $warning . "\n\n > This is the Node.js SDK for integrating with Appwrite from your Node.js server-side code. - If you're looking to integrate from the browser, you should check [appwrite/sdk-for-web](https://github.com/appwrite/sdk-for-web)"; - break; - case 'deno': - $config = new Deno(); - break; - case 'python': - $config = new Python(); - $config->setPipPackage('appwrite'); - $license = 'BSD License'; // license edited due to classifiers in pypi - break; - case 'ruby': - $config = new Ruby(); - $config->setGemPackage('appwrite'); - break; - case 'flutter': - $config = new Flutter(); - $config->setPackageName('appwrite'); - break; - case 'flutter-dev': - $config = new Flutter(); - $config->setPackageName('appwrite_dev'); - break; - case 'dart': - $config = new Dart(); - $config->setPackageName('dart_appwrite'); - $warning = $warning . "\n\n > This is the Dart SDK for integrating with Appwrite from your Dart server-side code. If you're looking for the Flutter SDK you should check [appwrite/sdk-for-flutter](https://github.com/appwrite/sdk-for-flutter)"; - break; - case 'go': - $config = new Go(); - break; - case 'swift': - $config = new Swift(); - $warning = $warning . "\n\n > This is the Swift SDK for integrating with Appwrite from your Swift server-side code. If you're looking for the Apple SDK you should check [appwrite/sdk-for-apple](https://github.com/appwrite/sdk-for-apple)"; - break; - case 'apple': - $config = new SwiftClient(); - break; - case 'dotnet': - $cover = ''; - $config = new DotNet(); - break; - case 'android': - $config = new Android(); - break; - case 'kotlin': - $config = new Kotlin(); - $warning = $warning . "\n\n > This is the Kotlin SDK for integrating with Appwrite from your Kotlin server-side code. If you're looking for the Android SDK you should check [appwrite/sdk-for-android](https://github.com/appwrite/sdk-for-android)"; - break; - default: - throw new Exception('Language "' . $language['key'] . '" not supported'); - break; - } - - Console::info("Generating {$language['name']} SDK..."); - - $sdk = new SDK($config, new Swagger2($spec)); - - $sdk - ->setName($language['name']) - ->setNamespace('io appwrite') - ->setDescription("Appwrite is an open-source backend as a service server that abstract and simplify complex and repetitive development tasks behind a very simple to use REST API. Appwrite aims to help you develop your apps faster and in a more secure way. Use the {$language['name']} SDK to integrate your app with the Appwrite server to easily start interacting with all of Appwrite backend APIs and tools. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)") - ->setShortDescription('Appwrite is an open-source self-hosted backend server that abstract and simplify complex and repetitive development tasks behind a very simple REST API') - ->setLicense($license) - ->setLicenseContent($licenseContent) - ->setVersion($language['version']) - ->setGitURL($language['url']) - ->setGitRepo($language['gitUrl']) - ->setGitRepoName($language['gitRepoName']) - ->setGitUserName($language['gitUserName']) - ->setLogo($cover) - ->setURL('https://appwrite.io') - ->setShareText('Appwrite is a backend as a service for building web or mobile apps') - ->setShareURL('http://appwrite.io') - ->setShareTags('JS,javascript,reactjs,angular,ios,android,serverless') - ->setShareVia('appwrite') - ->setWarning($warning) - ->setReadme($readme) - ->setGettingStarted($gettingStarted) - ->setChangelog($changelog) - ->setExamples($examples) - ->setTwitter(APP_SOCIAL_TWITTER_HANDLE) - ->setDiscord(APP_SOCIAL_DISCORD_CHANNEL, APP_SOCIAL_DISCORD) - ->setDefaultHeaders([ - 'X-Appwrite-Response-Format' => '0.15.0', - ]); - - try { - $sdk->generate($result); - } catch (Exception $exception) { - Console::error($exception->getMessage()); - } catch (Throwable $exception) { - Console::error($exception->getMessage()); - } - - $gitUrl = $language['gitUrl']; - $gitBranch = $language['gitBranch']; - - - if (!$production) { - $gitUrl = 'git@github.com:aw-tests/' . $language['gitRepoName'] . '.git'; - } - - if ($git && !empty($gitUrl)) { - \exec('rm -rf ' . $target . ' && \ - mkdir -p ' . $target . ' && \ - cd ' . $target . ' && \ - git init --initial-branch=' . $gitBranch . ' && \ - git remote add origin ' . $gitUrl . ' && \ - git fetch && \ - git pull ' . $gitUrl . ' && \ - rm -rf ' . $target . '/* && \ - cp -r ' . $result . '/* ' . $target . '/ && \ - git add . && \ - git commit -m "' . $message . '" && \ - git push -u origin ' . $gitBranch . ' - '); - - Console::success("Pushed {$language['name']} SDK to {$gitUrl}"); - - \exec('rm -rf ' . $target); - Console::success("Remove temp directory '{$target}' for {$language['name']} SDK"); - } - - $docDirectories = $language['docDirectories'] ?? ['']; - - if ($version === 'latest') { - continue; - } - - foreach ($docDirectories as $languageTitle => $path) { - $languagePath = strtolower($languageTitle !== 0 ? '/' . $languageTitle : ''); - \exec( - 'mkdir -p ' . $resultExamples . $languagePath . ' && \ - cp -r ' . $result . '/docs/examples' . $languagePath . ' ' . $resultExamples - ); - Console::success("Copied code examples for {$language['name']} SDK to: {$resultExamples}"); - } - } - } - - Console::exit(); - }); diff --git a/app/tasks/specs.php b/app/tasks/specs.php deleted file mode 100644 index 8e8fdb4e7c..0000000000 --- a/app/tasks/specs.php +++ /dev/null @@ -1,256 +0,0 @@ -task('specs') - ->param('version', 'latest', new Text(8), 'Spec version', true) - ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true) - ->action(function ($version, $mode) use ($register) { - $db = $register->get('db'); - $redis = $register->get('cache'); - $appRoutes = App::getRoutes(); - $response = new Response(new HttpResponse()); - $mocks = ($mode === 'mocks'); - - App::setResource('request', fn () => new Request()); - App::setResource('response', fn () => $response); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); - - $platforms = [ - 'client' => APP_PLATFORM_CLIENT, - 'server' => APP_PLATFORM_SERVER, - 'console' => APP_PLATFORM_CONSOLE, - ]; - - $authCounts = [ - 'client' => 1, - 'server' => 2, - 'console' => 1, - ]; - - $keys = [ - APP_PLATFORM_CLIENT => [ - 'Project' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-Project', - 'description' => 'Your project ID', - 'in' => 'header', - ], - 'JWT' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-JWT', - 'description' => 'Your secret JSON Web Token', - 'in' => 'header', - ], - 'Locale' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-Locale', - 'description' => '', - 'in' => 'header', - ], - ], - APP_PLATFORM_SERVER => [ - 'Project' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-Project', - 'description' => 'Your project ID', - 'in' => 'header', - ], - 'Key' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-Key', - 'description' => 'Your secret API key', - 'in' => 'header', - ], - 'JWT' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-JWT', - 'description' => 'Your secret JSON Web Token', - 'in' => 'header', - ], - 'Locale' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-Locale', - 'description' => '', - 'in' => 'header', - ], - ], - APP_PLATFORM_CONSOLE => [ - 'Project' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-Project', - 'description' => 'Your project ID', - 'in' => 'header', - ], - 'Key' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-Key', - 'description' => 'Your secret API key', - 'in' => 'header', - ], - 'JWT' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-JWT', - 'description' => 'Your secret JSON Web Token', - 'in' => 'header', - ], - 'Locale' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-Locale', - 'description' => '', - 'in' => 'header', - ], - 'Mode' => [ - 'type' => 'apiKey', - 'name' => 'X-Appwrite-Mode', - 'description' => '', - 'in' => 'header', - ], - ], - ]; - - foreach ($platforms as $platform) { - $routes = []; - $models = []; - $services = []; - - foreach ($appRoutes as $key => $method) { - foreach ($method as $route) { - /** @var \Utopia\Route $route */ - $routeSecurity = $route->getLabel('sdk.auth', []); - $sdkPlaforms = []; - - foreach ($routeSecurity as $value) { - switch ($value) { - case APP_AUTH_TYPE_SESSION: - $sdkPlaforms[] = APP_PLATFORM_CLIENT; - break; - case APP_AUTH_TYPE_KEY: - $sdkPlaforms[] = APP_PLATFORM_SERVER; - break; - case APP_AUTH_TYPE_JWT: - $sdkPlaforms[] = APP_PLATFORM_SERVER; - break; - case APP_AUTH_TYPE_ADMIN: - $sdkPlaforms[] = APP_PLATFORM_CONSOLE; - break; - } - } - - if (empty($routeSecurity)) { - $sdkPlaforms[] = APP_PLATFORM_CLIENT; - } - - if (!$route->getLabel('docs', true)) { - continue; - } - - if ($route->getLabel('sdk.mock', false) && !$mocks) { - continue; - } - - if (!$route->getLabel('sdk.mock', false) && $mocks) { - continue; - } - - if (empty($route->getLabel('sdk.namespace', null))) { - continue; - } - - if ($platform !== APP_PLATFORM_CONSOLE && !\in_array($platforms[$platform], $sdkPlaforms)) { - continue; - } - - $routes[] = $route; - } - } - - foreach (Config::getParam('services', []) as $service) { - if ( - !isset($service['docs']) // Skip service if not part of the public API - || !isset($service['sdk']) - || !$service['docs'] - || !$service['sdk'] - ) { - continue; - } - - $services[] = [ - 'name' => $service['key'] ?? '', - 'description' => $service['subtitle'] ?? '', - 'x-globalAttributes' => $service['globalAttributes'] ?? [], - ]; - } - - $models = $response->getModels(); - - foreach ($models as $key => $value) { - if ($platform !== APP_PLATFORM_CONSOLE && !$value->isPublic()) { - unset($models[$key]); - } - } - // var_dump($models); - $arguments = [new App('UTC'), $services, $routes, $models, $keys[$platform], $authCounts[$platform] ?? 0]; - foreach (['swagger2', 'open-api3'] as $format) { - $formatInstance = match ($format) { - 'swagger2' => new Swagger2(...$arguments), - 'open-api3' => new OpenAPI3(...$arguments), - default => throw new Exception('Format not found: ' . $format) - }; - - $specs = new Specification($formatInstance); - $endpoint = App::getEnv('_APP_HOME', '[HOSTNAME]'); - $email = App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); - - $formatInstance - ->setParam('name', APP_NAME) - ->setParam('description', 'Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)') - ->setParam('endpoint', 'https://HOSTNAME/v1') - ->setParam('version', APP_VERSION_STABLE) - ->setParam('terms', $endpoint . '/policy/terms') - ->setParam('support.email', $email) - ->setParam('support.url', $endpoint . '/support') - ->setParam('contact.name', APP_NAME . ' Team') - ->setParam('contact.email', $email) - ->setParam('contact.url', $endpoint . '/support') - ->setParam('license.name', 'BSD-3-Clause') - ->setParam('license.url', 'https://raw.githubusercontent.com/appwrite/appwrite/master/LICENSE') - ->setParam('docs.description', 'Full API docs, specs and tutorials') - ->setParam('docs.url', $endpoint . '/docs'); - - if ($mocks) { - $path = __DIR__ . '/../config/specs/' . $format . '-mocks-' . $platform . '.json'; - - if (!file_put_contents($path, json_encode($specs->parse()))) { - throw new Exception('Failed to save mocks spec file: ' . $path); - } - - Console::success('Saved mocks spec file: ' . realpath($path)); - - continue; - } - - $path = __DIR__ . '/../config/specs/' . $format . '-' . $version . '-' . $platform . '.json'; - - if (!file_put_contents($path, json_encode($specs->parse()))) { - throw new Exception('Failed to save spec file: ' . $path); - } - - Console::success('Saved spec file: ' . realpath($path)); - } - } - }); diff --git a/app/tasks/ssl.php b/app/tasks/ssl.php deleted file mode 100644 index 345da1f22e..0000000000 --- a/app/tasks/ssl.php +++ /dev/null @@ -1,24 +0,0 @@ -task('ssl') - ->desc('Validate server certificates') - ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) - ->action(function ($domain) { - Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain); - - (new Certificate()) - ->setDomain(new Document([ - 'domain' => $domain - ])) - ->setSkipRenewCheck(true) - ->trigger(); - }); diff --git a/app/tasks/usage.php b/app/tasks/usage.php deleted file mode 100644 index 7a22d7e79f..0000000000 --- a/app/tasks/usage.php +++ /dev/null @@ -1,164 +0,0 @@ -get('db'); - $redis = $register->get('cache'); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); - - if (!$database->exists($database->getDefaultDatabase(), 'projects')) { - throw new Exception('Projects collection not ready'); - } - break; // leave loop if successful - } catch (\Exception$e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; -} - -function getInfluxDB(Registry &$register): InfluxDatabase -{ - /** @var InfluxDB\Client $client */ - $client = $register->get('influxdb'); - $attempts = 0; - $max = 10; - $sleep = 1; - - do { // check if telegraf database is ready - try { - $attempts++; - $database = $client->selectDB('telegraf'); - if (in_array('telegraf', $client->listDatabases())) { - break; // leave the do-while if successful - } - } catch (\Throwable$th) { - Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('InfluxDB database not ready yet'); - } - sleep($sleep); - } - } while ($attempts < $max); - return $database; -} - -$logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) { - $logger = $register->get('logger'); - - if ($logger) { - $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); - - $log = new Log(); - $log->setNamespace("usage"); - $log->setServer(\gethostname()); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - - $log->addTag('code', $error->getCode()); - $log->addTag('verboseType', get_class($error)); - - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('detailedTrace', $error->getTrace()); - - $log->setAction($action); - - $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - $responseCode = $logger->addLog($log); - Console::info('Usage stats log pushed with status code: ' . $responseCode); - } - - Console::warning("Failed: {$error->getMessage()}"); - Console::warning($error->getTraceAsString()); -}; - -$cli - ->task('usage') - ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function () use ($register, $logError) { - Console::title('Usage Aggregation V1'); - Console::success(APP_NAME . ' usage aggregation process v1 has started'); - - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) - - $database = getDatabase($register, '_console'); - $influxDB = getInfluxDB($register); - - $usage = new Usage($database, $influxDB, $logError); - $usageDB = new UsageDB($database, $logError); - - $iterations = 0; - Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); - - $loopStart = microtime(true); - - /** - * Aggregate InfluxDB every 30 seconds - */ - $usage->collect(); - - if ($iterations % 30 != 0) { // return if 30 iterations has not passed - $iterations++; - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - return; - } - - $iterations = 0; // Reset iterations to prevent overflow when running for long time - /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating database counters."); - - $usageDB->collect(); - - $iterations++; - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - }, $interval); - }); diff --git a/app/tasks/vars.php b/app/tasks/vars.php deleted file mode 100644 index af972e218d..0000000000 --- a/app/tasks/vars.php +++ /dev/null @@ -1,25 +0,0 @@ -task('vars') - ->desc('List all the server environment variables') - ->action(function () { - $config = Config::getParam('variables', []); - $vars = []; - - foreach ($config as $category) { - foreach ($category['variables'] ?? [] as $var) { - $vars[] = $var; - } - } - - foreach ($vars as $key => $value) { - Console::log('- ' . $value['name'] . '=' . App::getEnv($value['name'], '')); - } - }); diff --git a/composer.lock b/composer.lock index f2bc50530e..b94d95b84a 100644 --- a/composer.lock +++ b/composer.lock @@ -5441,5 +5441,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/src/Appwrite/CLI/Tasks.php b/src/Appwrite/CLI/Tasks.php new file mode 100644 index 0000000000..7f07b8137a --- /dev/null +++ b/src/Appwrite/CLI/Tasks.php @@ -0,0 +1,13 @@ +addService('cliTasks', new TasksService()); + } +} diff --git a/src/Appwrite/Task/Doctor.php b/src/Appwrite/CLI/Tasks/Doctor.php similarity index 99% rename from src/Appwrite/Task/Doctor.php rename to src/Appwrite/CLI/Tasks/Doctor.php index 281efe4d35..35fe9d048e 100644 --- a/src/Appwrite/Task/Doctor.php +++ b/src/Appwrite/CLI/Tasks/Doctor.php @@ -1,6 +1,6 @@ addService('cliTasks', new Tasks()); - } -} From 0331776614740532f856026de14e99fe871ca502 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 2 Aug 2022 01:37:00 +0000 Subject: [PATCH 12/50] more naming refactor --- src/Appwrite/CLI/Tasks/Usage.php | 4 ++-- src/Appwrite/Platform/Action.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/CLI/Tasks/Usage.php b/src/Appwrite/CLI/Tasks/Usage.php index 78b41c1e06..810388cd8c 100644 --- a/src/Appwrite/CLI/Tasks/Usage.php +++ b/src/Appwrite/CLI/Tasks/Usage.php @@ -2,7 +2,7 @@ namespace Appwrite\CLI\Tasks; -use Appwrite\Platform\Action; +use Appwrite\Platform\Task; use Throwable; use Exception; use Appwrite\Stats\Usage as InfluxUsage; @@ -11,7 +11,7 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Validator\Authorization; -class Usage extends Action +class Usage extends Task { public const NAME = 'usage'; diff --git a/src/Appwrite/Platform/Action.php b/src/Appwrite/Platform/Action.php index 3982acfbe4..21c3f53159 100644 --- a/src/Appwrite/Platform/Action.php +++ b/src/Appwrite/Platform/Action.php @@ -3,7 +3,7 @@ namespace Appwrite\Platform; use Utopia\Database\Database; -use Utopia\Platform\Action as PlatformAction; +use Utopia\Platform\Action; use Exception; use InfluxDB\Database as InfluxDatabase; use Utopia\App; @@ -15,7 +15,7 @@ use Utopia\Registry\Registry; use Utopia\Logger\Log; use Throwable; -abstract class Action extends PlatformAction +abstract class Task extends Action { protected function logError(Registry $register, Throwable $error, string $action = 'syncUsageStats') { From b6ddf2a45149ca576b8d2d9f2aaaf2052173dacd Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 2 Aug 2022 01:41:12 +0000 Subject: [PATCH 13/50] remove unused import --- src/Appwrite/CLI/Tasks/Usage.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/CLI/Tasks/Usage.php b/src/Appwrite/CLI/Tasks/Usage.php index 810388cd8c..1b58809155 100644 --- a/src/Appwrite/CLI/Tasks/Usage.php +++ b/src/Appwrite/CLI/Tasks/Usage.php @@ -4,7 +4,6 @@ namespace Appwrite\CLI\Tasks; use Appwrite\Platform\Task; use Throwable; -use Exception; use Appwrite\Stats\Usage as InfluxUsage; use Appwrite\Stats\UsageDB; use Utopia\App; From 6b2b9d0fdeeeaea28f11a58371fb387c72119d57 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 2 Aug 2022 01:58:36 +0000 Subject: [PATCH 14/50] tasks get name function --- src/Appwrite/CLI/Tasks/Doctor.php | 5 ++++- src/Appwrite/CLI/Tasks/Install.php | 5 ++++- src/Appwrite/CLI/Tasks/Maintenance.php | 5 ++++- src/Appwrite/CLI/Tasks/Migrate.php | 6 +++++- src/Appwrite/CLI/Tasks/SDKs.php | 6 +++++- src/Appwrite/CLI/Tasks/SSL.php | 5 ++++- src/Appwrite/CLI/Tasks/Specs.php | 6 +++++- src/Appwrite/CLI/Tasks/Usage.php | 5 ++++- src/Appwrite/CLI/Tasks/Vars.php | 5 ++++- src/Appwrite/CLI/Tasks/Version.php | 5 ++++- src/Appwrite/CLI/TasksService.php | 20 ++++++++++---------- 11 files changed, 53 insertions(+), 20 deletions(-) diff --git a/src/Appwrite/CLI/Tasks/Doctor.php b/src/Appwrite/CLI/Tasks/Doctor.php index 35fe9d048e..600be03a2f 100644 --- a/src/Appwrite/CLI/Tasks/Doctor.php +++ b/src/Appwrite/CLI/Tasks/Doctor.php @@ -13,7 +13,10 @@ use Utopia\Platform\Action; class Doctor extends Action { - public const NAME = 'doctor'; + public static function getName(): string + { + return 'doctor'; + } public function __construct() { diff --git a/src/Appwrite/CLI/Tasks/Install.php b/src/Appwrite/CLI/Tasks/Install.php index fbf2ec5a3b..9bcc2e7096 100644 --- a/src/Appwrite/CLI/Tasks/Install.php +++ b/src/Appwrite/CLI/Tasks/Install.php @@ -14,7 +14,10 @@ use Utopia\Platform\Action; class Install extends Action { - public const NAME = 'install'; + public static function getName(): string + { + return 'install'; + } public function __construct() { diff --git a/src/Appwrite/CLI/Tasks/Maintenance.php b/src/Appwrite/CLI/Tasks/Maintenance.php index b535bdb7e1..7aae40cc11 100644 --- a/src/Appwrite/CLI/Tasks/Maintenance.php +++ b/src/Appwrite/CLI/Tasks/Maintenance.php @@ -17,7 +17,10 @@ use Utopia\Platform\Action; class Maintenance extends Action { - public const NAME = 'maintenance'; + public static function getName(): string + { + return 'maintenance'; + } protected function getConsoleDB(): Database { diff --git a/src/Appwrite/CLI/Tasks/Migrate.php b/src/Appwrite/CLI/Tasks/Migrate.php index 1e5d06a7c7..d1e41a02f0 100644 --- a/src/Appwrite/CLI/Tasks/Migrate.php +++ b/src/Appwrite/CLI/Tasks/Migrate.php @@ -16,11 +16,15 @@ use Swoole\Event; class Migrate extends Action { - public const NAME = 'migrate'; + public static function getName(): string + { + return 'migrate'; + } public function __construct() { $this + ->desc('Migrate Appwrite to new version') ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) ->callback(fn ($version) => $this->action($version)); } diff --git a/src/Appwrite/CLI/Tasks/SDKs.php b/src/Appwrite/CLI/Tasks/SDKs.php index ea9fc51452..a9be206fbe 100644 --- a/src/Appwrite/CLI/Tasks/SDKs.php +++ b/src/Appwrite/CLI/Tasks/SDKs.php @@ -27,11 +27,15 @@ use Throwable; class SDKs extends Action { - public const NAME = 'sdks'; + public static function getName(): string + { + return 'sdks'; + } public function __construct() { $this + ->desc('Generate Appwrite SDKs') ->callback(fn () => $this->action()); } diff --git a/src/Appwrite/CLI/Tasks/SSL.php b/src/Appwrite/CLI/Tasks/SSL.php index 79400e9eb2..6379253789 100644 --- a/src/Appwrite/CLI/Tasks/SSL.php +++ b/src/Appwrite/CLI/Tasks/SSL.php @@ -11,7 +11,10 @@ use Utopia\Validator\Hostname; class SSL extends Action { - public const NAME = 'ssl'; + public static function getName(): string + { + return 'ssl'; + } public function __construct() { diff --git a/src/Appwrite/CLI/Tasks/Specs.php b/src/Appwrite/CLI/Tasks/Specs.php index bcf1911ff3..5c030a4880 100644 --- a/src/Appwrite/CLI/Tasks/Specs.php +++ b/src/Appwrite/CLI/Tasks/Specs.php @@ -18,11 +18,15 @@ use Exception; class Specs extends Action { - public const NAME = 'specs'; + public static function getName(): string + { + return 'specs'; + } public function __construct() { $this + ->desc('Generate Appwrite API specifications') ->param('version', 'latest', new Text(8), 'Spec version', true) ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true) ->callback(fn ($version, $mode) => $this->action($version, $mode)); diff --git a/src/Appwrite/CLI/Tasks/Usage.php b/src/Appwrite/CLI/Tasks/Usage.php index 1b58809155..b40f17c069 100644 --- a/src/Appwrite/CLI/Tasks/Usage.php +++ b/src/Appwrite/CLI/Tasks/Usage.php @@ -12,7 +12,10 @@ use Utopia\Database\Validator\Authorization; class Usage extends Task { - public const NAME = 'usage'; + public static function getName(): string + { + return 'usage'; + } public function __construct() { diff --git a/src/Appwrite/CLI/Tasks/Vars.php b/src/Appwrite/CLI/Tasks/Vars.php index 442e1dc4ce..7286cc5382 100644 --- a/src/Appwrite/CLI/Tasks/Vars.php +++ b/src/Appwrite/CLI/Tasks/Vars.php @@ -9,7 +9,10 @@ use Utopia\Platform\Action; class Vars extends Action { - public const NAME = 'vars'; + public static function getName(): string + { + return 'vars'; + } public function __construct() { diff --git a/src/Appwrite/CLI/Tasks/Version.php b/src/Appwrite/CLI/Tasks/Version.php index 0a53482ce1..164b8b0766 100644 --- a/src/Appwrite/CLI/Tasks/Version.php +++ b/src/Appwrite/CLI/Tasks/Version.php @@ -8,7 +8,10 @@ use Utopia\Platform\Action; class Version extends Action { - public const NAME = 'version'; + public static function getName(): string + { + return 'version'; + } public function __construct() { diff --git a/src/Appwrite/CLI/TasksService.php b/src/Appwrite/CLI/TasksService.php index c91b3a9777..c6dba81dbc 100644 --- a/src/Appwrite/CLI/TasksService.php +++ b/src/Appwrite/CLI/TasksService.php @@ -20,15 +20,15 @@ class TasksService extends Service { $this->type = self::TYPE_CLI; $this - ->addAction(Version::NAME, new Version()) - ->addAction(Usage::NAME, new Usage()) - ->addAction(Vars::NAME, new Vars()) - ->addAction(SSL::NAME, new SSL()) - ->addAction(Doctor::NAME, new Doctor()) - ->addAction(Install::NAME, new Install()) - ->addAction(Maintenance::NAME, new Maintenance()) - ->addAction(Migrate::NAME, new Migrate()) - ->addAction(SDKs::NAME, new SDKs()) - ->addAction(Specs::NAME, new Specs()); + ->addAction(Version::getName(), new Version()) + ->addAction(Usage::getName(), new Usage()) + ->addAction(Vars::getName(), new Vars()) + ->addAction(SSL::getName(), new SSL()) + ->addAction(Doctor::getName(), new Doctor()) + ->addAction(Install::getName(), new Install()) + ->addAction(Maintenance::getName(), new Maintenance()) + ->addAction(Migrate::getName(), new Migrate()) + ->addAction(SDKs::getName(), new SDKs()) + ->addAction(Specs::getName(), new Specs()); } } From fda6ba7c8cb364329ce289b659bd09d9f638bd5e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 2 Aug 2022 02:03:30 +0000 Subject: [PATCH 15/50] fix lint error --- src/Appwrite/CLI/Tasks/Doctor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/CLI/Tasks/Doctor.php b/src/Appwrite/CLI/Tasks/Doctor.php index 600be03a2f..85d389b846 100644 --- a/src/Appwrite/CLI/Tasks/Doctor.php +++ b/src/Appwrite/CLI/Tasks/Doctor.php @@ -16,7 +16,7 @@ class Doctor extends Action public static function getName(): string { return 'doctor'; - } + } public function __construct() { From b28d10ad65193defa5d55a0e3aae5ccbc2b87681 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 3 Aug 2022 04:41:48 +0000 Subject: [PATCH 16/50] upgrade framework --- composer.json | 2 +- composer.lock | 25 ++++++++----------------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/composer.json b/composer.json index 65ae9d60b4..31341f9018 100644 --- a/composer.json +++ b/composer.json @@ -42,7 +42,7 @@ "ext-sockets": "*", "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.10.*", - "utopia-php/framework": "dev-master as 0.19.21", + "utopia-php/framework": "0.20.*", "utopia-php/logger": "0.3.*", "utopia-php/abuse": "0.7.*", "utopia-php/analytics": "0.2.*", diff --git a/composer.lock b/composer.lock index b94d95b84a..8188b14fac 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "39572e8195342eb83dd40d12314f2e6c", + "content-hash": "4f3868304dc2774906290ca8d36b0d06", "packages": [ { "name": "adhocore/jwt", @@ -2169,7 +2169,7 @@ }, { "name": "utopia-php/framework", - "version": "dev-master", + "version": "0.20.0", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", @@ -2188,7 +2188,6 @@ "phpunit/phpunit": "^9.5.10", "vimeo/psalm": "4.13.1" }, - "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -2213,7 +2212,7 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/master" + "source": "https://github.com/utopia-php/framework/tree/0.20.0" }, "time": "2022-07-30T09:55:28+00:00" }, @@ -2447,14 +2446,14 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "9d5fd12dfa1b3932313362a90ffafd5467525be0" + "reference": "cc41247d0636648ba854afba8ebf68be441cfd1b" }, "require": { "ext-json": "*", "ext-redis": "*", "php": ">=8.0", "utopia-php/cli": "0.13.*", - "utopia-php/framework": "dev-master as 0.19.21" + "utopia-php/framework": "0.20.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -2499,7 +2498,7 @@ "upf", "utopia" ], - "time": "2022-08-01T06:39:29+00:00" + "time": "2022-08-03T04:41:22+00:00" }, { "name": "utopia-php/preloader", @@ -5407,17 +5406,9 @@ "time": "2022-05-17T05:48:52+00:00" } ], - "aliases": [ - { - "package": "utopia-php/framework", - "version": "9999999-dev", - "alias": "0.19.21", - "alias_normalized": "0.19.21.0" - } - ], + "aliases": [], "minimum-stability": "stable", "stability-flags": { - "utopia-php/framework": 20, "utopia-php/platform": 20 }, "prefer-stable": false, @@ -5441,5 +5432,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } From 0d62f30d7bdd1108a64f1c017b60d5158751ca3d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 4 Aug 2022 12:36:04 +0545 Subject: [PATCH 17/50] rename file --- src/Appwrite/Platform/{Action.php => Task.php} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Appwrite/Platform/{Action.php => Task.php} (100%) diff --git a/src/Appwrite/Platform/Action.php b/src/Appwrite/Platform/Task.php similarity index 100% rename from src/Appwrite/Platform/Action.php rename to src/Appwrite/Platform/Task.php From bd1d9834bee5c1f78691c15e246e7c4abfb509e5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 09:32:17 +0000 Subject: [PATCH 18/50] fix install task --- src/Appwrite/CLI/Tasks/Install.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Appwrite/CLI/Tasks/Install.php b/src/Appwrite/CLI/Tasks/Install.php index 9bcc2e7096..b1e980b7f4 100644 --- a/src/Appwrite/CLI/Tasks/Install.php +++ b/src/Appwrite/CLI/Tasks/Install.php @@ -186,6 +186,16 @@ class Install extends Action if (empty($input[$var['name']])) { $input[$var['name']] = $var['default']; } + + if ($var['filter'] === 'domainTarget') { + if ($input[$var['name']] !== 'localhost') { + Console::warning("\nIf you haven't already done so, set the following record for {$input[$var['name']]} on your DNS provider:\n"); + $mask = "%-15.15s %-10.10s %-30.30s\n"; + printf($mask, "Type", "Name", "Value"); + printf($mask, "A or AAAA", "@", ""); + Console::warning("\nUse 'AAAA' if you're using an IPv6 address and 'A' if you're using an IPv4 address.\n"); + } + } } $templateForCompose = new View(__DIR__ . '/../views/install/compose.phtml'); From e5b3d74515ac3cdad1a4307be62c6446cf1fd2b3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 09:34:02 +0000 Subject: [PATCH 19/50] update migrate task --- src/Appwrite/CLI/Tasks/Migrate.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Appwrite/CLI/Tasks/Migrate.php b/src/Appwrite/CLI/Tasks/Migrate.php index b28f0f6b82..e9f0f0dec3 100644 --- a/src/Appwrite/CLI/Tasks/Migrate.php +++ b/src/Appwrite/CLI/Tasks/Migrate.php @@ -13,7 +13,6 @@ use Utopia\Database\Database; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Validator\Text; -use Swoole\Event; class Migrate extends Action { @@ -105,7 +104,7 @@ class Migrate extends Action Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...'); } - Event::wait(); // Wait for Coroutines to finish + Swoole\Event::wait(); // Wait for Coroutines to finish $redis->flushAll(); Console::success('Data Migration Completed'); } From e5ad8d63f03e05451a9be9f31c314f4f42c9ffe8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 09:42:48 +0000 Subject: [PATCH 20/50] fix usage task --- src/Appwrite/CLI/Tasks/Usage.php | 96 +++++++++++++++++--------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/src/Appwrite/CLI/Tasks/Usage.php b/src/Appwrite/CLI/Tasks/Usage.php index b40f17c069..addf682be3 100644 --- a/src/Appwrite/CLI/Tasks/Usage.php +++ b/src/Appwrite/CLI/Tasks/Usage.php @@ -3,12 +3,14 @@ namespace Appwrite\CLI\Tasks; use Appwrite\Platform\Task; -use Throwable; -use Appwrite\Stats\Usage as InfluxUsage; -use Appwrite\Stats\UsageDB; +use Appwrite\Usage\Calculators\Aggregator; +use Appwrite\Usage\Calculators\Database; +use Appwrite\Usage\Calculators\TimeSeries; +use InfluxDB\Database as InfluxDatabase; use Utopia\App; use Utopia\CLI\Console; -use Utopia\Database\Validator\Authorization; +use Utopia\Database\Database as UtopiaDatabase; +use Utopia\Validator\WhiteList; class Usage extends Task { @@ -20,65 +22,67 @@ class Usage extends Task public function __construct() { $this + ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->callback(fn () => $this->action()); + ->callback(fn ($type) => $this->action($type)); } - public function action() + + protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void { - global $register; + $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) + $usage = new TimeSeries($database, $influxDB, $logError); - Authorization::disable(); - Authorization::setDefaultStatus(false); - - $logError = fn(Throwable $error, string $action = 'syncUsageStats') => $this->logError($register, $error, $action); - - Console::title('Usage Aggregation V1'); - Console::success(APP_NAME . ' usage aggregation process v1 has started'); - - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) - - $database = self::getDatabase($register, '_console'); - $influxDB = self::getInfluxDB($register); - - $usage = new InfluxUsage($database, $influxDB, $logError); - $usageDB = new UsageDB($database, $logError); - - $iterations = 0; - Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) { + Console::loop(function () use ($interval, $usage) { $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); - + Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds"); $loopStart = microtime(true); - /** - * Aggregate InfluxDB every 30 seconds - */ $usage->collect(); - if ($iterations % 30 != 0) { // return if 30 iterations has not passed - $iterations++; - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - return; - } - - $iterations = 0; // Reset iterations to prevent overflow when running for long time - /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ + $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating database counters."); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); + } - $usageDB->collect(); + protected function aggregateDatabase(UtopiaDatabase $database, callable $logError): void + { + $interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default) + $usage = new Database($database, $logError); + $aggregrator = new Aggregator($database, $logError); - $iterations++; + Console::loop(function () use ($interval, $usage, $aggregrator) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating database usage every {$interval} seconds."); + $loopStart = microtime(true); + $usage->collect(); + $aggregrator->collect(); $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); } + + public function action(string $type) + { + global $register; + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); + + $database = $this->getDatabase($register, '_console'); + $influxDB = $this->getInfluxDB($register); + + switch ($type) { + case 'timeseries': + $this->aggregateTimeseries($database, $influxDB, $this->logError); + break; + case 'database': + $this->aggregateDatabase($database, $this->logError); + break; + default: + Console::error("Unsupported usage aggregation type"); + } + } } From 72e112ade8486870416bf695911f6932b6972f5a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 14 Oct 2022 11:01:45 +0000 Subject: [PATCH 21/50] fix usage task error --- src/Appwrite/CLI/Tasks/Usage.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/CLI/Tasks/Usage.php b/src/Appwrite/CLI/Tasks/Usage.php index addf682be3..d5360e1091 100644 --- a/src/Appwrite/CLI/Tasks/Usage.php +++ b/src/Appwrite/CLI/Tasks/Usage.php @@ -11,6 +11,7 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Validator\WhiteList; +use Throwable; class Usage extends Task { @@ -73,13 +74,14 @@ class Usage extends Task $database = $this->getDatabase($register, '_console'); $influxDB = $this->getInfluxDB($register); + $logError = fn(Throwable $error, string $action = 'syncUsageStats') => $this->logError($register, $error, $action); switch ($type) { case 'timeseries': - $this->aggregateTimeseries($database, $influxDB, $this->logError); + $this->aggregateTimeseries($database, $influxDB, $logError); break; case 'database': - $this->aggregateDatabase($database, $this->logError); + $this->aggregateDatabase($database, $logError); break; default: Console::error("Unsupported usage aggregation type"); From 8269ec225d18dd36cc5b24b1a4a2e81349cab0ae Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 16 Oct 2022 07:10:47 +0000 Subject: [PATCH 22/50] use latest releases --- composer.json | 4 ++-- composer.lock | 57 ++++++++++++++++++++++++--------------------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/composer.json b/composer.json index 3dc7fae747..939b99aef6 100644 --- a/composer.json +++ b/composer.json @@ -43,13 +43,13 @@ "ext-sockets": "*", "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.11.*", - "utopia-php/framework": "0.21.*", + "utopia-php/framework": "0.22.*", "utopia-php/logger": "0.3.*", "utopia-php/abuse": "0.13.*", "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.14.*", "utopia-php/cache": "0.6.*", - "utopia-php/platform": "dev-feat-upgrade-framework", + "utopia-php/platform": "0.2.*", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.25.*", diff --git a/composer.lock b/composer.lock index cd38d1180f..33ff9a06b0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b55f1a94691129653bc6ff0ac5f66e13", + "content-hash": "a19fd30d9ecff10a0ab60808a1f62495", "packages": [ { "name": "adhocore/jwt", @@ -2178,24 +2178,24 @@ }, { "name": "utopia-php/framework", - "version": "0.21.1", + "version": "0.22.1", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "c81789b87a917da2daf336738170ebe01f50ea18" + "reference": "9f35d36ed4b8fa1c92962c77ef02b49c2f5919df" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/c81789b87a917da2daf336738170ebe01f50ea18", - "reference": "c81789b87a917da2daf336738170ebe01f50ea18", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/9f35d36ed4b8fa1c92962c77ef02b49c2f5919df", + "reference": "9f35d36ed4b8fa1c92962c77ef02b49c2f5919df", "shasum": "" }, "require": { "php": ">=8.0.0" }, "require-dev": { - "phpunit/phpunit": "^9.5.10", - "vimeo/psalm": "4.13.1" + "phpunit/phpunit": "^9.5.25", + "vimeo/psalm": "^4.27.0" }, "type": "library", "autoload": { @@ -2221,9 +2221,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.21.1" + "source": "https://github.com/utopia-php/framework/tree/0.22.1" }, - "time": "2022-09-07T09:56:28+00:00" + "time": "2022-10-07T14:51:40+00:00" }, { "name": "utopia-php/image", @@ -2451,18 +2451,18 @@ }, { "name": "utopia-php/platform", - "version": "dev-feat-upgrade-framework", + "version": "0.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "4c2fcb8b9f046948ab812552e4e448057efef936" + "reference": "5a3640f4f34b394ed34efda351101c7928643819" }, "require": { "ext-json": "*", "ext-redis": "*", "php": ">=8.0", "utopia-php/cli": "0.13.*", - "utopia-php/framework": "0.21.*" + "utopia-php/framework": "0.22.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -2493,12 +2493,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "Light and Fast Platform Library", "keywords": [ "cache", @@ -2507,7 +2501,7 @@ "upf", "utopia" ], - "time": "2022-09-01T11:59:42+00:00" + "time": "2022-10-16T07:08:51+00:00" }, { "name": "utopia-php/preloader", @@ -3479,25 +3473,30 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.6.1", + "version": "1.6.2", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "77a32518733312af16a44300404e945338981de3" + "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/77a32518733312af16a44300404e945338981de3", - "reference": "77a32518733312af16a44300404e945338981de3", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/48f445a408c131e38cab1c235aa6d2bb7a0bb20d", + "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d", "shasum": "" }, "require": { - "php": "^7.2 || ^8.0", + "php": "^7.4 || ^8.0", "phpdocumentor/reflection-common": "^2.0" }, "require-dev": { "ext-tokenizer": "*", - "psalm/phar": "^4.8" + "phpstan/extension-installer": "^1.1", + "phpstan/phpstan": "^1.8", + "phpstan/phpstan-phpunit": "^1.1", + "phpunit/phpunit": "^9.5", + "rector/rector": "^0.13.9", + "vimeo/psalm": "^4.25" }, "type": "library", "extra": { @@ -3523,9 +3522,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.1" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.2" }, - "time": "2022-03-15T21:29:03+00:00" + "time": "2022-10-14T12:47:21+00:00" }, { "name": "phpspec/prophecy", @@ -5420,9 +5419,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/platform": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From d7b1224628d76d66fbde1298627f790a76640011 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 16 Oct 2022 08:21:53 +0000 Subject: [PATCH 23/50] fix upgrade utopia swoole --- composer.json | 2 +- composer.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index 939b99aef6..f8eee2eca2 100644 --- a/composer.json +++ b/composer.json @@ -57,7 +57,7 @@ "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", - "utopia-php/swoole": "0.3.*", + "utopia-php/swoole": "0.4.*", "utopia-php/storage": "0.11.*", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", diff --git a/composer.lock b/composer.lock index 33ff9a06b0..f845b953ac 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a19fd30d9ecff10a0ab60808a1f62495", + "content-hash": "c2fa5b94159eb9fc1e0d414ae507ba9e", "packages": [ { "name": "adhocore/jwt", @@ -2665,16 +2665,16 @@ }, { "name": "utopia-php/swoole", - "version": "0.3.3", + "version": "0.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "8312df69233b5dcd3992de88f131f238002749de" + "reference": "536e1f3e78fc0197e4a8ed81b1bf2636a3bc4538" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/8312df69233b5dcd3992de88f131f238002749de", - "reference": "8312df69233b5dcd3992de88f131f238002749de", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/536e1f3e78fc0197e4a8ed81b1bf2636a3bc4538", + "reference": "536e1f3e78fc0197e4a8ed81b1bf2636a3bc4538", "shasum": "" }, "require": { @@ -2715,9 +2715,9 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.3.3" + "source": "https://github.com/utopia-php/swoole/tree/0.4.0" }, - "time": "2022-01-20T09:58:43+00:00" + "time": "2022-10-08T14:32:43+00:00" }, { "name": "utopia-php/system", From 904b8aca8d90f45422042937da36646e27791123 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 02:37:39 +0000 Subject: [PATCH 24/50] latest cli version --- composer.json | 6 ++--- composer.lock | 73 ++++++++++++++++++++++++--------------------------- 2 files changed, 38 insertions(+), 41 deletions(-) diff --git a/composer.json b/composer.json index f52586277a..9bc0cfdeda 100644 --- a/composer.json +++ b/composer.json @@ -49,8 +49,8 @@ "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.15.*", "utopia-php/cache": "0.6.*", - "utopia-php/platform": "0.2.*", - "utopia-php/cli": "0.13.*", + "utopia-php/platform": "dev-feat-cli-update", + "utopia-php/cli": "0.14.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.26.*", "utopia-php/locale": "0.4.*", @@ -61,7 +61,7 @@ "utopia-php/storage": "0.11.*", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", - "utopia-php/orchestration": "0.6.*", + "utopia-php/orchestration": "dev-feat-cli-update", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", diff --git a/composer.lock b/composer.lock index 1f69ffffca..72180816f0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3bcc6cfb6966fef9fdba36dd31f07b5b", + "content-hash": "32ae4fa68179e9a5e4b18840ab9db57f", "packages": [ { "name": "adhocore/jwt", @@ -693,16 +693,16 @@ }, { "name": "guzzlehttp/psr7", - "version": "2.4.1", + "version": "2.4.3", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379" + "reference": "67c26b443f348a51926030c83481b85718457d3d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/69568e4293f4fa993f3b0e51c9723e1e17c41379", - "reference": "69568e4293f4fa993f3b0e51c9723e1e17c41379", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/67c26b443f348a51926030c83481b85718457d3d", + "reference": "67c26b443f348a51926030c83481b85718457d3d", "shasum": "" }, "require": { @@ -792,7 +792,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.4.1" + "source": "https://github.com/guzzle/psr7/tree/2.4.3" }, "funding": [ { @@ -808,7 +808,7 @@ "type": "tidelift" } ], - "time": "2022-08-28T14:45:39+00:00" + "time": "2022-10-26T14:07:24+00:00" }, { "name": "influxdb/influxdb-php", @@ -1950,16 +1950,16 @@ }, { "name": "utopia-php/cli", - "version": "0.13.0", + "version": "0.14.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "69e68f8ed525fe162fae950a0507ed28a0f179bc" + "reference": "c30ef985a4e739758a0d95eb0706b357b6d8c086" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/69e68f8ed525fe162fae950a0507ed28a0f179bc", - "reference": "69e68f8ed525fe162fae950a0507ed28a0f179bc", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/c30ef985a4e739758a0d95eb0706b357b6d8c086", + "reference": "c30ef985a4e739758a0d95eb0706b357b6d8c086", "shasum": "" }, "require": { @@ -1968,7 +1968,7 @@ }, "require-dev": { "phpunit/phpunit": "^9.3", - "vimeo/psalm": "4.0.1" + "squizlabs/php_codesniffer": "^3.6" }, "type": "library", "autoload": { @@ -1997,9 +1997,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.13.0" + "source": "https://github.com/utopia-php/cli/tree/0.14.0" }, - "time": "2022-04-26T08:41:22+00:00" + "time": "2022-10-09T10:19:07+00:00" }, { "name": "utopia-php/config", @@ -2390,21 +2390,21 @@ }, { "name": "utopia-php/orchestration", - "version": "0.6.0", + "version": "dev-feat-cli-update", "source": { "type": "git", "url": "https://github.com/utopia-php/orchestration.git", - "reference": "94263976413871efb6b16157a7101a81df3b6d78" + "reference": "dd6d30e70d3d71e49a10f6281fc8e7c0b9fd901a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/94263976413871efb6b16157a7101a81df3b6d78", - "reference": "94263976413871efb6b16157a7101a81df3b6d78", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/dd6d30e70d3d71e49a10f6281fc8e7c0b9fd901a", + "reference": "dd6d30e70d3d71e49a10f6281fc8e7c0b9fd901a", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/cli": "0.13.*" + "utopia-php/cli": "0.14.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -2420,12 +2420,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "Lite & fast micro PHP abstraction library for container orchestration", "keywords": [ "docker", @@ -2439,23 +2433,23 @@ ], "support": { "issues": "https://github.com/utopia-php/orchestration/issues", - "source": "https://github.com/utopia-php/orchestration/tree/0.6.0" + "source": "https://github.com/utopia-php/orchestration/tree/feat-cli-update" }, - "time": "2022-07-13T16:47:18+00:00" + "time": "2022-10-28T02:36:14+00:00" }, { "name": "utopia-php/platform", - "version": "0.2.0", + "version": "dev-feat-cli-update", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "5a3640f4f34b394ed34efda351101c7928643819" + "reference": "accc17f6dd6ae404f86691e768dd73eecb0778e0" }, "require": { "ext-json": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/cli": "0.13.*", + "utopia-php/cli": "0.14.*", "utopia-php/framework": "0.22.*" }, "require-dev": { @@ -2495,7 +2489,7 @@ "upf", "utopia" ], - "time": "2022-10-16T07:08:51+00:00" + "time": "2022-10-28T02:27:04+00:00" }, { "name": "utopia-php/preloader", @@ -3589,16 +3583,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.17", + "version": "9.2.18", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8" + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/aa94dc41e8661fe90c7316849907cba3007b10d8", - "reference": "aa94dc41e8661fe90c7316849907cba3007b10d8", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/12fddc491826940cf9b7e88ad9664cf51f0f6d0a", + "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a", "shasum": "" }, "require": { @@ -3654,7 +3648,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.17" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.18" }, "funding": [ { @@ -3662,7 +3656,7 @@ "type": "github" } ], - "time": "2022-08-30T12:24:04+00:00" + "time": "2022-10-27T13:35:33+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5413,7 +5407,10 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/platform": 20, + "utopia-php/orchestration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From f080bd3e68477c77f75559bdb35954f9ce16bbc0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 03:36:20 +0000 Subject: [PATCH 25/50] using dependency injection in CLI --- app/cli.php | 41 +++++++++++++++++++++++++++++++ composer.lock | 4 +-- src/Appwrite/CLI/Tasks/Doctor.php | 8 +++--- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/app/cli.php b/app/cli.php index ef0445e67f..3060adc636 100644 --- a/app/cli.php +++ b/app/cli.php @@ -4,11 +4,52 @@ require_once __DIR__ . '/init.php'; require_once __DIR__ . '/controllers/general.php'; use Appwrite\CLI\Tasks; +use Utopia\CLI\CLI; use Utopia\Database\Validator\Authorization; use Utopia\Platform\Service; +use Utopia\App; +use Utopia\CLI\Console; +use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Cache; +use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Database; Authorization::disable(); +CLI::setResource('register', fn()=>$register); + +$attempts = 0; +$max = 10; +$sleep = 1; + +do { + try { + $attempts++; + $db = $register->get('db'); + $redis = $register->get('cache'); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep($sleep); + } +} while ($attempts < $max); + +CLI::setResource('db', fn () => $db); +CLI::setResource('cache', fn () => $redis); + +CLI::setResource('dbForConsole', function ($db, $cache) { + $cache = new Cache(new RedisCache($cache)); + + $database = new Database(new MariaDB($db), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace('_console'); + + return $database; +}, ['db', 'cache']); + $cliPlatform = new Tasks(); $cliPlatform->init(Service::TYPE_CLI); diff --git a/composer.lock b/composer.lock index 72180816f0..416a5c1376 100644 --- a/composer.lock +++ b/composer.lock @@ -2443,7 +2443,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "accc17f6dd6ae404f86691e768dd73eecb0778e0" + "reference": "07e825132492c7d7c9920dbeb527833bda21155d" }, "require": { "ext-json": "*", @@ -2489,7 +2489,7 @@ "upf", "utopia" ], - "time": "2022-10-28T02:27:04+00:00" + "time": "2022-10-28T03:15:57+00:00" }, { "name": "utopia-php/preloader", diff --git a/src/Appwrite/CLI/Tasks/Doctor.php b/src/Appwrite/CLI/Tasks/Doctor.php index 85d389b846..2b728b47c8 100644 --- a/src/Appwrite/CLI/Tasks/Doctor.php +++ b/src/Appwrite/CLI/Tasks/Doctor.php @@ -10,6 +10,7 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; use Utopia\Domains\Domain; use Utopia\Platform\Action; +use Utopia\Registry\Registry; class Doctor extends Action { @@ -22,13 +23,12 @@ class Doctor extends Action { $this ->desc('Validate server health') - ->callback(fn () => $this->action()); + ->inject('register') + ->callback(fn (Registry $register) => $this->action($register)); } - public function action(): void + public function action(Registry $register): void { - global $register; - Console::log(" __ ____ ____ _ _ ____ __ ____ ____ __ __ / _\ ( _ \( _ \/ )( \( _ \( )(_ _)( __) ( )/ \ / \ ) __/ ) __/\ /\ / ) / )( )( ) _) _ )(( O ) From 592b2dcd978f4a3fdd53eba7fbfa613c72815f70 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 03:57:59 +0000 Subject: [PATCH 26/50] use injection --- src/Appwrite/CLI/Tasks/Maintenance.php | 42 +++----------------------- 1 file changed, 5 insertions(+), 37 deletions(-) diff --git a/src/Appwrite/CLI/Tasks/Maintenance.php b/src/Appwrite/CLI/Tasks/Maintenance.php index 252dc25a93..85973e6500 100644 --- a/src/Appwrite/CLI/Tasks/Maintenance.php +++ b/src/Appwrite/CLI/Tasks/Maintenance.php @@ -23,45 +23,15 @@ class Maintenance extends Action return 'maintenance'; } - protected function getConsoleDB(): Database - { - global $register; - - $attempts = 0; - - do { - try { - $attempts++; - $cache = new Cache(new RedisCache($register->get('cache'))); - $database = new Database(new MariaDB($register->get('db')), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); // Main DB - - if (!$database->exists($database->getDefaultDatabase(), 'certificates')) { - throw new \Exception('Console project not ready'); - } - - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; - } - public function __construct() { $this ->desc('Schedules maintenance tasks and publishes them to resque') - ->callback(fn () => $this->action()); + ->inject('dbForConsole') + ->callback(fn (Database $dbForConsole) => $this->action($dbForConsole)); } - public function action(): void + public function action(Database $dbForConsole): void { Console::title('Maintenance V1'); Console::success(APP_NAME . ' maintenance process v1 has started'); @@ -160,9 +130,7 @@ class Maintenance extends Action $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) { - $database = $this->getConsoleDB(); - + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention, $dbForConsole) { $time = DateTime::now(); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); @@ -172,7 +140,7 @@ class Maintenance extends Action notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); notifyDeleteConnections(); notifyDeleteExpiredSessions(); - renewCertificates($database); + renewCertificates($dbForConsole); notifyDeleteCache($cacheRetention); }, $interval); } From 85794beb959e10215db5efaade2c9032badecc49 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 04:00:44 +0000 Subject: [PATCH 27/50] inject dependency for spec --- src/Appwrite/CLI/Tasks/Specs.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/CLI/Tasks/Specs.php b/src/Appwrite/CLI/Tasks/Specs.php index ebca58abf6..dab6827b33 100644 --- a/src/Appwrite/CLI/Tasks/Specs.php +++ b/src/Appwrite/CLI/Tasks/Specs.php @@ -12,6 +12,7 @@ use Swoole\Http\Response as HttpResponse; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; +use Utopia\Registry\Registry; use Utopia\Request; use Utopia\Validator\WhiteList; @@ -28,13 +29,12 @@ class Specs extends Action ->desc('Generate Appwrite API specifications') ->param('version', 'latest', new Text(16), 'Spec version', true) ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true) - ->callback(fn ($version, $mode) => $this->action($version, $mode)); + ->inject('register') + ->callback(fn (string $version, string $mode, Registry $register) => $this->action($version, $mode, $register)); } - public function action(string $version, string $mode): void + public function action(string $version, string $mode, Registry $register): void { - global $register; - $db = $register->get('db'); $redis = $register->get('cache'); $appRoutes = App::getRoutes(); From 30a541a5c996de4dead62e8cd6fffbcee9ac1044 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 04:09:34 +0000 Subject: [PATCH 28/50] dependency injection in usage task --- app/cli.php | 23 ++++++++++++ src/Appwrite/CLI/Tasks/Usage.php | 19 +++++----- src/Appwrite/Platform/Task.php | 62 ++------------------------------ 3 files changed, 35 insertions(+), 69 deletions(-) diff --git a/app/cli.php b/app/cli.php index 3060adc636..0dd4fad71e 100644 --- a/app/cli.php +++ b/app/cli.php @@ -50,6 +50,29 @@ CLI::setResource('dbForConsole', function ($db, $cache) { return $database; }, ['db', 'cache']); +/** @var InfluxDB\Client $client */ +$client = $register->get('influxdb'); +$attempts = 0; +$max = 10; +$sleep = 1; + +do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if (in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful + } + } catch (\Throwable$th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } +} while ($attempts < $max); +CLI::setResource('influxdb', fn() => $database); + $cliPlatform = new Tasks(); $cliPlatform->init(Service::TYPE_CLI); diff --git a/src/Appwrite/CLI/Tasks/Usage.php b/src/Appwrite/CLI/Tasks/Usage.php index d5360e1091..3ac9ec0d51 100644 --- a/src/Appwrite/CLI/Tasks/Usage.php +++ b/src/Appwrite/CLI/Tasks/Usage.php @@ -12,6 +12,7 @@ use Utopia\CLI\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Validator\WhiteList; use Throwable; +use Utopia\Registry\Registry; class Usage extends Task { @@ -23,9 +24,12 @@ class Usage extends Task public function __construct() { $this - ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->callback(fn ($type) => $this->action($type)); + ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) + ->inject('dbForConsole') + ->inject('influxdb') + ->inject('register') + ->callback(fn ($type, $dbForConsole, $influxDB, $register) => $this->action($type, $dbForConsole, $influxDB, $register)); } @@ -66,22 +70,19 @@ class Usage extends Task }, $interval); } - public function action(string $type) + public function action(string $type, UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register) { - global $register; Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $database = $this->getDatabase($register, '_console'); - $influxDB = $this->getInfluxDB($register); - $logError = fn(Throwable $error, string $action = 'syncUsageStats') => $this->logError($register, $error, $action); + $logError = fn(Throwable $error, string $action = 'syncUsageStats') => $this->logError($register, $error, "usage", $action); switch ($type) { case 'timeseries': - $this->aggregateTimeseries($database, $influxDB, $logError); + $this->aggregateTimeseries($dbForConsole, $influxDB, $logError); break; case 'database': - $this->aggregateDatabase($database, $logError); + $this->aggregateDatabase($dbForConsole, $logError); break; default: Console::error("Unsupported usage aggregation type"); diff --git a/src/Appwrite/Platform/Task.php b/src/Appwrite/Platform/Task.php index 21c3f53159..23f73c222b 100644 --- a/src/Appwrite/Platform/Task.php +++ b/src/Appwrite/Platform/Task.php @@ -17,7 +17,7 @@ use Throwable; abstract class Task extends Action { - protected function logError(Registry $register, Throwable $error, string $action = 'syncUsageStats') + protected function logError(Registry $register, Throwable $error, string $namespace, string $action) { $logger = $register->get('logger'); @@ -25,7 +25,7 @@ abstract class Task extends Action $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $log = new Log(); - $log->setNamespace("usage"); + $log->setNamespace($namespace); $log->setServer(\gethostname()); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); @@ -51,62 +51,4 @@ abstract class Task extends Action Console::warning("Failed: {$error->getMessage()}"); Console::warning($error->getTraceAsString()); } - - protected function getDatabase(Registry &$register, string $namespace): Database - { - $attempts = 0; - - do { - try { - $attempts++; - - $db = $register->get('db'); - $redis = $register->get('cache'); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); - - if (!$database->exists($database->getDefaultDatabase(), 'projects')) { - throw new Exception('Projects collection not ready'); - } - break; // leave loop if successful - } catch (\Exception$e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; - } - - protected function getInfluxDB(Registry &$register): InfluxDatabase - { - /** @var InfluxDB\Client $client */ - $client = $register->get('influxdb'); - $attempts = 0; - $max = 10; - $sleep = 1; - - do { // check if telegraf database is ready - try { - $attempts++; - $database = $client->selectDB('telegraf'); - if (in_array('telegraf', $client->listDatabases())) { - break; // leave the do-while if successful - } - } catch (\Throwable$th) { - Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('InfluxDB database not ready yet'); - } - sleep($sleep); - } - } while ($attempts < $max); - return $database; - } } From dedd24254a857ef476ad38ce47a232fd5efe4f2f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 07:35:03 +0000 Subject: [PATCH 29/50] update depending libraries --- composer.json | 4 ++-- composer.lock | 25 +++++++++++-------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/composer.json b/composer.json index 9bc0cfdeda..b47b8c79a8 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.15.*", "utopia-php/cache": "0.6.*", - "utopia-php/platform": "dev-feat-cli-update", + "utopia-php/platform": "0.3.*", "utopia-php/cli": "0.14.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.26.*", @@ -61,7 +61,7 @@ "utopia-php/storage": "0.11.*", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", - "utopia-php/orchestration": "dev-feat-cli-update", + "utopia-php/orchestration": "0.7.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", diff --git a/composer.lock b/composer.lock index 416a5c1376..5ce55529cd 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "32ae4fa68179e9a5e4b18840ab9db57f", + "content-hash": "f767c5ed523160615244a04e770a7f79", "packages": [ { "name": "adhocore/jwt", @@ -2390,16 +2390,16 @@ }, { "name": "utopia-php/orchestration", - "version": "dev-feat-cli-update", + "version": "0.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/orchestration.git", - "reference": "dd6d30e70d3d71e49a10f6281fc8e7c0b9fd901a" + "reference": "484df2c9275a77722fe05f3630cf0e346dcb372e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/dd6d30e70d3d71e49a10f6281fc8e7c0b9fd901a", - "reference": "dd6d30e70d3d71e49a10f6281fc8e7c0b9fd901a", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/484df2c9275a77722fe05f3630cf0e346dcb372e", + "reference": "484df2c9275a77722fe05f3630cf0e346dcb372e", "shasum": "" }, "require": { @@ -2433,17 +2433,17 @@ ], "support": { "issues": "https://github.com/utopia-php/orchestration/issues", - "source": "https://github.com/utopia-php/orchestration/tree/feat-cli-update" + "source": "https://github.com/utopia-php/orchestration/tree/0.7.0" }, - "time": "2022-10-28T02:36:14+00:00" + "time": "2022-10-28T07:23:38+00:00" }, { "name": "utopia-php/platform", - "version": "dev-feat-cli-update", + "version": "0.3.0", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "07e825132492c7d7c9920dbeb527833bda21155d" + "reference": "9cebec37b96e9f38c1cc0a68a793304606423897" }, "require": { "ext-json": "*", @@ -2489,7 +2489,7 @@ "upf", "utopia" ], - "time": "2022-10-28T03:15:57+00:00" + "time": "2022-10-28T07:26:09+00:00" }, { "name": "utopia-php/preloader", @@ -5407,10 +5407,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/platform": 20, - "utopia-php/orchestration": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 14aca37d2ef16c8efcfdf91577b4bab91adff51a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 07:36:02 +0000 Subject: [PATCH 30/50] inject register on migrate --- src/Appwrite/CLI/Tasks/Migrate.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/CLI/Tasks/Migrate.php b/src/Appwrite/CLI/Tasks/Migrate.php index e9f0f0dec3..7e36247a55 100644 --- a/src/Appwrite/CLI/Tasks/Migrate.php +++ b/src/Appwrite/CLI/Tasks/Migrate.php @@ -12,6 +12,7 @@ use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -26,12 +27,12 @@ class Migrate extends Action $this ->desc('Migrate Appwrite to new version') ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) - ->callback(fn ($version) => $this->action($version)); + ->inject('register') + ->callback(fn ($version, $register) => $this->action($version, $register)); } - public function action($version) + public function action(string $version, Registry $register) { - global $register; Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { Console::error("Version {$version} not found."); From e7a7267c9fd7022e9c5dad3c9d7bd325b3927450 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 08:49:05 +0000 Subject: [PATCH 31/50] move resource code inside the callback --- app/cli.php | 82 ++++++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/app/cli.php b/app/cli.php index 0dd4fad71e..d0b0cf66a1 100644 --- a/app/cli.php +++ b/app/cli.php @@ -13,31 +13,33 @@ use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; +use Utopia\Registry\Registry; Authorization::disable(); CLI::setResource('register', fn()=>$register); -$attempts = 0; -$max = 10; -$sleep = 1; - -do { - try { - $attempts++; - $db = $register->get('db'); - $redis = $register->get('cache'); - break; // leave the do-while if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); +CLI::setResource('db', function(Registry $register) { + $attempts = 0; + $max = 10; + $sleep = 1; + do { + try { + $attempts++; + $db = $register->get('db'); + $redis = $register->get('cache'); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep($sleep); } - sleep($sleep); - } -} while ($attempts < $max); + } while ($attempts < $max); + return $db; +}, ['register']); -CLI::setResource('db', fn () => $db); CLI::setResource('cache', fn () => $redis); CLI::setResource('dbForConsole', function ($db, $cache) { @@ -50,28 +52,30 @@ CLI::setResource('dbForConsole', function ($db, $cache) { return $database; }, ['db', 'cache']); -/** @var InfluxDB\Client $client */ -$client = $register->get('influxdb'); -$attempts = 0; -$max = 10; -$sleep = 1; - -do { // check if telegraf database is ready - try { - $attempts++; - $database = $client->selectDB('telegraf'); - if (in_array('telegraf', $client->listDatabases())) { - break; // leave the do-while if successful +CLI::setResource('influxdb', function(Registry $register){ + /** @var InfluxDB\Client $client */ + $client = $register->get('influxdb'); + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if (in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful + } + } catch (\Throwable$th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); } - } catch (\Throwable$th) { - Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('InfluxDB database not ready yet'); - } - sleep($sleep); - } -} while ($attempts < $max); -CLI::setResource('influxdb', fn() => $database); + } while ($attempts < $max); + return $database; +}, ['register']); $cliPlatform = new Tasks(); $cliPlatform->init(Service::TYPE_CLI); From 4acd10f2e1e29a8de387444107811a2373b0e4a3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 08:56:45 +0000 Subject: [PATCH 32/50] error logger in resources --- app/cli.php | 38 ++++++++++++++++++++++++++++++ src/Appwrite/CLI/Tasks/Usage.php | 11 +++++---- src/Appwrite/Platform/Task.php | 40 +------------------------------- 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/app/cli.php b/app/cli.php index d0b0cf66a1..dc2e62c88c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -13,6 +13,7 @@ use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; +use Utopia\Logger\Log; use Utopia\Registry\Registry; Authorization::disable(); @@ -77,6 +78,43 @@ CLI::setResource('influxdb', function(Registry $register){ return $database; }, ['register']); +CLI::setResource('logError', function(Registry $register){ + return function(Throwable $error, string $namespace, string $action) use($register) + { + $logger = $register->get('logger'); + + if ($logger) { + $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace($namespace); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + + $log->setAction($action); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $responseCode = $logger->addLog($log); + Console::info('Usage stats log pushed with status code: ' . $responseCode); + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); + }; +}); + $cliPlatform = new Tasks(); $cliPlatform->init(Service::TYPE_CLI); diff --git a/src/Appwrite/CLI/Tasks/Usage.php b/src/Appwrite/CLI/Tasks/Usage.php index 3ac9ec0d51..fbd7d38277 100644 --- a/src/Appwrite/CLI/Tasks/Usage.php +++ b/src/Appwrite/CLI/Tasks/Usage.php @@ -29,7 +29,8 @@ class Usage extends Task ->inject('dbForConsole') ->inject('influxdb') ->inject('register') - ->callback(fn ($type, $dbForConsole, $influxDB, $register) => $this->action($type, $dbForConsole, $influxDB, $register)); + ->inject('logError') + ->callback(fn ($type, $dbForConsole, $influxDB, $register, $logError) => $this->action($type, $dbForConsole, $influxDB, $register, $logError)); } @@ -70,19 +71,19 @@ class Usage extends Task }, $interval); } - public function action(string $type, UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register) + public function action(string $type, UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $logError) { Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $logError = fn(Throwable $error, string $action = 'syncUsageStats') => $this->logError($register, $error, "usage", $action); + $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($register, $error, "usage", $action); switch ($type) { case 'timeseries': - $this->aggregateTimeseries($dbForConsole, $influxDB, $logError); + $this->aggregateTimeseries($dbForConsole, $influxDB, $errorLogger); break; case 'database': - $this->aggregateDatabase($dbForConsole, $logError); + $this->aggregateDatabase($dbForConsole, $errorLogger); break; default: Console::error("Unsupported usage aggregation type"); diff --git a/src/Appwrite/Platform/Task.php b/src/Appwrite/Platform/Task.php index 23f73c222b..1376ab1d9f 100644 --- a/src/Appwrite/Platform/Task.php +++ b/src/Appwrite/Platform/Task.php @@ -2,53 +2,15 @@ namespace Appwrite\Platform; -use Utopia\Database\Database; use Utopia\Platform\Action; use Exception; -use InfluxDB\Database as InfluxDatabase; use Utopia\App; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; use Utopia\CLI\Console; -use Utopia\Database\Adapter\MariaDB; use Utopia\Registry\Registry; use Utopia\Logger\Log; use Throwable; abstract class Task extends Action { - protected function logError(Registry $register, Throwable $error, string $namespace, string $action) - { - $logger = $register->get('logger'); - - if ($logger) { - $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); - - $log = new Log(); - $log->setNamespace($namespace); - $log->setServer(\gethostname()); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - - $log->addTag('code', $error->getCode()); - $log->addTag('verboseType', get_class($error)); - - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('detailedTrace', $error->getTrace()); - - $log->setAction($action); - - $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - $responseCode = $logger->addLog($log); - Console::info('Usage stats log pushed with status code: ' . $responseCode); - } - - Console::warning("Failed: {$error->getMessage()}"); - Console::warning($error->getTraceAsString()); - } + } From 5a0a197355e868b7f41bae45df1ca0d9db17cc76 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 08:58:09 +0000 Subject: [PATCH 33/50] composer update --- composer.json | 4 ---- composer.lock | 31 +++++++++++++------------------ 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/composer.json b/composer.json index b47b8c79a8..092f6131dc 100644 --- a/composer.json +++ b/composer.json @@ -75,10 +75,6 @@ { "url": "https://github.com/appwrite/runtimes.git", "type": "git" - }, - { - "url": "https://github.com/utopia-php/platform.git", - "type": "git" } ], "require-dev": { diff --git a/composer.lock b/composer.lock index 97849946dd..b3fa99e3fe 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f767c5ed523160615244a04e770a7f79", + "content-hash": "1c3cf7fd488e2eb61af0ddad1f43a446", "packages": [ { "name": "adhocore/jwt", @@ -2445,6 +2445,12 @@ "url": "https://github.com/utopia-php/platform.git", "reference": "9cebec37b96e9f38c1cc0a68a793304606423897" }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/9cebec37b96e9f38c1cc0a68a793304606423897", + "reference": "9cebec37b96e9f38c1cc0a68a793304606423897", + "shasum": "" + }, "require": { "ext-json": "*", "ext-redis": "*", @@ -2462,22 +2468,7 @@ "Utopia\\Platform\\": "src/Platform" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Tests\\": "tests/Platform" - } - }, - "scripts": { - "format": [ - "vendor/bin/phpcbf" - ], - "lint": [ - "vendor/bin/phpcs" - ], - "test": [ - "docker-compose up -d && sleep 10 && docker-compose exec web vendor/bin/phpunit --configuration phpunit.xml" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -2489,6 +2480,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/platform/issues", + "source": "https://github.com/utopia-php/platform/tree/0.3.0" + }, "time": "2022-10-28T07:26:09+00:00" }, { @@ -5431,5 +5426,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } From 07a2aed52fd6562c2bf421e7484db7da17dc397e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 28 Oct 2022 09:00:59 +0000 Subject: [PATCH 34/50] fix formatting --- app/cli.php | 11 +++++------ src/Appwrite/Platform/Task.php | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/app/cli.php b/app/cli.php index dc2e62c88c..2c5bbc8d30 100644 --- a/app/cli.php +++ b/app/cli.php @@ -20,7 +20,7 @@ Authorization::disable(); CLI::setResource('register', fn()=>$register); -CLI::setResource('db', function(Registry $register) { +CLI::setResource('db', function (Registry $register) { $attempts = 0; $max = 10; $sleep = 1; @@ -53,13 +53,13 @@ CLI::setResource('dbForConsole', function ($db, $cache) { return $database; }, ['db', 'cache']); -CLI::setResource('influxdb', function(Registry $register){ +CLI::setResource('influxdb', function (Registry $register) { /** @var InfluxDB\Client $client */ $client = $register->get('influxdb'); $attempts = 0; $max = 10; $sleep = 1; - + do { // check if telegraf database is ready try { $attempts++; @@ -78,9 +78,8 @@ CLI::setResource('influxdb', function(Registry $register){ return $database; }, ['register']); -CLI::setResource('logError', function(Registry $register){ - return function(Throwable $error, string $namespace, string $action) use($register) - { +CLI::setResource('logError', function (Registry $register) { + return function (Throwable $error, string $namespace, string $action) use ($register) { $logger = $register->get('logger'); if ($logger) { diff --git a/src/Appwrite/Platform/Task.php b/src/Appwrite/Platform/Task.php index 1376ab1d9f..d1eb7fa88b 100644 --- a/src/Appwrite/Platform/Task.php +++ b/src/Appwrite/Platform/Task.php @@ -12,5 +12,4 @@ use Throwable; abstract class Task extends Action { - } From 826a9540d057a951c98121cb400ed32ce13fb20a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 30 Oct 2022 05:36:31 +0000 Subject: [PATCH 35/50] fix cache --- app/cli.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/cli.php b/app/cli.php index 2c5bbc8d30..c80ec6df06 100644 --- a/app/cli.php +++ b/app/cli.php @@ -28,7 +28,6 @@ CLI::setResource('db', function (Registry $register) { try { $attempts++; $db = $register->get('db'); - $redis = $register->get('cache'); break; // leave the do-while if successful } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); @@ -41,7 +40,7 @@ CLI::setResource('db', function (Registry $register) { return $db; }, ['register']); -CLI::setResource('cache', fn () => $redis); +CLI::setResource('cache', fn ($register) => $register->get('cache'), ['register']); CLI::setResource('dbForConsole', function ($db, $cache) { $cache = new Cache(new RedisCache($cache)); From f56a29476090d69b5fe6dd741e41a9664d4c0b92 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 30 Oct 2022 05:38:22 +0000 Subject: [PATCH 36/50] fix error logger callback --- app/cli.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/cli.php b/app/cli.php index c80ec6df06..a1a080fc31 100644 --- a/app/cli.php +++ b/app/cli.php @@ -111,7 +111,7 @@ CLI::setResource('logError', function (Registry $register) { Console::warning("Failed: {$error->getMessage()}"); Console::warning($error->getTraceAsString()); }; -}); +}, ['register']); $cliPlatform = new Tasks(); $cliPlatform->init(Service::TYPE_CLI); From a74ddc70261e3ed52c8f30befeeb2789988ebfa2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 31 Oct 2022 08:02:32 +0000 Subject: [PATCH 37/50] composer update --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index b3fa99e3fe..c3b63a95a0 100644 --- a/composer.lock +++ b/composer.lock @@ -5426,5 +5426,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } From 9cdce06d96bade481d974d84ad29cad9bfbf9110 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 2 Nov 2022 06:23:31 +0000 Subject: [PATCH 38/50] fix usage task --- src/Appwrite/CLI/Tasks/Usage.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/CLI/Tasks/Usage.php b/src/Appwrite/CLI/Tasks/Usage.php index fbd7d38277..e3e9db865a 100644 --- a/src/Appwrite/CLI/Tasks/Usage.php +++ b/src/Appwrite/CLI/Tasks/Usage.php @@ -28,9 +28,8 @@ class Usage extends Task ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->inject('dbForConsole') ->inject('influxdb') - ->inject('register') ->inject('logError') - ->callback(fn ($type, $dbForConsole, $influxDB, $register, $logError) => $this->action($type, $dbForConsole, $influxDB, $register, $logError)); + ->callback(fn ($type, $dbForConsole, $influxDB, $logError) => $this->action($type, $dbForConsole, $influxDB, $logError)); } @@ -71,12 +70,12 @@ class Usage extends Task }, $interval); } - public function action(string $type, UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $logError) + public function action(string $type, UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, callable $logError) { Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($register, $error, "usage", $action); + $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action); switch ($type) { case 'timeseries': From f5e66a96e7efc38994c32c46f0268a0e04c9f7bd Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 13 Nov 2022 08:36:31 +0000 Subject: [PATCH 39/50] fix volume sync task --- .../Tasks/{volume-sync.php => VolumeSync.php} | 30 ++++++++++++++----- src/Appwrite/CLI/TasksService.php | 2 ++ 2 files changed, 24 insertions(+), 8 deletions(-) rename src/Appwrite/CLI/Tasks/{volume-sync.php => VolumeSync.php} (58%) diff --git a/src/Appwrite/CLI/Tasks/volume-sync.php b/src/Appwrite/CLI/Tasks/VolumeSync.php similarity index 58% rename from src/Appwrite/CLI/Tasks/volume-sync.php rename to src/Appwrite/CLI/Tasks/VolumeSync.php index 8d5fec201b..3fd84c7d22 100644 --- a/src/Appwrite/CLI/Tasks/volume-sync.php +++ b/src/Appwrite/CLI/Tasks/VolumeSync.php @@ -4,16 +4,29 @@ global $cli; use Utopia\CLI\Console; use Utopia\Database\DateTime; +use Utopia\Platform\Action; use Utopia\Validator\Integer; use Utopia\Validator\Text; -$cli - ->task('volume-sync') - ->desc('Runs rsync to sync certificates between the storage mount and traefik.') - ->param('source', null, new Text(255), 'Source path to sync from.', false) - ->param('destination', null, new Text(255), 'Destination path to sync to.', false) - ->param('interval', null, new Integer(true), 'Interval to run rsync', false) - ->action(function ($source, $destination, $interval) { +class VolumeSync extends Action +{ + public static function getName(): string + { + return 'volume-sync'; + } + + public function __construct() + { + $this + ->desc('Runs rsync to sync certificates between the storage mount and traefik.') + ->param('source', null, new Text(255), 'Source path to sync from.', false) + ->param('destination', null, new Text(255), 'Destination path to sync to.', false) + ->param('interval', null, new Integer(true), 'Interval to run rsync', false) + ->callback(fn ($source, $destination, $interval) => $this->action($source, $destination, $interval)); + } + + public function action(string $source, string $destination, int $interval) + { Console::title('RSync V1'); Console::success(APP_NAME . ' rsync process v1 has started'); @@ -42,4 +55,5 @@ $cli Console::success($stdout); Console::error($stderr); }, $interval); - }); + } +} diff --git a/src/Appwrite/CLI/TasksService.php b/src/Appwrite/CLI/TasksService.php index c6dba81dbc..12617ec33f 100644 --- a/src/Appwrite/CLI/TasksService.php +++ b/src/Appwrite/CLI/TasksService.php @@ -13,6 +13,7 @@ use Appwrite\CLI\Tasks\SSL; use Appwrite\CLI\Tasks\Usage; use Appwrite\CLI\Tasks\Vars; use Appwrite\CLI\Tasks\Version; +use VolumeSync; class TasksService extends Service { @@ -29,6 +30,7 @@ class TasksService extends Service ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) + ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()); } } From 2dea0ec3b8c6027721ba459840927f14bf504ad4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 13 Nov 2022 10:29:53 +0000 Subject: [PATCH 40/50] use latest framework version --- composer.json | 2 +- composer.lock | 34 ++++++++++++++-------------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/composer.json b/composer.json index 40e17b5fe6..25ab2a3f86 100644 --- a/composer.json +++ b/composer.json @@ -44,7 +44,7 @@ "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.11.*", "utopia-php/platform": "0.3.*", - "utopia-php/framework": "0.22.*", + "utopia-php/framework": "0.25.*", "utopia-php/logger": "0.3.*", "utopia-php/abuse": "0.16.*", "utopia-php/analytics": "0.2.*", diff --git a/composer.lock b/composer.lock index a816d587b6..a66a959b7c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "87af69017761fb788e8be70ab5c37bba", + "content-hash": "bd99fe9f008b3bf3a68b1e3d0b515cc9", "packages": [ { "name": "adhocore/jwt", @@ -1945,16 +1945,16 @@ }, { "name": "utopia-php/framework", - "version": "0.22.1", + "version": "0.25.0", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "9f35d36ed4b8fa1c92962c77ef02b49c2f5919df" + "reference": "c524f681254255c8204fbf7919c53bf3b4982636" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/9f35d36ed4b8fa1c92962c77ef02b49c2f5919df", - "reference": "9f35d36ed4b8fa1c92962c77ef02b49c2f5919df", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/c524f681254255c8204fbf7919c53bf3b4982636", + "reference": "c524f681254255c8204fbf7919c53bf3b4982636", "shasum": "" }, "require": { @@ -1974,12 +1974,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple, light and advanced PHP framework", "keywords": [ "framework", @@ -1988,9 +1982,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.22.1" + "source": "https://github.com/utopia-php/framework/tree/0.25.0" }, - "time": "2022-10-07T14:51:40+00:00" + "time": "2022-11-02T09:49:57+00:00" }, { "name": "utopia-php/image", @@ -2212,16 +2206,16 @@ }, { "name": "utopia-php/platform", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "9cebec37b96e9f38c1cc0a68a793304606423897" + "reference": "fe9f64420957dc8fb6201d22b499572f021411e4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/9cebec37b96e9f38c1cc0a68a793304606423897", - "reference": "9cebec37b96e9f38c1cc0a68a793304606423897", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/fe9f64420957dc8fb6201d22b499572f021411e4", + "reference": "fe9f64420957dc8fb6201d22b499572f021411e4", "shasum": "" }, "require": { @@ -2229,7 +2223,7 @@ "ext-redis": "*", "php": ">=8.0", "utopia-php/cli": "0.14.*", - "utopia-php/framework": "0.22.*" + "utopia-php/framework": "0.25.*" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -2255,9 +2249,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/0.3.0" + "source": "https://github.com/utopia-php/platform/tree/0.3.1" }, - "time": "2022-10-28T07:26:09+00:00" + "time": "2022-11-10T07:04:24+00:00" }, { "name": "utopia-php/pools", From ab7eb01438ca7347c85b99f75ca003fa794cbe54 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 14 Nov 2022 15:35:49 +0545 Subject: [PATCH 41/50] Update src/Appwrite/CLI/Tasks/Usage.php Co-authored-by: Eldad A. Fux --- src/Appwrite/CLI/Tasks/Usage.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/CLI/Tasks/Usage.php b/src/Appwrite/CLI/Tasks/Usage.php index e3e9db865a..9231c90f66 100644 --- a/src/Appwrite/CLI/Tasks/Usage.php +++ b/src/Appwrite/CLI/Tasks/Usage.php @@ -32,7 +32,6 @@ class Usage extends Task ->callback(fn ($type, $dbForConsole, $influxDB, $logError) => $this->action($type, $dbForConsole, $influxDB, $logError)); } - protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void { $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) From ae0b9e78aa19424f76667101c4fec168da653d34 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 14 Nov 2022 10:01:41 +0000 Subject: [PATCH 42/50] refactor namespace and naming --- composer.lock | 2 +- src/Appwrite/CLI/Tasks.php | 13 ---------- src/Appwrite/Platform/Appwrite.php | 13 ++++++++++ .../TasksService.php => Platform/Tasks.php} | 26 +++++++++---------- .../{CLI => Platform}/Tasks/Doctor.php | 2 +- .../{CLI => Platform}/Tasks/Install.php | 2 +- .../{CLI => Platform}/Tasks/Maintenance.php | 2 +- .../{CLI => Platform}/Tasks/Migrate.php | 2 +- src/Appwrite/{CLI => Platform}/Tasks/SDKs.php | 2 +- src/Appwrite/{CLI => Platform}/Tasks/SSL.php | 2 +- .../{CLI => Platform}/Tasks/Specs.php | 2 +- .../{CLI => Platform}/Tasks/Usage.php | 2 +- src/Appwrite/{CLI => Platform}/Tasks/Vars.php | 2 +- .../{CLI => Platform}/Tasks/Version.php | 2 +- .../{CLI => Platform}/Tasks/VolumeSync.php | 2 +- 15 files changed, 38 insertions(+), 38 deletions(-) delete mode 100644 src/Appwrite/CLI/Tasks.php create mode 100644 src/Appwrite/Platform/Appwrite.php rename src/Appwrite/{CLI/TasksService.php => Platform/Tasks.php} (62%) rename src/Appwrite/{CLI => Platform}/Tasks/Doctor.php (99%) rename src/Appwrite/{CLI => Platform}/Tasks/Install.php (99%) rename src/Appwrite/{CLI => Platform}/Tasks/Maintenance.php (99%) rename src/Appwrite/{CLI => Platform}/Tasks/Migrate.php (98%) rename src/Appwrite/{CLI => Platform}/Tasks/SDKs.php (99%) rename src/Appwrite/{CLI => Platform}/Tasks/SSL.php (96%) rename src/Appwrite/{CLI => Platform}/Tasks/Specs.php (99%) rename src/Appwrite/{CLI => Platform}/Tasks/Usage.php (98%) rename src/Appwrite/{CLI => Platform}/Tasks/Vars.php (95%) rename src/Appwrite/{CLI => Platform}/Tasks/Version.php (92%) rename src/Appwrite/{CLI => Platform}/Tasks/VolumeSync.php (98%) diff --git a/composer.lock b/composer.lock index a66a959b7c..fbc8d019d7 100644 --- a/composer.lock +++ b/composer.lock @@ -5245,5 +5245,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/src/Appwrite/CLI/Tasks.php b/src/Appwrite/CLI/Tasks.php deleted file mode 100644 index 7f07b8137a..0000000000 --- a/src/Appwrite/CLI/Tasks.php +++ /dev/null @@ -1,13 +0,0 @@ -addService('cliTasks', new TasksService()); - } -} diff --git a/src/Appwrite/Platform/Appwrite.php b/src/Appwrite/Platform/Appwrite.php new file mode 100644 index 0000000000..187cabdeea --- /dev/null +++ b/src/Appwrite/Platform/Appwrite.php @@ -0,0 +1,13 @@ +addService('tasks', new Tasks()); + } +} diff --git a/src/Appwrite/CLI/TasksService.php b/src/Appwrite/Platform/Tasks.php similarity index 62% rename from src/Appwrite/CLI/TasksService.php rename to src/Appwrite/Platform/Tasks.php index 12617ec33f..2ca169a201 100644 --- a/src/Appwrite/CLI/TasksService.php +++ b/src/Appwrite/Platform/Tasks.php @@ -1,21 +1,21 @@ Date: Mon, 14 Nov 2022 10:09:39 +0000 Subject: [PATCH 43/50] remove base action --- src/Appwrite/Platform/Task.php | 15 --------------- src/Appwrite/Platform/Tasks/Usage.php | 5 ++--- 2 files changed, 2 insertions(+), 18 deletions(-) delete mode 100644 src/Appwrite/Platform/Task.php diff --git a/src/Appwrite/Platform/Task.php b/src/Appwrite/Platform/Task.php deleted file mode 100644 index d1eb7fa88b..0000000000 --- a/src/Appwrite/Platform/Task.php +++ /dev/null @@ -1,15 +0,0 @@ - Date: Mon, 14 Nov 2022 10:10:47 +0000 Subject: [PATCH 44/50] refactor service namespace --- src/Appwrite/Platform/Appwrite.php | 1 + src/Appwrite/Platform/{ => Services}/Tasks.php | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) rename src/Appwrite/Platform/{ => Services}/Tasks.php (96%) diff --git a/src/Appwrite/Platform/Appwrite.php b/src/Appwrite/Platform/Appwrite.php index 187cabdeea..9ef6f38c36 100644 --- a/src/Appwrite/Platform/Appwrite.php +++ b/src/Appwrite/Platform/Appwrite.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform; +use Appwrite\Platform\Services\Tasks; use Utopia\Platform\Platform; class Appwrite extends Platform diff --git a/src/Appwrite/Platform/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php similarity index 96% rename from src/Appwrite/Platform/Tasks.php rename to src/Appwrite/Platform/Services/Tasks.php index 2ca169a201..7f6a062ed4 100644 --- a/src/Appwrite/Platform/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -1,6 +1,6 @@ Date: Mon, 14 Nov 2022 10:24:16 +0000 Subject: [PATCH 45/50] update callback code --- src/Appwrite/Platform/Tasks/Doctor.php | 2 +- src/Appwrite/Platform/Tasks/Install.php | 2 +- src/Appwrite/Platform/Tasks/Maintenance.php | 2 +- src/Appwrite/Platform/Tasks/Migrate.php | 2 +- src/Appwrite/Platform/Tasks/SDKs.php | 2 +- src/Appwrite/Platform/Tasks/SSL.php | 2 +- src/Appwrite/Platform/Tasks/Specs.php | 2 +- src/Appwrite/Platform/Tasks/Usage.php | 2 +- src/Appwrite/Platform/Tasks/Vars.php | 2 +- src/Appwrite/Platform/Tasks/VolumeSync.php | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Doctor.php b/src/Appwrite/Platform/Tasks/Doctor.php index 38af7c7376..9f32aed746 100644 --- a/src/Appwrite/Platform/Tasks/Doctor.php +++ b/src/Appwrite/Platform/Tasks/Doctor.php @@ -25,7 +25,7 @@ class Doctor extends Action $this ->desc('Validate server health') ->inject('register') - ->callback(fn (Registry $register) => $this->action($register)); + ->callback($this->action); } public function action(Registry $register): void diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index 219a03129d..d7dcbcc1fb 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -28,7 +28,7 @@ class Install extends Action ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true) ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true) ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true) - ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive)); + ->callback($this->action); } public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index fe659c8746..b084ed0901 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -25,7 +25,7 @@ class Maintenance extends Action $this ->desc('Schedules maintenance tasks and publishes them to resque') ->inject('dbForConsole') - ->callback(fn (Database $dbForConsole) => $this->action($dbForConsole)); + ->callback($this->action); } public function action(Database $dbForConsole): void diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index d7f591ac1a..a27a27c256 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -25,7 +25,7 @@ class Migrate extends Action ->desc('Migrate Appwrite to new version') ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) ->inject('register') - ->callback(fn ($version, $register) => $this->action($version, $register)); + ->callback($this->action); } public function action(string $version, Registry $register) diff --git a/src/Appwrite/Platform/Tasks/SDKs.php b/src/Appwrite/Platform/Tasks/SDKs.php index fe312f8ef2..a7e6379d8e 100644 --- a/src/Appwrite/Platform/Tasks/SDKs.php +++ b/src/Appwrite/Platform/Tasks/SDKs.php @@ -36,7 +36,7 @@ class SDKs extends Action { $this ->desc('Generate Appwrite SDKs') - ->callback(fn () => $this->action()); + ->callback($this->action); } public function action(): void diff --git a/src/Appwrite/Platform/Tasks/SSL.php b/src/Appwrite/Platform/Tasks/SSL.php index 43026b0753..b98d9772f1 100644 --- a/src/Appwrite/Platform/Tasks/SSL.php +++ b/src/Appwrite/Platform/Tasks/SSL.php @@ -21,7 +21,7 @@ class SSL extends Action $this ->desc('Validate server certificates') ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) - ->callback(fn ($domain) => $this->action($domain)); + ->callback($this->action); } public function action(string $domain): void diff --git a/src/Appwrite/Platform/Tasks/Specs.php b/src/Appwrite/Platform/Tasks/Specs.php index 10f322f196..c948459f84 100644 --- a/src/Appwrite/Platform/Tasks/Specs.php +++ b/src/Appwrite/Platform/Tasks/Specs.php @@ -35,7 +35,7 @@ class Specs extends Action ->param('version', 'latest', new Text(16), 'Spec version', true) ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true) ->inject('register') - ->callback(fn (string $version, string $mode, Registry $register) => $this->action($version, $mode, $register)); + ->callback($this->action); } public function action(string $version, string $mode, Registry $register): void diff --git a/src/Appwrite/Platform/Tasks/Usage.php b/src/Appwrite/Platform/Tasks/Usage.php index 73686109d8..169e2ee536 100644 --- a/src/Appwrite/Platform/Tasks/Usage.php +++ b/src/Appwrite/Platform/Tasks/Usage.php @@ -28,7 +28,7 @@ class Usage extends Action ->inject('dbForConsole') ->inject('influxdb') ->inject('logError') - ->callback(fn ($type, $dbForConsole, $influxDB, $logError) => $this->action($type, $dbForConsole, $influxDB, $logError)); + ->callback($this->action); } protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void diff --git a/src/Appwrite/Platform/Tasks/Vars.php b/src/Appwrite/Platform/Tasks/Vars.php index c97f77a9da..3f4e71bc4b 100644 --- a/src/Appwrite/Platform/Tasks/Vars.php +++ b/src/Appwrite/Platform/Tasks/Vars.php @@ -18,7 +18,7 @@ class Vars extends Action { $this ->desc('List all the server environment variables') - ->callback(fn () => $this->action()); + ->callback($this->action); } public function action(): void diff --git a/src/Appwrite/Platform/Tasks/VolumeSync.php b/src/Appwrite/Platform/Tasks/VolumeSync.php index 6197b20fbd..56a61c52ef 100644 --- a/src/Appwrite/Platform/Tasks/VolumeSync.php +++ b/src/Appwrite/Platform/Tasks/VolumeSync.php @@ -22,7 +22,7 @@ class VolumeSync extends Action ->param('source', null, new Text(255), 'Source path to sync from.', false) ->param('destination', null, new Text(255), 'Destination path to sync to.', false) ->param('interval', null, new Integer(true), 'Interval to run rsync', false) - ->callback(fn ($source, $destination, $interval) => $this->action($source, $destination, $interval)); + ->callback($this->action); } public function action(string $source, string $destination, int $interval) From 5fbf961a9a6c4e80143e99e7045895609794be11 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 14 Nov 2022 10:29:10 +0000 Subject: [PATCH 46/50] fix cli --- app/cli.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/cli.php b/app/cli.php index a1a080fc31..b4be840172 100644 --- a/app/cli.php +++ b/app/cli.php @@ -3,7 +3,7 @@ require_once __DIR__ . '/init.php'; require_once __DIR__ . '/controllers/general.php'; -use Appwrite\CLI\Tasks; +use Appwrite\Platform\Appwrite; use Utopia\CLI\CLI; use Utopia\Database\Validator\Authorization; use Utopia\Platform\Service; @@ -113,8 +113,8 @@ CLI::setResource('logError', function (Registry $register) { }; }, ['register']); -$cliPlatform = new Tasks(); -$cliPlatform->init(Service::TYPE_CLI); +$platform = new Appwrite(); +$platform->init(Service::TYPE_CLI); -$cli = $cliPlatform->getCli(); +$cli = $platform->getCli(); $cli->run(); From 0570e26a48d2a999226d9c6d4ae1b67d7c4c0714 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 14 Nov 2022 10:36:21 +0000 Subject: [PATCH 47/50] revert back callback changes --- src/Appwrite/Platform/Tasks/Doctor.php | 2 +- src/Appwrite/Platform/Tasks/Install.php | 2 +- src/Appwrite/Platform/Tasks/Maintenance.php | 2 +- src/Appwrite/Platform/Tasks/Migrate.php | 2 +- src/Appwrite/Platform/Tasks/SDKs.php | 2 +- src/Appwrite/Platform/Tasks/SSL.php | 2 +- src/Appwrite/Platform/Tasks/Specs.php | 2 +- src/Appwrite/Platform/Tasks/Usage.php | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Doctor.php b/src/Appwrite/Platform/Tasks/Doctor.php index 9f32aed746..38af7c7376 100644 --- a/src/Appwrite/Platform/Tasks/Doctor.php +++ b/src/Appwrite/Platform/Tasks/Doctor.php @@ -25,7 +25,7 @@ class Doctor extends Action $this ->desc('Validate server health') ->inject('register') - ->callback($this->action); + ->callback(fn (Registry $register) => $this->action($register)); } public function action(Registry $register): void diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index d7dcbcc1fb..219a03129d 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -28,7 +28,7 @@ class Install extends Action ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true) ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true) ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true) - ->callback($this->action); + ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive)); } public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index b084ed0901..fe659c8746 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -25,7 +25,7 @@ class Maintenance extends Action $this ->desc('Schedules maintenance tasks and publishes them to resque') ->inject('dbForConsole') - ->callback($this->action); + ->callback(fn (Database $dbForConsole) => $this->action($dbForConsole)); } public function action(Database $dbForConsole): void diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index a27a27c256..d7f591ac1a 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -25,7 +25,7 @@ class Migrate extends Action ->desc('Migrate Appwrite to new version') ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) ->inject('register') - ->callback($this->action); + ->callback(fn ($version, $register) => $this->action($version, $register)); } public function action(string $version, Registry $register) diff --git a/src/Appwrite/Platform/Tasks/SDKs.php b/src/Appwrite/Platform/Tasks/SDKs.php index a7e6379d8e..fe312f8ef2 100644 --- a/src/Appwrite/Platform/Tasks/SDKs.php +++ b/src/Appwrite/Platform/Tasks/SDKs.php @@ -36,7 +36,7 @@ class SDKs extends Action { $this ->desc('Generate Appwrite SDKs') - ->callback($this->action); + ->callback(fn () => $this->action()); } public function action(): void diff --git a/src/Appwrite/Platform/Tasks/SSL.php b/src/Appwrite/Platform/Tasks/SSL.php index b98d9772f1..43026b0753 100644 --- a/src/Appwrite/Platform/Tasks/SSL.php +++ b/src/Appwrite/Platform/Tasks/SSL.php @@ -21,7 +21,7 @@ class SSL extends Action $this ->desc('Validate server certificates') ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) - ->callback($this->action); + ->callback(fn ($domain) => $this->action($domain)); } public function action(string $domain): void diff --git a/src/Appwrite/Platform/Tasks/Specs.php b/src/Appwrite/Platform/Tasks/Specs.php index c948459f84..10f322f196 100644 --- a/src/Appwrite/Platform/Tasks/Specs.php +++ b/src/Appwrite/Platform/Tasks/Specs.php @@ -35,7 +35,7 @@ class Specs extends Action ->param('version', 'latest', new Text(16), 'Spec version', true) ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true) ->inject('register') - ->callback($this->action); + ->callback(fn (string $version, string $mode, Registry $register) => $this->action($version, $mode, $register)); } public function action(string $version, string $mode, Registry $register): void diff --git a/src/Appwrite/Platform/Tasks/Usage.php b/src/Appwrite/Platform/Tasks/Usage.php index 169e2ee536..73686109d8 100644 --- a/src/Appwrite/Platform/Tasks/Usage.php +++ b/src/Appwrite/Platform/Tasks/Usage.php @@ -28,7 +28,7 @@ class Usage extends Action ->inject('dbForConsole') ->inject('influxdb') ->inject('logError') - ->callback($this->action); + ->callback(fn ($type, $dbForConsole, $influxDB, $logError) => $this->action($type, $dbForConsole, $influxDB, $logError)); } protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void From f469d1ba450c8011867fa0b38bf5e3cac06e1d74 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 14 Nov 2022 10:38:17 +0000 Subject: [PATCH 48/50] fix volume sync --- src/Appwrite/Platform/Tasks/VolumeSync.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/VolumeSync.php b/src/Appwrite/Platform/Tasks/VolumeSync.php index 56a61c52ef..6197b20fbd 100644 --- a/src/Appwrite/Platform/Tasks/VolumeSync.php +++ b/src/Appwrite/Platform/Tasks/VolumeSync.php @@ -22,7 +22,7 @@ class VolumeSync extends Action ->param('source', null, new Text(255), 'Source path to sync from.', false) ->param('destination', null, new Text(255), 'Destination path to sync to.', false) ->param('interval', null, new Integer(true), 'Interval to run rsync', false) - ->callback($this->action); + ->callback(fn ($source, $destination, $interval) => $this->action($source, $destination, $interval)); } public function action(string $source, string $destination, int $interval) From 920cff8ce73d1eb592c0f4b1ca569e4cd524a1b7 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 14 Nov 2022 10:39:37 +0000 Subject: [PATCH 49/50] fix vars --- src/Appwrite/Platform/Tasks/Vars.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Vars.php b/src/Appwrite/Platform/Tasks/Vars.php index 3f4e71bc4b..c97f77a9da 100644 --- a/src/Appwrite/Platform/Tasks/Vars.php +++ b/src/Appwrite/Platform/Tasks/Vars.php @@ -18,7 +18,7 @@ class Vars extends Action { $this ->desc('List all the server environment variables') - ->callback($this->action); + ->callback(fn () => $this->action()); } public function action(): void From 2c255a73942f420f1b3127e9d9485e8ccf32b204 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 14 Nov 2022 10:46:11 +0000 Subject: [PATCH 50/50] fix cli --- app/cli.php | 83 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/app/cli.php b/app/cli.php index b4be840172..b176326191 100644 --- a/app/cli.php +++ b/app/cli.php @@ -9,52 +9,75 @@ use Utopia\Database\Validator\Authorization; use Utopia\Platform\Service; use Utopia\App; use Utopia\CLI\Console; -use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; -use Utopia\Database\Adapter\MariaDB; +use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\Document; use Utopia\Logger\Log; +use Utopia\Pools\Group; use Utopia\Registry\Registry; Authorization::disable(); CLI::setResource('register', fn()=>$register); -CLI::setResource('db', function (Registry $register) { - $attempts = 0; - $max = 10; - $sleep = 1; - do { - try { - $attempts++; - $db = $register->get('db'); - break; // leave the do-while if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < $max); - return $db; +CLI::setResource('cache', function ($pools) { + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['pools']); + +CLI::setResource('pools', function (Registry $register) { + return $register->get('pools'); }, ['register']); -CLI::setResource('cache', fn ($register) => $register->get('cache'), ['register']); +CLI::setResource('dbForConsole', function ($pools, $cache) { + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; -CLI::setResource('dbForConsole', function ($db, $cache) { - $cache = new Cache(new RedisCache($cache)); + $database = new Database($dbAdapter, $cache); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); + $database->setNamespace('console'); return $database; -}, ['db', 'cache']); +}, ['pools', 'cache']); + +CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { + $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; + }; + + return $getProjectDB; +}, ['pools', 'dbForConsole', 'cache']); CLI::setResource('influxdb', function (Registry $register) { - /** @var InfluxDB\Client $client */ - $client = $register->get('influxdb'); + $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */ $attempts = 0; $max = 10; $sleep = 1; @@ -66,7 +89,7 @@ CLI::setResource('influxdb', function (Registry $register) { if (in_array('telegraf', $client->listDatabases())) { break; // leave the do-while if successful } - } catch (\Throwable$th) { + } catch (\Throwable $th) { Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); if ($attempts >= $max) { throw new \Exception('InfluxDB database not ready yet');