From e79cb913ed81c936fb92cfa1dbd6b734c53c0dea Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 23 May 2023 18:58:49 +0300 Subject: [PATCH] calculating users per project task --- Dockerfile | 1 + bin/users-calc | 3 + composer.json | 4 +- composer.lock | 215 +++++++++++++--------- src/Appwrite/Platform/Services/Tasks.php | 4 +- src/Appwrite/Platform/Tasks/UsersCalc.php | 163 ++++++++++++++++ 6 files changed, 295 insertions(+), 95 deletions(-) create mode 100644 bin/users-calc create mode 100644 src/Appwrite/Platform/Tasks/UsersCalc.php diff --git a/Dockerfile b/Dockerfile index bf3f4f1634..9056e2a733 100755 --- a/Dockerfile +++ b/Dockerfile @@ -119,6 +119,7 @@ RUN mkdir -p /storage/uploads && \ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ chmod +x /usr/local/bin/clear-card-cache && \ + chmod +x /usr/local/bin/users-calc && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ diff --git a/bin/users-calc b/bin/users-calc new file mode 100644 index 0000000000..3466d19e41 --- /dev/null +++ b/bin/users-calc @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php users-calc $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 1eec45940f..6fba2c92ab 100644 --- a/composer.json +++ b/composer.json @@ -72,8 +72,8 @@ "influxdb/influxdb-php": "1.15.2", "phpmailer/phpmailer": "6.6.0", "chillerlan/php-qrcode": "4.3.3", - "adhocore/jwt": "1.1.2", - "slickdeals/statsd": "3.1.0" + "slickdeals/statsd": "3.1.0", + "league/csv": "^9.0.0" }, "repositories": [ { diff --git a/composer.lock b/composer.lock index 5349ca36b1..8f236ac2aa 100644 --- a/composer.lock +++ b/composer.lock @@ -4,65 +4,8 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ca2a083ff1c0d0c77942674400137793", + "content-hash": "b7aa119fed3ade9c7dd254a199da3855", "packages": [ - { - "name": "adhocore/jwt", - "version": "1.1.2", - "source": { - "type": "git", - "url": "https://github.com/adhocore/php-jwt.git", - "reference": "6c434af7170090bb7a8880d2bc220a2254ba7899" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/adhocore/php-jwt/zipball/6c434af7170090bb7a8880d2bc220a2254ba7899", - "reference": "6c434af7170090bb7a8880d2bc220a2254ba7899", - "shasum": "" - }, - "require": { - "php": "^7.0 || ^8.0" - }, - "require-dev": { - "phpunit/phpunit": "^6.5 || ^7.5" - }, - "type": "library", - "autoload": { - "psr-4": { - "Ahc\\Jwt\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Jitendra Adhikari", - "email": "jiten.adhikary@gmail.com" - } - ], - "description": "Ultra lightweight JSON web token (JWT) library for PHP5.5+.", - "keywords": [ - "auth", - "json-web-token", - "jwt", - "jwt-auth", - "jwt-php", - "token" - ], - "support": { - "issues": "https://github.com/adhocore/php-jwt/issues", - "source": "https://github.com/adhocore/php-jwt/tree/1.1.2" - }, - "funding": [ - { - "url": "https://paypal.me/ji10", - "type": "custom" - } - ], - "time": "2021-02-20T09:56:44+00:00" - }, { "name": "appwrite/php-clamav", "version": "1.1.0", @@ -408,21 +351,21 @@ }, { "name": "guzzlehttp/guzzle", - "version": "7.6.1", + "version": "7.7.0", "source": { "type": "git", "url": "https://github.com/guzzle/guzzle.git", - "reference": "8444a2bacf1960bc6a2b62ed86b8e72e11eebe51" + "reference": "fb7566caccf22d74d1ab270de3551f72a58399f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle/zipball/8444a2bacf1960bc6a2b62ed86b8e72e11eebe51", - "reference": "8444a2bacf1960bc6a2b62ed86b8e72e11eebe51", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/fb7566caccf22d74d1ab270de3551f72a58399f5", + "reference": "fb7566caccf22d74d1ab270de3551f72a58399f5", "shasum": "" }, "require": { "ext-json": "*", - "guzzlehttp/promises": "^1.5", + "guzzlehttp/promises": "^1.5.3 || ^2.0", "guzzlehttp/psr7": "^1.9.1 || ^2.4.5", "php": "^7.2.5 || ^8.0", "psr/http-client": "^1.0", @@ -434,7 +377,8 @@ "require-dev": { "bamarni/composer-bin-plugin": "^1.8.1", "ext-curl": "*", - "php-http/client-integration-tests": "^3.0", + "php-http/client-integration-tests": "dev-master#2c025848417c1135031fdf9c728ee53d0a7ceaee as 3.0.999", + "php-http/message-factory": "^1.1", "phpunit/phpunit": "^8.5.29 || ^9.5.23", "psr/log": "^1.1 || ^2.0 || ^3.0" }, @@ -513,7 +457,7 @@ ], "support": { "issues": "https://github.com/guzzle/guzzle/issues", - "source": "https://github.com/guzzle/guzzle/tree/7.6.1" + "source": "https://github.com/guzzle/guzzle/tree/7.7.0" }, "funding": [ { @@ -529,38 +473,37 @@ "type": "tidelift" } ], - "time": "2023-05-15T20:43:01+00:00" + "time": "2023-05-21T14:04:53+00:00" }, { "name": "guzzlehttp/promises", - "version": "1.5.2", + "version": "2.0.0", "source": { "type": "git", "url": "https://github.com/guzzle/promises.git", - "reference": "b94b2807d85443f9719887892882d0329d1e2598" + "reference": "3a494dc7dc1d7d12e511890177ae2d0e6c107da6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/promises/zipball/b94b2807d85443f9719887892882d0329d1e2598", - "reference": "b94b2807d85443f9719887892882d0329d1e2598", + "url": "https://api.github.com/repos/guzzle/promises/zipball/3a494dc7dc1d7d12e511890177ae2d0e6c107da6", + "reference": "3a494dc7dc1d7d12e511890177ae2d0e6c107da6", "shasum": "" }, "require": { - "php": ">=5.5" + "php": "^7.2.5 || ^8.0" }, "require-dev": { - "symfony/phpunit-bridge": "^4.4 || ^5.1" + "bamarni/composer-bin-plugin": "^1.8.1", + "phpunit/phpunit": "^8.5.29 || ^9.5.23" }, "type": "library", "extra": { - "branch-alias": { - "dev-master": "1.5-dev" + "bamarni-bin": { + "bin-links": true, + "forward-command": false } }, "autoload": { - "files": [ - "src/functions_include.php" - ], "psr-4": { "GuzzleHttp\\Promise\\": "src/" } @@ -597,7 +540,7 @@ ], "support": { "issues": "https://github.com/guzzle/promises/issues", - "source": "https://github.com/guzzle/promises/tree/1.5.2" + "source": "https://github.com/guzzle/promises/tree/2.0.0" }, "funding": [ { @@ -613,7 +556,7 @@ "type": "tidelift" } ], - "time": "2022-08-28T14:55:35+00:00" + "time": "2023-05-21T13:50:22+00:00" }, { "name": "guzzlehttp/psr7", @@ -864,6 +807,93 @@ }, "time": "2022-11-29T16:25:20+00:00" }, + { + "name": "league/csv", + "version": "9.9.0", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/csv.git", + "reference": "b4418ede47fbd88facc34e40a16c8ce9153b961b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/b4418ede47fbd88facc34e40a16c8ce9153b961b", + "reference": "b4418ede47fbd88facc34e40a16c8ce9153b961b", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "php": "^8.1.2" + }, + "require-dev": { + "doctrine/collections": "^2.1.2", + "ext-dom": "*", + "ext-xdebug": "*", + "friendsofphp/php-cs-fixer": "^v3.14.3", + "phpbench/phpbench": "^1.2.8", + "phpstan/phpstan": "^1.10.4", + "phpstan/phpstan-deprecation-rules": "^1.1.2", + "phpstan/phpstan-phpunit": "^1.3.10", + "phpstan/phpstan-strict-rules": "^1.5.0", + "phpunit/phpunit": "^10.0.14" + }, + "suggest": { + "ext-dom": "Required to use the XMLConverter and the HTMLConverter classes", + "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "9.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "League\\Csv\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ignace Nyamagana Butera", + "email": "nyamsprod@gmail.com", + "homepage": "https://github.com/nyamsprod/", + "role": "Developer" + } + ], + "description": "CSV data manipulation made easy in PHP", + "homepage": "https://csv.thephpleague.com", + "keywords": [ + "convert", + "csv", + "export", + "filter", + "import", + "read", + "transform", + "write" + ], + "support": { + "docs": "https://csv.thephpleague.com", + "issues": "https://github.com/thephpleague/csv/issues", + "rss": "https://github.com/thephpleague/csv/releases.atom", + "source": "https://github.com/thephpleague/csv" + }, + "funding": [ + { + "url": "https://github.com/sponsors/nyamsprod", + "type": "github" + } + ], + "time": "2023-03-11T15:57:12+00:00" + }, { "name": "matomo/device-detector", "version": "6.0.0", @@ -3165,16 +3195,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.4", + "version": "v4.15.5", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "6bb5176bc4af8bcb7d926f88718db9b96a2d4290" + "reference": "11e2663a5bc9db5d714eedb4277ee300403b4a9e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/6bb5176bc4af8bcb7d926f88718db9b96a2d4290", - "reference": "6bb5176bc4af8bcb7d926f88718db9b96a2d4290", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/11e2663a5bc9db5d714eedb4277ee300403b4a9e", + "reference": "11e2663a5bc9db5d714eedb4277ee300403b4a9e", "shasum": "" }, "require": { @@ -3215,9 +3245,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.4" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.5" }, - "time": "2023-03-05T19:49:14+00:00" + "time": "2023-05-19T20:20:00+00:00" }, { "name": "phar-io/manifest", @@ -3568,22 +3598,23 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.20.4", + "version": "1.21.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "7d568c87a9df9c5f7e8b5f075fc469aa8cb0a4cd" + "reference": "6df62b08faef4f899772bc7c3bbabb93d2b7a21c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/7d568c87a9df9c5f7e8b5f075fc469aa8cb0a4cd", - "reference": "7d568c87a9df9c5f7e8b5f075fc469aa8cb0a4cd", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/6df62b08faef4f899772bc7c3bbabb93d2b7a21c", + "reference": "6df62b08faef4f899772bc7c3bbabb93d2b7a21c", "shasum": "" }, "require": { "php": "^7.2 || ^8.0" }, "require-dev": { + "nikic/php-parser": "^4.15", "php-parallel-lint/php-parallel-lint": "^1.2", "phpstan/extension-installer": "^1.0", "phpstan/phpstan": "^1.5", @@ -3607,9 +3638,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.20.4" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.21.0" }, - "time": "2023-05-02T09:19:37+00:00" + "time": "2023-05-17T13:13:44+00:00" }, { "name": "phpunit/php-code-coverage", @@ -5457,5 +5488,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index b82dca422f..e60f06044e 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; +use Appwrite\Platform\Tasks\UsersCalc; class Tasks extends Service { @@ -41,6 +42,7 @@ class Tasks extends Service ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) ->addAction(VolumeSync::getName(), new VolumeSync()) - ->addAction(Specs::getName(), new Specs()); + ->addAction(Specs::getName(), new Specs()) + ->addAction(UsersCalc::getName(), new UsersCalc()); } } diff --git a/src/Appwrite/Platform/Tasks/UsersCalc.php b/src/Appwrite/Platform/Tasks/UsersCalc.php new file mode 100644 index 0000000000..f59c158338 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/UsersCalc.php @@ -0,0 +1,163 @@ +desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('register') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($pools, $cache, $dbForConsole, $register); + }); + } + + public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + { + + Console::title('Cloud Users calculation V1'); + Console::success(APP_NAME . ' cloud Users calculation has started'); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/users_stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects'); + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + /** Get Project ID */ + $stats['Project ID'] = $project->getId(); + + /** Get Project Name */ + $stats['Project Name'] = $project->getAttribute('name'); + + /** Get Total Users */ + $stats['users'] = $dbForProject->count('users', [], APP_LIMIT_COUNT); + + $csv->insertOne(array_values($stats)); + } catch (\Throwable $th) { + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + + Console::log('Iterated through ' . ($count - 1) . '/' . $totalProjects . ' projects...'); + + /** @var PHPMailer $mail */ + $mail = $register->get('smtp'); + + $mail->clearAddresses(); + $mail->clearAllRecipients(); + $mail->clearReplyTos(); + $mail->clearAttachments(); + $mail->clearBCCs(); + $mail->clearCCs(); + + try { + /** Addresses */ + $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); + $recipients = explode(',', App::getEnv('_APP_HAMSTER_RECIPIENTS', 'shimon@appwrite.io')); + + foreach ($recipients as $recipient) { + $mail->addAddress($recipient); + } + + /** Attachments */ + $mail->addAttachment($this->path); + + /** Content */ + $mail->Subject = "Cloud Report for {$this->date}"; + $mail->Body = "Please find the daily cloud report atttached"; + + $mail->send(); + Console::success('Email has been sent!'); + } catch (Exception $e) { + Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + } + } + $pools + ->get('console') + ->reclaim(); + } +}