Merge branch '1.4.x' into chore-migration-enum-attribute-size

Dockerfile

@@ -105,7 +105,8 @@ RUN chmod +x /usr/local/bin/hamster && \
     chmod +x /usr/local/bin/delete-orphaned-projects && \
     chmod +x /usr/local/bin/clear-card-cache && \
     chmod +x /usr/local/bin/calc-users-stats && \
-    chmod +x /usr/local/bin/calc-tier-stats
+    chmod +x /usr/local/bin/calc-tier-stats && \
+    chmod +x /usr/local/bin/get-migration-stats
 
 # Letsencrypt Permissions
 RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

bin/get-migration-stats

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php get-migration-stats $@

src/Appwrite/Platform/Services/Tasks.php

@@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync;
 use Appwrite\Platform\Tasks\CalcTierStats;
 use Appwrite\Platform\Tasks\Upgrade;
 use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
+use Appwrite\Platform\Tasks\GetMigrationStats;
 use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
 
 class Tasks extends Service
@@ -44,6 +45,7 @@ class Tasks extends Service
             ->addAction(CalcTierStats::getName(), new CalcTierStats())
             ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
             ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
+            ->addAction(GetMigrationStats::getName(), new GetMigrationStats())
 
         ;
     }

src/Appwrite/Platform/Tasks/CalcTierStats.php

@@ -41,6 +41,7 @@ class CalcTierStats extends Action
         'Functions',
         'Deployments',
         'Executions',
+        'Migrations',
     ];
 
     protected string $directory = '/usr/local';
@@ -99,8 +100,8 @@ class CalcTierStats extends Action
 
         $projects = [$console];
         $count = 0;
-        $limit = 30;
-        $sum = 30;
+        $limit = 100;
+        $sum = 100;
         $offset = 0;
         while (!empty($projects)) {
             foreach ($projects as $project) {
@@ -200,7 +201,7 @@ class CalcTierStats extends Action
 
                     try {
                         /** Get Domains */
-                        $stats['Domains'] = $dbForConsole->count('domains', [
+                        $stats['Domains'] = $dbForConsole->count('rules', [
                             Query::equal('projectInternalId', [$project->getInternalId()]),
                         ]);
                     } catch (\Throwable) {
@@ -290,6 +291,13 @@ class CalcTierStats extends Action
                         $stats['Executions'] = 0;
                     }
 
+                    /** Get Total Migrations */
+                    try {
+                        $stats['Migrations'] = $dbForProject->count('migrations', []);
+                    } catch (\Throwable) {
+                        $stats['Migrations'] = 0;
+                    }
+
                     $csv->insertOne(array_values($stats));
                 } catch (\Throwable $th) {
                     Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . '  line no: ' . $th->getline() . ' with message: ' . $th->getMessage());

src/Appwrite/Platform/Tasks/GetMigrationStats.php

@@ -0,0 +1,187 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Exception;
+use League\Csv\CannotInsertRecord;
+use Utopia\App;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Query;
+use League\Csv\Writer;
+use PHPMailer\PHPMailer\PHPMailer;
+use Utopia\Pools\Group;
+use Utopia\Registry\Registry;
+
+class GetMigrationStats extends Action
+{
+    /*
+     * Csv cols headers
+     */
+    private array $columns = [
+        'Project ID',
+        '$id',
+        '$createdAt',
+        'status',
+        'stage',
+        'source'
+    ];
+
+    protected string $directory = '/usr/local';
+    protected string $path;
+    protected string $date;
+
+    public static function getName(): string
+    {
+        return 'get-migration-stats';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Get stats for projects')
+            ->inject('pools')
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->inject('register')
+            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
+                $this->action($pools, $cache, $dbForConsole, $register);
+            });
+    }
+
+    /**
+     * @throws \Utopia\Exception
+     * @throws CannotInsertRecord
+     */
+    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+    {
+        //docker compose exec -t appwrite get-migration-stats
+
+        Console::title('Migration stats calculation V1');
+        Console::success(APP_NAME . ' Migration stats calculation has started');
+
+        /* Initialise new Utopia app */
+        $app = new App('UTC');
+        $console = $app->getResource('console');
+
+        /** CSV stuff */
+        $this->date = date('Y-m-d');
+        $this->path = "{$this->directory}/migration_stats_{$this->date}.csv";
+        $csv = Writer::createFromPath($this->path, 'w');
+        $csv->insertOne($this->columns);
+
+        /** Database connections */
+        $totalProjects = $dbForConsole->count('projects');
+        Console::success("Found a total of: {$totalProjects} projects");
+
+        $projects = [$console];
+        $count = 0;
+        $limit = 100;
+        $sum = 100;
+        $offset = 0;
+        while (!empty($projects)) {
+            foreach ($projects as $project) {
+
+                /**
+                 * Skip user projects with id 'console'
+                 */
+                if ($project->getId() === 'console') {
+                    continue;
+                }
+
+                Console::info("Getting stats for {$project->getId()}");
+
+                try {
+                    $db = $project->getAttribute('database');
+                    $adapter = $pools
+                        ->get($db)
+                        ->pop()
+                        ->getResource();
+
+                    $dbForProject = new Database($adapter, $cache);
+                    $dbForProject->setDefaultDatabase('appwrite');
+                    $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                    /** Get Project ID */
+                    $stats['Project ID'] = $project->getId();
+
+                    /** Get Migration details */
+                    $migrations = $dbForProject->find('migrations', [
+                        Query::limit(500)
+                    ]);
+
+                    $migrations = array_map(function ($migration) use ($project) {
+                        return [
+                            $project->getId(),
+                            $migration->getAttribute('$id'),
+                            $migration->getAttribute('$createdAt'),
+                            $migration->getAttribute('status'),
+                            $migration->getAttribute('stage'),
+                            $migration->getAttribute('source'),
+                        ];
+                    }, $migrations);
+
+                    if (!empty($migrations)) {
+                        $csv->insertAll($migrations);
+                    }
+                } catch (\Throwable $th) {
+                    Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . '  line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
+                } finally {
+                    $pools
+                        ->get($db)
+                        ->reclaim();
+                }
+            }
+
+            $sum = \count($projects);
+
+            $projects = $dbForConsole->find('projects', [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+
+            $offset = $offset + $limit;
+            $count = $count + $sum;
+        }
+
+        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
+
+        $pools
+            ->get('console')
+            ->reclaim();
+
+        /** @var PHPMailer $mail */
+        $mail = $register->get('smtp');
+
+        $mail->clearAddresses();
+        $mail->clearAllRecipients();
+        $mail->clearReplyTos();
+        $mail->clearAttachments();
+        $mail->clearBCCs();
+        $mail->clearCCs();
+
+        try {
+            /** Addresses */
+            $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
+            $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
+
+            foreach ($recipients as $recipient) {
+                $mail->addAddress($recipient);
+            }
+
+            /** Attachments */
+            $mail->addAttachment($this->path);
+
+            /** Content */
+            $mail->Subject = "Migration Report for {$this->date}";
+            $mail->Body = "Please find the migration report atttached";
+            $mail->send();
+            Console::success('Email has been sent!');
+        } catch (Exception $e) {
+            Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
+        }
+    }
+}

src/Appwrite/Platform/Tasks/Hamster.php

@@ -149,6 +149,9 @@ class Hamster extends Action
                 /** Get Total Teams */
                 $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
 
+                /** Get Total Migrations */
+                $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT);
+
                 /** Get Total Members */
                 $teamInternalId = $project->getAttribute('teamInternalId', null);
                 if ($teamInternalId) {