Merge pull request #5705 from appwrite/delete-tables-dpm

delete unnecessary project collections task

Dockerfile

@@ -121,6 +121,7 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/clear-card-cache && \
     chmod +x /usr/local/bin/calc-users-stats && \
     chmod +x /usr/local/bin/calc-tier-stats && \
+    chmod +x /usr/local/bin/patch-delete-project-collections && \
     chmod +x /usr/local/bin/maintenance &&  \
     chmod +x /usr/local/bin/volume-sync && \
     chmod +x /usr/local/bin/usage && \

bin/patch-delete-project-collections

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php patch-delete-project-collections $@

composer.lock

@@ -3070,16 +3070,16 @@
         },
         {
             "name": "matthiasmullie/minify",
-            "version": "1.3.70",
+            "version": "1.3.71",
             "source": {
                 "type": "git",
                 "url": "https://github.com/matthiasmullie/minify.git",
-                "reference": "2807d9f9bece6877577ad44acb5c801bb3ae536b"
+                "reference": "ae42a47d7fecc1fbb7277b2f2d84c37a33edc3b1"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/2807d9f9bece6877577ad44acb5c801bb3ae536b",
-                "reference": "2807d9f9bece6877577ad44acb5c801bb3ae536b",
+                "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/ae42a47d7fecc1fbb7277b2f2d84c37a33edc3b1",
+                "reference": "ae42a47d7fecc1fbb7277b2f2d84c37a33edc3b1",
                 "shasum": ""
             },
             "require": {
@@ -3129,7 +3129,7 @@
             ],
             "support": {
                 "issues": "https://github.com/matthiasmullie/minify/issues",
-                "source": "https://github.com/matthiasmullie/minify/tree/1.3.70"
+                "source": "https://github.com/matthiasmullie/minify/tree/1.3.71"
             },
             "funding": [
                 {
@@ -3137,7 +3137,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2022-12-09T12:56:44+00:00"
+            "time": "2023-04-25T20:33:03+00:00"
         },
         {
             "name": "matthiasmullie/path-converter",
@@ -5451,16 +5451,16 @@
         },
         {
             "name": "twig/twig",
-            "version": "v3.6.0",
+            "version": "v3.6.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/twigphp/Twig.git",
-                "reference": "106c170d08e8415d78be2d16c3d057d0d108262b"
+                "reference": "7e7d5839d4bec168dfeef0ac66d5c5a2edbabffd"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/twigphp/Twig/zipball/106c170d08e8415d78be2d16c3d057d0d108262b",
-                "reference": "106c170d08e8415d78be2d16c3d057d0d108262b",
+                "url": "https://api.github.com/repos/twigphp/Twig/zipball/7e7d5839d4bec168dfeef0ac66d5c5a2edbabffd",
+                "reference": "7e7d5839d4bec168dfeef0ac66d5c5a2edbabffd",
                 "shasum": ""
             },
             "require": {
@@ -5506,7 +5506,7 @@
             ],
             "support": {
                 "issues": "https://github.com/twigphp/Twig/issues",
-                "source": "https://github.com/twigphp/Twig/tree/v3.6.0"
+                "source": "https://github.com/twigphp/Twig/tree/v3.6.1"
             },
             "funding": [
                 {
@@ -5518,7 +5518,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-05-03T19:06:57+00:00"
+            "time": "2023-06-08T12:52:13+00:00"
         }
     ],
     "aliases": [],

src/Appwrite/Platform/Services/Tasks.php

@@ -21,6 +21,7 @@ use Appwrite\Platform\Tasks\Version;
 use Appwrite\Platform\Tasks\VolumeSync;
 use Appwrite\Platform\Tasks\CalcUsersStats;
 use Appwrite\Platform\Tasks\CalcTierStats;
+use Appwrite\Platform\Tasks\PatchDeleteProjectCollections;
 
 class Tasks extends Service
 {
@@ -46,6 +47,7 @@ class Tasks extends Service
             ->addAction(Specs::getName(), new Specs())
             ->addAction(CalcUsersStats::getName(), new CalcUsersStats())
             ->addAction(CalcTierStats::getName(), new CalcTierStats())
+            ->addAction(PatchDeleteProjectCollections::getName(), new PatchDeleteProjectCollections())
         ;
     }
 }

src/Appwrite/Platform/Tasks/PatchDeleteProjectCollections.php

@@ -0,0 +1,129 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\App;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Query;
+use Utopia\Pools\Group;
+use Utopia\Validator\Numeric;
+
+class PatchDeleteProjectCollections extends Action
+{
+    private array $names = [
+        'webhooks',
+        'platforms',
+        'schedules',
+        'projects',
+        'domains',
+        'certificates',
+        'keys',
+        'realtime',
+    ];
+
+    public static function getName(): string
+    {
+        return 'patch-delete-project-collections';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Delete unnecessary project collections')
+            ->param('offset', 0, new Numeric(), 'Resume deletion from param pos', true)
+            ->inject('pools')
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->callback(function (int $offset, Group $pools, Cache $cache, Database $dbForConsole) {
+                $this->action($offset, $pools, $cache, $dbForConsole);
+            });
+    }
+
+    public function action(int $offset, Group $pools, Cache $cache, Database $dbForConsole): void
+    {
+        //docker compose exec -t appwrite patch-delete-project-collections
+
+        Console::title('Delete project collections V1');
+        Console::success(APP_NAME . ' delete project collections has started');
+
+        /* Initialise new Utopia app */
+        $app = new App('UTC');
+        $console = $app->getResource('console');
+
+        /** Database connections */
+        $totalProjects = $dbForConsole->count('projects');
+        Console::success("Found a total of: {$totalProjects} projects");
+
+        $projects = [$console];
+        $count = 0;
+        $limit = 50;
+        $sum = 50;
+        $offset = $offset;
+        while (!empty($projects)) {
+            foreach ($projects as $project) {
+
+                /**
+                 * Skip user projects with id 'console'
+                 */
+                if ($project->getId() === 'console') {
+                    continue;
+                }
+
+                Console::info("Deleting collections for {$project->getId()}");
+
+                try {
+                    $db = $project->getAttribute('database');
+                    $adapter = $pools
+                        ->get($db)
+                        ->pop()
+                        ->getResource();
+
+                    $dbForProject = new Database($adapter, $cache);
+                    $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
+                    $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                    foreach ($this->names as $name) {
+                        if (empty($name)) {
+                            continue;
+                        }
+                        if ($dbForProject->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
+                            if ($dbForProject->deleteCollection($name)) {
+                                Console::log('Deleted ' . $name);
+                            } else {
+                                Console::error('Failed to delete ' . $name);
+                            }
+                        }
+                    }
+                } catch (\Throwable $th) {
+                    Console::error('Failed  on project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
+                } finally {
+                    $pools
+                        ->get($db)
+                        ->reclaim();
+                }
+            }
+
+            $sum = \count($projects);
+
+            $projects = $dbForConsole->find('projects', [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+
+            if (!empty($projects)) {
+                Console::log('Querying..... offset=' . $offset . ' , limit=' . $limit  . ', count=' . $count);
+            }
+
+            $offset = $offset + $limit;
+            $count = $count + $sum;
+        }
+        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
+        $pools
+            ->get('console')
+            ->reclaim();
+    }
+}