From 0f3c43897621e892e1b1dceed6378cf385112803 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 8 Nov 2023 12:05:51 +0200 Subject: [PATCH 1/5] added inf metric --- .../Platform/Tasks/DeleteOrphanedProjects.php | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2824f4e286..aeaab4b248 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -13,6 +13,8 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Registry\Registry; +use Utopia\Validator\Boolean; +use Utopia\Validator\Hostname; class DeleteOrphanedProjects extends Action { @@ -26,25 +28,23 @@ class DeleteOrphanedProjects extends Action $this ->desc('Get stats for projects') + ->param('commit', false, new boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') ->inject('dbForConsole') ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); + ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($commit, $pools, $cache, $dbForConsole, $register); }); } - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void { Console::title('Delete orphaned projects V1'); Console::success(APP_NAME . ' Delete orphaned projects started'); - /** @var array $collections */ - $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; - /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -54,7 +54,7 @@ class DeleteOrphanedProjects extends Action $totalProjects = $dbForConsole->count('projects'); Console::success("Found a total of: {$totalProjects} projects"); - $orphans = 0; + $orphans = 1; $count = 0; $limit = 30; $sum = 30; @@ -80,17 +80,22 @@ class DeleteOrphanedProjects extends Action $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); $collectionsCreated = $dbForProject->count(Database::METADATA); - $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))'; - if ($collectionsCreated < (count($collectionsConfig) + 2)) { - Console::error($message); + if ($collectionsCreated === 0) { + if ($commit === true) { + Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); + $this->deleteProject($dbForConsole, $project->getId()); + } else { + Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + } $orphans++; - } else { - Console::log($message); } } catch (\Throwable $th) { - //$dbForConsole->deleteDocument('projects', $project->getId()); - //Console::success('Deleting project (' . $project->getId() . ')'); - Console::error(' (0) collections where found for project (' . $project->getId() . ')'); + if ($commit === true) { + Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); + $this->deleteProject($dbForConsole, $project->getId()); + } else { + Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + } $orphans++; } finally { $pools @@ -110,6 +115,15 @@ class DeleteOrphanedProjects extends Action $count = $count + $sum; } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); + } + + private function deleteProject(Database $dbForConsole, $projectId): void + { + try { + $dbForConsole->deleteDocument('projects', $projectId); + } catch (\Throwable $th) { + Console::error('Error when trying to delete project (' . $projectId . ') ' . $th->getMessage()); + } } } From dd0bf66212450041ab7518a0f958c8fdbb9c8166 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:24:55 +0200 Subject: [PATCH 2/5] delete orphaned projects task --- .../Platform/Tasks/DeleteOrphanedProjects.php | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index aeaab4b248..2af9c54a6e 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -2,11 +2,9 @@ namespace Appwrite\Platform\Tasks; -use PHPMailer\PHPMailer\PHPMailer; use Utopia\App; use Utopia\Config\Config; use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -14,7 +12,6 @@ use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Validator\Boolean; -use Utopia\Validator\Hostname; class DeleteOrphanedProjects extends Action { @@ -27,7 +24,7 @@ class DeleteOrphanedProjects extends Action { $this - ->desc('Get stats for projects') + ->desc('Delete orphaned projects') ->param('commit', false, new boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') @@ -45,6 +42,9 @@ class DeleteOrphanedProjects extends Action Console::title('Delete orphaned projects V1'); Console::success(APP_NAME . ' Delete orphaned projects started'); + /** @var array $collections */ + $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; + /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -55,6 +55,7 @@ class DeleteOrphanedProjects extends Action Console::success("Found a total of: {$totalProjects} projects"); $orphans = 1; + $cnt = 0; $count = 0; $limit = 30; $sum = 30; @@ -79,24 +80,43 @@ class DeleteOrphanedProjects extends Action $dbForProject = new Database($adapter, $cache); $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); - $collectionsCreated = $dbForProject->count(Database::METADATA); - if ($collectionsCreated === 0) { - if ($commit === true) { - Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); - $this->deleteProject($dbForConsole, $project->getId()); - } else { - Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + $collectionsCreated = 0; + $cnt++; + if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { + $collectionsCreated = $dbForProject->count(Database::METADATA); + } + + $msg = '(' . $cnt . ') ignoring found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; + /** + * +2 == audit+abuse + */ + if ($collectionsCreated === (count($collectionsConfig) + 2)) { + Console::log($msg . ' ignoring....'); + continue; + } + + Console::log($msg); + + if ($collectionsCreated > 0) { + $collections = $dbForProject->find(Database::METADATA, []); + foreach ($collections as $collection) { + if ($commit) { + $dbForProject->deleteCollection($collection->getId()); + $dbForConsole->deleteCachedCollection($collection->getId()); + } + Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); } + } + if ($commit) { + $dbForConsole->deleteDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); + } + + Console::info('--Deleting project no (' . $project->getInternalId() . ')'); + $orphans++; - } } catch (\Throwable $th) { - if ($commit === true) { - Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); - $this->deleteProject($dbForConsole, $project->getId()); - } else { - Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); - } - $orphans++; + Console::error('Error: ' . $th->getMessage()); } finally { $pools ->get($db) @@ -117,13 +137,4 @@ class DeleteOrphanedProjects extends Action Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); } - - private function deleteProject(Database $dbForConsole, $projectId): void - { - try { - $dbForConsole->deleteDocument('projects', $projectId); - } catch (\Throwable $th) { - Console::error('Error when trying to delete project (' . $projectId . ') ' . $th->getMessage()); - } - } } From 60c0f4c97393d8d665da5ab7d7b697fafa69abbc Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:27:20 +0200 Subject: [PATCH 3/5] delete orphaned projects task --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2af9c54a6e..2da3c91384 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -86,9 +86,9 @@ class DeleteOrphanedProjects extends Action $collectionsCreated = $dbForProject->count(Database::METADATA); } - $msg = '(' . $cnt . ') ignoring found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; + $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; /** - * +2 == audit+abuse + * +2 = audit+abuse */ if ($collectionsCreated === (count($collectionsConfig) + 2)) { Console::log($msg . ' ignoring....'); From 3d9ee8bc521367d8242de2e4035970d44a9050da Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:28:48 +0200 Subject: [PATCH 4/5] delete orphaned projects task --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2da3c91384..ee9931bde4 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -25,7 +25,7 @@ class DeleteOrphanedProjects extends Action $this ->desc('Delete orphaned projects') - ->param('commit', false, new boolean(true), 'Commit project deletion', true) + ->param('commit', false, new Boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') ->inject('dbForConsole') From 6b5e734dfaedfb6fb291d7dab6a7a883dcc5a555 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 15 Nov 2023 20:09:50 +0200 Subject: [PATCH 5/5] indentation small fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index ee9931bde4..5f0ecbe1db 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -112,9 +112,9 @@ class DeleteOrphanedProjects extends Action $dbForConsole->deleteCachedDocument('projects', $project->getId()); } - Console::info('--Deleting project no (' . $project->getInternalId() . ')'); + Console::info('--Deleting project no (' . $project->getInternalId() . ')'); - $orphans++; + $orphans++; } catch (\Throwable $th) { Console::error('Error: ' . $th->getMessage()); } finally {