From 9e9fb6414db6ecbc4395f2fd9df07b93337e6733 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 5 Feb 2021 10:05:26 +0100 Subject: [PATCH 1/7] add certificates to deletion worker --- app/workers/deletes.php | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 2aa1fddc9..92ee86d48 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -52,6 +52,9 @@ class DeletesV1 case Database::SYSTEM_COLLECTION_COLLECTIONS: $this->deleteDocuments($document, $projectId); break; + case Database::SYSTEM_COLLECTION_DOMAINS: + $this->deleteCertificates($document); + break; default: Console::error('No lazy delete operation available for document of type: '.$document->getCollection()); break; @@ -305,6 +308,18 @@ class DeletesV1 Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } + protected function deleteCertificates(Document $document) + { + $domain = $document->getAttribute('domain', null); + $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; + + if($domain && is_dir($directory)) { + array_map('unlink', glob("$directory/*.*")); + rmdir($directory); + Console::info("Deleted certificate files for domain {$domain}"); + } + } + /** * @return Database; */ From ef29a87f615c3fbb14caf7f6a6e17aa513b0b22a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 5 Feb 2021 10:05:46 +0100 Subject: [PATCH 2/7] queue deletion worker for certificates --- app/controllers/api/projects.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index ef8694625..bee94871a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -1566,7 +1566,8 @@ App::delete('/v1/projects/:projectId/domains/:domainId') ->param('domainId', null, new UID(), 'Domain unique ID.') ->inject('response') ->inject('consoleDB') - ->action(function ($projectId, $domainId, $response, $consoleDB) { + ->inject('deletes') + ->action(function ($projectId, $domainId, $response, $consoleDB, $deletes) { /** @var Appwrite\Utopia\Response $response */ /** @var Appwrite\Database\Database $consoleDB */ @@ -1582,7 +1583,12 @@ App::delete('/v1/projects/:projectId/domains/:domainId') throw new Exception('Domain not found', 404); } - if (!$consoleDB->deleteDocument($domain->getId())) { + if ($consoleDB->deleteDocument($domain->getId())) { + $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $domain) + ; + } else { throw new Exception('Failed to remove domains from DB', 500); } From f97c87a3e9c19c7f78b0ddc991029cd6944f0e9f Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 5 Feb 2021 11:57:43 +0100 Subject: [PATCH 3/7] adds certificates to deletes worker --- app/controllers/api/projects.php | 2 +- app/init.php | 1 + app/workers/deletes.php | 16 ++++++++++------ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index bee94871a..8781e6edb 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -1585,7 +1585,7 @@ App::delete('/v1/projects/:projectId/domains/:domainId') if ($consoleDB->deleteDocument($domain->getId())) { $deletes - ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('type', DELETE_TYPE_CERTIFICATES) ->setParam('document', $domain) ; } else { diff --git a/app/init.php b/app/init.php index 977efd94d..085b5347c 100644 --- a/app/init.php +++ b/app/init.php @@ -60,6 +60,7 @@ const DELETE_TYPE_DOCUMENT = 'document'; const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; +const DELETE_TYPE_CERTIFICATES = 'certificates'; $register = new Registry(); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 92ee86d48..a329afe24 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -38,7 +38,7 @@ class DeletesV1 switch (strval($type)) { case DELETE_TYPE_DOCUMENT: $document = $this->args['document']; - $document = new Document($document); + $document = new Document($document); switch (strval($document->getCollection())) { case Database::SYSTEM_COLLECTION_PROJECTS: $this->deleteProject($document); @@ -52,9 +52,6 @@ class DeletesV1 case Database::SYSTEM_COLLECTION_COLLECTIONS: $this->deleteDocuments($document, $projectId); break; - case Database::SYSTEM_COLLECTION_DOMAINS: - $this->deleteCertificates($document); - break; default: Console::error('No lazy delete operation available for document of type: '.$document->getCollection()); break; @@ -72,6 +69,11 @@ class DeletesV1 case DELETE_TYPE_ABUSE: $this->deleteAbuseLogs($this->args['timestamp']); break; + + case DELETE_TYPE_CERTIFICATES: + $document = new Document($this->args['document']); + $this->deleteCertificates($document); + break; default: Console::error('No delete operation for type: '.$type); @@ -310,13 +312,15 @@ class DeletesV1 protected function deleteCertificates(Document $document) { - $domain = $document->getAttribute('domain', null); + $domain = $document->getAttribute('domain'); $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; if($domain && is_dir($directory)) { array_map('unlink', glob("$directory/*.*")); rmdir($directory); - Console::info("Deleted certificate files for domain {$domain}"); + Console::info("Deleted certificate files for {$domain}"); + } else { + Console::info("No certificate files found for {$domain}"); } } From 078b5360d3a8ff1ec2cdfbd21189090324ab38cc Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 5 Feb 2021 12:18:12 +0100 Subject: [PATCH 4/7] adds traversal check to deleteion --- app/workers/deletes.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index a329afe24..426dcf1ed 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -314,9 +314,10 @@ class DeletesV1 { $domain = $document->getAttribute('domain'); $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; + $checkTraversal = realpath($directory) === $directory; - if($domain && is_dir($directory)) { - array_map('unlink', glob("$directory/*.*")); + if($domain && $checkTraversal && is_dir($directory)) { + array_map('unlink', glob($directory.'/*.*')); rmdir($directory); Console::info("Deleted certificate files for {$domain}"); } else { From 4e9aa6fc3f0a5a77e409b51c4f348944e6119970 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 5 Feb 2021 12:56:46 +0100 Subject: [PATCH 5/7] add missing env variable to certificate worker --- app/views/install/compose.phtml | 1 + 1 file changed, 1 insertion(+) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 9d5c3d084..9ef20c491 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -207,6 +207,7 @@ services: - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_DOMAIN_TARGET - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA From a35f7e5dcea6ec9e0fefdc806245360835a99d45 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 5 Feb 2021 13:24:12 +0100 Subject: [PATCH 6/7] fix deletes workers volumes --- app/views/install/compose.phtml | 1 + docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 9ef20c491..faa88276d 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -179,6 +179,7 @@ services: volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw + - appwrite-certificates:/storage/certificates:rw environment: - _APP_ENV - _APP_REDIS_HOST diff --git a/docker-compose.yml b/docker-compose.yml index 69672b519..ca902e356 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -213,6 +213,7 @@ services: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - appwrite-functions:/storage/functions:rw + - appwrite-certificates:/storage/certificates:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: From ac42cc21eafe907187e0731e7c568bc2be8f876c Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 8 Feb 2021 10:01:52 +0100 Subject: [PATCH 7/7] fixes certificate removal on project deletion --- app/controllers/api/projects.php | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 8781e6edb..ed0d3a443 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -477,15 +477,22 @@ App::delete('/v1/projects/:projectId') ; foreach (['keys', 'webhooks', 'tasks', 'platforms', 'domains'] as $key) { // Delete all children (keys, webhooks, tasks [stop tasks?], platforms) - $list = $project->getAttribute('webhooks', []); - - foreach ($list as $document) { /* @var $document Document */ - if (!$consoleDB->deleteDocument($projectId)) { + $list = $project->getAttribute($key, []); + foreach ($list as $document) { + /** @var Document $document */ + if ($consoleDB->deleteDocument($document->getId())) { + if ($document->getCollection() == Database::SYSTEM_COLLECTION_DOMAINS) { + $deletes + ->setParam('type', DELETE_TYPE_CERTIFICATES) + ->setParam('document', $document) + ; + } + } else { throw new Exception('Failed to remove project document ('.$key.')] from DB', 500); } } } - + if (!$consoleDB->deleteDocument($project->getAttribute('teamId', null))) { throw new Exception('Failed to remove project team from DB', 500); }