From 4b0ef4598bd0eaae87d3b80dde7e81481ec1e0ac Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 11 Jan 2023 19:37:06 +1300 Subject: [PATCH 1/5] Fix deletes worker not deleting project database tables --- app/workers/deletes.php | 50 +++++++++++++++--- src/Appwrite/Resque/Worker.php | 92 ++++++++++++++++++++++------------ 2 files changed, 105 insertions(+), 37 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 5dc7e8d737..ca5299e8af 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -172,6 +172,7 @@ class DeletesV1 extends Worker /** * @param Document $document database document * @param string $projectId + * @throws Exception */ protected function deleteDatabase(Document $document, string $projectId): void { @@ -191,6 +192,7 @@ class DeletesV1 extends Worker /** * @param Document $document teams document * @param string $projectId + * @throws Exception */ protected function deleteCollection(Document $document, string $projectId): void { @@ -217,6 +219,7 @@ class DeletesV1 extends Worker /** * @param string $hourlyUsageRetentionDatetime + * @throws Exception */ protected function deleteUsageStats(string $hourlyUsageRetentionDatetime) { @@ -232,6 +235,7 @@ class DeletesV1 extends Worker /** * @param Document $document teams document * @param string $projectId + * @throws Exception */ protected function deleteMemberships(Document $document, string $projectId): void { @@ -245,13 +249,37 @@ class DeletesV1 extends Worker /** * @param Document $document project document + * @throws Exception */ protected function deleteProject(Document $document): void { - $projectId = $document->getId(); + // Delete project tables + $dbForProject = $this->getProjectDBFromDocument($document); - // Delete all DBs - $this->getProjectDB($projectId)->delete($projectId); + $limit = 50; + $offset = 0; + + while (true) { + $collections = $dbForProject->listCollections($limit, $offset); + + if (empty($collections)) { + break; + } + + foreach ($collections as $collection) { + $dbForProject->deleteCollection($collection->getId()); + } + + $offset += $limit; + } + + // Delete metadata tables + try { + $dbForProject->deleteCollection('_metadata'); + } catch (Exception) { + // Ignore: deleteCollection tries to delete a metadata entry after the collection is deleted, + // which will throw an exception here because the metadata collection is already deleted. + } // Delete all storage directories $uploads = $this->getFilesDevice($document->getId()); @@ -264,6 +292,7 @@ class DeletesV1 extends Worker /** * @param Document $document user document * @param string $projectId + * @throws Exception */ protected function deleteUser(Document $document, string $projectId): void { @@ -305,6 +334,7 @@ class DeletesV1 extends Worker /** * @param string $datetime + * @throws Exception */ protected function deleteExecutionLogs(string $datetime): void { @@ -337,6 +367,7 @@ class DeletesV1 extends Worker /** * @param string $datetime + * @throws Exception */ protected function deleteRealtimeUsage(string $datetime): void { @@ -393,6 +424,7 @@ class DeletesV1 extends Worker /** * @param string $resource * @param string $projectId + * @throws Exception */ protected function deleteAuditLogsByResource(string $resource, string $projectId): void { @@ -406,6 +438,7 @@ class DeletesV1 extends Worker /** * @param Document $document function document * @param string $projectId + * @throws Exception */ protected function deleteFunction(Document $document, string $projectId): void { @@ -479,6 +512,7 @@ class DeletesV1 extends Worker /** * @param Document $document deployment document * @param string $projectId + * @throws Exception */ protected function deleteDeployment(Document $document, string $projectId): void { @@ -528,9 +562,10 @@ class DeletesV1 extends Worker /** * @param Document $document to be deleted * @param Database $database to delete it from - * @param callable $callback to perform after document is deleted + * @param callable|null $callback to perform after document is deleted * * @return bool + * @throws \Utopia\Database\Exception\Authorization */ protected function deleteById(Document $document, Database $database, callable $callback = null): bool { @@ -550,6 +585,7 @@ class DeletesV1 extends Worker /** * @param callable $callback + * @throws Exception */ protected function deleteForProjectIds(callable $callback): void { @@ -584,9 +620,10 @@ class DeletesV1 extends Worker /** * @param string $collection collectionID - * @param Query[] $queries + * @param array $queries * @param Database $database - * @param callable $callback + * @param callable|null $callback + * @throws Exception */ protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void { @@ -620,6 +657,7 @@ class DeletesV1 extends Worker /** * @param Document $document certificates document + * @throws \Utopia\Database\Exception\Authorization */ protected function deleteCertificates(Document $document): void { diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index dd7cebd084..b01bace9ea 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -2,22 +2,23 @@ namespace Appwrite\Resque; +use Exception; use Utopia\App; -use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Cache; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; use Utopia\Storage\Device; -use Utopia\Storage\Storage; -use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Linode; -use Utopia\Storage\Device\Wasabi; -use Utopia\Storage\Device\Backblaze; +use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; -use Exception; -use Utopia\Database\Validator\Authorization; +use Utopia\Storage\Device\Wasabi; +use Utopia\Storage\Storage; abstract class Worker { @@ -53,7 +54,7 @@ abstract class Worker * @return void * @throws \Exception|\Throwable */ - public function init() + public function init(): void { throw new Exception("Please implement init method in worker"); } @@ -65,7 +66,7 @@ abstract class Worker * @return void * @throws \Exception|\Throwable */ - public function run() + public function run(): void { throw new Exception("Please implement run method in worker"); } @@ -77,7 +78,7 @@ abstract class Worker * @return void * @throws \Exception|\Throwable */ - public function shutdown() + public function shutdown(): void { throw new Exception("Please implement shutdown method in worker"); } @@ -151,17 +152,18 @@ abstract class Worker /** * Register callback. Will be executed when error occurs. * @param callable $callback - * @param Throwable $error - * @return self + * @return void */ public static function error(callable $callback): void { - \array_push(self::$errorCallbacks, $callback); + self::$errorCallbacks[] = $callback; } + /** * Get internal project database * @param string $projectId * @return Database + * @throws Exception */ protected function getProjectDB(string $projectId): Database { @@ -177,9 +179,23 @@ abstract class Worker return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId()); } + /** + * Get internal project database given the project document + * + * Allows avoiding race conditions when modifying the projects collection + * @param Document $project + * @return Database + * @throws Exception + */ + protected function getProjectDBFromDocument(Document $project): Database + { + return $this->getDB(self::DATABASE_PROJECT, project: $project); + } + /** * Get console database * @return Database + * @throws Exception */ protected function getConsoleDB(): Database { @@ -187,24 +203,35 @@ abstract class Worker } /** - * Get console database - * @param string $type One of (internal, external, console) - * @param string $projectId of internal or external DB + * Get database + * @param string $type One of (project, console) + * @param string $projectId of project or console DB + * @param string $projectInternalId + * @param Document|null $project * @return Database + * @throws Exception */ - private function getDB(string $type, string $projectId = '', string $projectInternalId = ''): Database - { + private function getDB( + string $type, + string $projectId = '', + string $projectInternalId = '', + ?Document $project = null + ): Database { global $register; - $namespace = ''; $sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary + if ($project !== null) { + $projectId = $project->getId(); + $projectInternalId = $project->getInternalId(); + } + switch ($type) { case self::DATABASE_PROJECT: if (!$projectId) { throw new \Exception('ProjectID not provided - cannot get database'); } - $namespace = "_{$projectInternalId}"; + $namespace = "_$projectInternalId"; break; case self::DATABASE_CONSOLE: $namespace = "_console"; @@ -212,12 +239,11 @@ abstract class Worker break; default: throw new \Exception('Unknown database type: ' . $type); - break; } $attempts = 0; - do { + while (true) { try { $attempts++; $cache = new Cache(new RedisCache($register->get('cache'))); @@ -225,8 +251,12 @@ abstract class Worker $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $database->setNamespace($namespace); // Main DB - if (!empty($projectId) && !$database->getDocument('projects', $projectId)->isEmpty()) { - throw new \Exception("Project does not exist: {$projectId}"); + if ( + $project === null + && !empty($projectId) + && !$database->getDocument('projects', $projectId)->isEmpty() + ) { + throw new \Exception("Project does not exist: $projectId"); } if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), Database::METADATA)) { @@ -235,13 +265,13 @@ abstract class Worker break; // leave loop if successful } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); + Console::warning("Database not ready. Retrying connection ($attempts)..."); if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { throw new \Exception('Failed to connect to database: ' . $e->getMessage()); } sleep($sleep); } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + } return $database; } @@ -251,7 +281,7 @@ abstract class Worker * @param string $projectId of the project * @return Device */ - protected function getFunctionsDevice($projectId): Device + protected function getFunctionsDevice(string $projectId): Device { return $this->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); } @@ -261,7 +291,7 @@ abstract class Worker * @param string $projectId of the project * @return Device */ - protected function getFilesDevice($projectId): Device + protected function getFilesDevice(string $projectId): Device { return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); } @@ -272,7 +302,7 @@ abstract class Worker * @param string $projectId of the project * @return Device */ - protected function getBuildsDevice($projectId): Device + protected function getBuildsDevice(string $projectId): Device { return $this->getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId); } @@ -282,7 +312,7 @@ abstract class Worker * @param string $root path of the device * @return Device */ - public function getDevice($root): Device + public function getDevice(string $root): Device { switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { case Storage::DEVICE_LOCAL: From d49de9c6cfe5e6302391bb9e7989b6000f329cd5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 12 Jan 2023 22:27:20 +1300 Subject: [PATCH 2/5] Use getProjectDB instead of new function --- app/workers/deletes.php | 2 +- src/Appwrite/Resque/Worker.php | 31 ++++++++++--------------------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index ca5299e8af..605f989ee1 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -254,7 +254,7 @@ class DeletesV1 extends Worker protected function deleteProject(Document $document): void { // Delete project tables - $dbForProject = $this->getProjectDBFromDocument($document); + $dbForProject = $this->getProjectDB($document->getId(), $document); $limit = 50; $offset = 0; diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index b01bace9ea..6de22c6441 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -165,31 +165,20 @@ abstract class Worker * @return Database * @throws Exception */ - protected function getProjectDB(string $projectId): Database + protected function getProjectDB(string $projectId, ?Document $project = null): Database { - $consoleDB = $this->getConsoleDB(); + if ($project === null) { + $consoleDB = $this->getConsoleDB(); - if ($projectId === 'console') { - return $consoleDB; + if ($projectId === 'console') { + return $consoleDB; + } + + /** @var Document $project */ + $project = Authorization::skip(fn() => $consoleDB->getDocument('projects', $projectId)); } - /** @var Document $project */ - $project = Authorization::skip(fn() => $consoleDB->getDocument('projects', $projectId)); - - return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId()); - } - - /** - * Get internal project database given the project document - * - * Allows avoiding race conditions when modifying the projects collection - * @param Document $project - * @return Database - * @throws Exception - */ - protected function getProjectDBFromDocument(Document $project): Database - { - return $this->getDB(self::DATABASE_PROJECT, project: $project); + return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId(), $project); } /** From 004bb82688e401385aebcc898fb24cf9129da469 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 12 Jan 2023 22:42:07 +1300 Subject: [PATCH 3/5] Fix functions + builds not deleted --- app/workers/deletes.php | 12 +++++++++--- src/Appwrite/Resque/Worker.php | 5 +++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 605f989ee1..2fbaf73b18 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -253,8 +253,10 @@ class DeletesV1 extends Worker */ protected function deleteProject(Document $document): void { + $projectId = $document->getId(); + // Delete project tables - $dbForProject = $this->getProjectDB($document->getId(), $document); + $dbForProject = $this->getProjectDB($projectId, $document); $limit = 50; $offset = 0; @@ -282,10 +284,14 @@ class DeletesV1 extends Worker } // Delete all storage directories - $uploads = $this->getFilesDevice($document->getId()); - $cache = new Local(APP_STORAGE_CACHE . '/app-' . $document->getId()); + $uploads = $this->getFilesDevice($projectId); + $functions = $this->getFunctionsDevice($projectId); + $builds = $this->getBuildsDevice($projectId); + $cache = $this->getCacheDevice($projectId); $uploads->delete($uploads->getRoot(), true); + $functions->delete($functions->getRoot(), true); + $builds->delete($builds->getRoot(), true); $cache->delete($cache->getRoot(), true); } diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 6de22c6441..bfaab0eeae 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -296,6 +296,11 @@ abstract class Worker return $this->getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId); } + protected function getCacheDevice(string $projectId): Device + { + return $this->getDevice(APP_STORAGE_CACHE . '/app-' . $projectId); + } + /** * Get Device based on selected storage environment * @param string $root path of the device From 536d36be7d346159429a1aca090085e72e3d3ba3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 17 Jan 2023 17:31:14 +1300 Subject: [PATCH 4/5] Fix delete recursion --- app/workers/deletes.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 2fbaf73b18..e840f86097 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -258,11 +258,8 @@ class DeletesV1 extends Worker // Delete project tables $dbForProject = $this->getProjectDB($projectId, $document); - $limit = 50; - $offset = 0; - while (true) { - $collections = $dbForProject->listCollections($limit, $offset); + $collections = $dbForProject->listCollections(); if (empty($collections)) { break; @@ -271,8 +268,6 @@ class DeletesV1 extends Worker foreach ($collections as $collection) { $dbForProject->deleteCollection($collection->getId()); } - - $offset += $limit; } // Delete metadata tables From c9f4b7ca23a6bec131e7488e5b9584ba2e4e1733 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 27 Jan 2023 16:46:38 +1300 Subject: [PATCH 5/5] Delete project domains and certificates --- app/workers/deletes.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index e840f86097..d866e088b9 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -255,6 +255,17 @@ class DeletesV1 extends Worker { $projectId = $document->getId(); + // Delete project domains and certificates + $dbForConsole = $this->getConsoleDB(); + + $domains = $dbForConsole->find('domains', [ + Query::equal('projectInternalId', [$document->getInternalId()]) + ]); + + foreach ($domains as $domain) { + $this->deleteCertificates($domain); + } + // Delete project tables $dbForProject = $this->getProjectDB($projectId, $document);