diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 55487eee5..eb6bd5ed4 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -235,7 +235,7 @@ App::delete('/v1/database/collections/:collectionId') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_NONE) ->param('collectionId', '', new UID(), 'Collection unique ID.') - ->action(function ($collectionId, $response, $projectDB, $webhooks, $audits) { + ->action(function ($collectionId, $response, $projectDB, $webhooks, $audits, $deletes) { /** @var Appwrite\Utopia\Response $response */ /** @var Appwrite\Database\Database $projectDB */ /** @var Appwrite\Event\Event $webhooks */ @@ -250,7 +250,11 @@ App::delete('/v1/database/collections/:collectionId') if (!$projectDB->deleteDocument($collectionId)) { throw new Exception('Failed to remove collection from DB', 500); } - + + $deletes + ->setParam('document', $collection) + ; + $webhooks ->setParam('payload', $response->output($collection, Response::MODEL_COLLECTION)) ; @@ -262,7 +266,7 @@ App::delete('/v1/database/collections/:collectionId') ; $response->noContent(); - }, ['response', 'projectDB', 'webhooks', 'audits']); + }, ['response', 'projectDB', 'webhooks', 'audits', 'deletes']); App::post('/v1/database/collections/:collectionId/documents') ->desc('Create Document') diff --git a/app/workers/deletes.php b/app/workers/deletes.php index d5f7c9284..d017ee1fe 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -10,31 +10,43 @@ use Appwrite\Database\Database; use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Document; +use Appwrite\Database\Validator\Authorization; use Appwrite\Storage\Device\Local; +use Utopia\CLI\Console; use Utopia\Config\Config; class DeletesV1 { public $args = []; + protected $consoleDB = null; + public function setUp(): void { } public function perform() { + $projectId = $this->args['projectId']; $document = $this->args['document']; + $document = new Document($document); - switch ($document->getCollection()) { + switch (strval($document->getCollection())) { case Database::SYSTEM_COLLECTION_PROJECTS: $this->deleteProject($document); break; - case Database::SYSTEM_COLLECTION_USERS: - $this->deleteUser($document); + case Database::SYSTEM_COLLECTION_FUNCTIONS: + $this->deleteFunction($document, $projectId); + break; + case Database::SYSTEM_COLLECTION_USERS: + $this->deleteUser($document, $projectId); + break; + case Database::SYSTEM_COLLECTION_COLLECTIONS: + $this->deleteDocuments($document,$projectId); break; - default: + Console::error('No lazy delete operation available for document of type: '.$document->getCollection()); break; } } @@ -43,50 +55,163 @@ class DeletesV1 { // ... Remove environment for this job } + + protected function deleteDocuments(Document $document, $projectId) + { + $collectionId = $document->getId(); + + // Delete Documents in the deleted collection + $this->deleteByGroup([ + '$collection='.$collectionId + ], $this->getProjectDB($projectId)); + } protected function deleteProject(Document $document) { - global $register; - - $consoleDB = new Database(); - $consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); - $consoleDB->setNamespace('app_console'); // Main DB - $consoleDB->setMocks(Config::getParam('collections', [])); - // Delete all DBs - $consoleDB->deleteNamespace($document->getId()); + $this->getConsoleDB()->deleteNamespace($document->getId()); $uploads = new Local(APP_STORAGE_UPLOADS.'/app-'.$document->getId()); $cache = new Local(APP_STORAGE_CACHE.'/app-'.$document->getId()); + // Delete all storage directories $uploads->delete($uploads->getRoot(), true); $cache->delete($cache->getRoot(), true); } - protected function deleteUser(Document $user) + protected function deleteUser(Document $document, $projectId) { - global $projectDB; - - $tokens = $user->getAttribute('tokens', []); + $tokens = $document->getAttribute('tokens', []); foreach ($tokens as $token) { - if (!$projectDB->deleteDocument($token->getId())) { + if (!$this->getProjectDB($projectId)->deleteDocument($token->getId())) { throw new Exception('Failed to remove token from DB', 500); } } - $memberships = $projectDB->getCollection([ - 'limit' => 2000, // TODO add members limit - 'offset' => 0, - 'filters' => [ - '$collection='.Database::SYSTEM_COLLECTION_MEMBERSHIPS, - 'userId='.$user->getId(), - ], - ]); + // Delete Memberships + $this->deleteByGroup([ + '$collection='.Database::SYSTEM_COLLECTION_MEMBERSHIPS, + 'userId='.$document->getId(), + ], $this->getProjectDB($projectId)); + } - foreach ($memberships as $membership) { - if (!$projectDB->deleteDocument($membership->getId())) { - throw new Exception('Failed to remove team membership from DB', 500); + protected function deleteFunction(Document $document, $projectId) + { + $projectDB = $this->getProjectDB($projectId); + $device = new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId); + + // Delete Tags + $this->deleteByGroup([ + '$collection='.Database::SYSTEM_COLLECTION_TAGS, + 'functionId='.$document->getId(), + ], $projectDB, function(Document $document) use ($device) { + + if ($device->delete($document->getAttribute('path', ''))) { + Console::success('Delete code tag: '.$document->getAttribute('path', '')); + } + else { + Console::error('Dailed to delete code tag: '.$document->getAttribute('path', '')); + } + }); + + // Delete Executions + $this->deleteByGroup([ + '$collection='.Database::SYSTEM_COLLECTION_EXECUTIONS, + 'functionId='.$document->getId(), + ], $projectDB); + } + + protected function deleteById(Document $document, Database $database, callable $callback = null): bool + { + Authorization::disable(); + + if($database->deleteDocument($document->getId())) { + Console::success('Deleted document "'.$document->getId().'" successfully'); + + if(is_callable($callback)) { + $callback($document); + } + + return true; + } + else { + Console::error('Failed to delete document: '.$document->getId()); + return false; + } + + Authorization::reset(); + } + + protected function deleteByGroup(array $filters, Database $database, callable $callback = null) + { + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while($sum === $limit) { + $chunk++; + + Authorization::disable(); + + $results = $database->getCollection([ + 'limit' => $limit, + 'offset' => 0, + 'orderField' => '$id', + 'orderType' => 'ASC', + 'orderCast' => 'string', + 'filters' => $filters, + ]); + + Authorization::reset(); + + $sum = count($results); + + Console::info('Deleting chunk #'.$chunk.'. Found '.$sum.' documents'); + + foreach ($results as $document) { + $this->deleteById($document, $database, $callback); + $count++; } } + + $executionEnd = \microtime(true); + + Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } -} + + /** + * @return Database; + */ + protected function getConsoleDB(): Database + { + global $register; + + if($this->consoleDB === null) { + $this->consoleDB = new Database(); + $this->consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); + $this->consoleDB->setNamespace('app_console'); // Main DB + $this->consoleDB->setMocks(Config::getParam('collections', [])); + } + + return $this->consoleDB; + } + + /** + * @return Database; + */ + protected function getProjectDB($projectId): Database + { + global $register; + + $projectDB = new Database(); + $projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); + $projectDB->setNamespace('app_'.$projectId); // Main DB + $projectDB->setMocks(Config::getParam('collections', [])); + + return $projectDB; + } +} \ No newline at end of file diff --git a/debug/nginx-stdout---supervisor-DAF7HT.log b/debug/nginx-stdout---supervisor-DAF7HT.log new file mode 100644 index 000000000..e69de29bb diff --git a/debug/php7-fpm-stdout---supervisor-TGTF4g.log b/debug/php7-fpm-stdout---supervisor-TGTF4g.log new file mode 100644 index 000000000..e69de29bb diff --git a/debug/supervisord.pid b/debug/supervisord.pid new file mode 100644 index 000000000..920a13966 --- /dev/null +++ b/debug/supervisord.pid @@ -0,0 +1 @@ +43 diff --git a/debug/v1-schedule-stdout---supervisor-cNs2Cv.log b/debug/v1-schedule-stdout---supervisor-cNs2Cv.log new file mode 100644 index 000000000..7570e1d21 --- /dev/null +++ b/debug/v1-schedule-stdout---supervisor-cNs2Cv.log @@ -0,0 +1 @@ +*** Starting scheduler worker