diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index ed21760c67..d132273b5d 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -680,9 +680,9 @@ App::delete('/v1/databases/:databaseId') ->param('databaseId', '', new UID(), 'Database ID.') ->inject('response') ->inject('dbForProject') + ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForDeletes') - ->action(function (string $databaseId, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) { + ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $database = $dbForProject->getDocument('databases', $databaseId); @@ -697,9 +697,9 @@ App::delete('/v1/databases/:databaseId') $dbForProject->deleteCachedDocument('databases', $database->getId()); $dbForProject->deleteCachedCollection('databases_' . $database->getInternalId()); - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($database); + $queueForDatabase + ->setType(DATABASE_TYPE_DELETE_DATABASE) + ->setDatabase($database); $queueForEvents ->setParam('databaseId', $database->getId()) @@ -1058,10 +1058,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId') ->param('collectionId', '', new UID(), 'Collection ID.') ->inject('response') ->inject('dbForProject') - ->inject('mode') + ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForDeletes') - ->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, string $mode, Event $queueForEvents, Delete $queueForDeletes) { + ->inject('mode') + ->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, string $mode) { $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); @@ -1081,9 +1081,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId') $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($collection); + $queueForDatabase + ->setType(DATABASE_TYPE_DELETE_COLLECTION) + ->setDatabase($database) + ->setCollection($collection); $queueForEvents ->setContext('database', $database) @@ -2337,8 +2338,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->setType(DATABASE_TYPE_DELETE_ATTRIBUTE) ->setCollection($collection) ->setDatabase($db) - ->setDocument($attribute) - ; + ->setDocument($attribute); // Select response model based on type and format $type = $attribute->getAttribute('type'); @@ -3524,10 +3524,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') - ->inject('queueForEvents') ->inject('queueForDeletes') + ->inject('queueForEvents') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes, string $mode) { + ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, string $mode) { $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); diff --git a/app/init.php b/app/init.php index bc1e95e077..df9e6be990 100644 --- a/app/init.php +++ b/app/init.php @@ -146,6 +146,8 @@ const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute'; const DATABASE_TYPE_CREATE_INDEX = 'createIndex'; const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute'; const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex'; +const DATABASE_TYPE_DELETE_COLLECTION = 'deleteCollection'; +const DATABASE_TYPE_DELETE_DATABASE = 'deleteDatabase'; // Build Worker Types const BUILD_TYPE_DEPLOYMENT = 'deployment'; const BUILD_TYPE_RETRY = 'retry'; diff --git a/composer.lock b/composer.lock index 60a8bf6393..2751ee2098 100644 --- a/composer.lock +++ b/composer.lock @@ -2463,22 +2463,23 @@ }, { "name": "utopia-php/logger", - "version": "0.3.1", + "version": "0.3.2", "source": { "type": "git", "url": "https://github.com/utopia-php/logger.git", - "reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc" + "reference": "9151b7d16eab18d4c37c34643041cc0f33ca4a6c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/logger/zipball/de623f1ec1c672c795d113dd25c5bf212f7ef4fc", - "reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc", + "url": "https://api.github.com/repos/utopia-php/logger/zipball/9151b7d16eab18d4c37c34643041cc0f33ca4a6c", + "reference": "9151b7d16eab18d4c37c34643041cc0f33ca4a6c", "shasum": "" }, "require": { "php": ">=8.0" }, "require-dev": { + "laravel/pint": "1.2.*", "phpstan/phpstan": "1.9.x-dev", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.0.1" @@ -2510,9 +2511,9 @@ ], "support": { "issues": "https://github.com/utopia-php/logger/issues", - "source": "https://github.com/utopia-php/logger/tree/0.3.1" + "source": "https://github.com/utopia-php/logger/tree/0.3.2" }, - "time": "2023-02-10T15:52:50+00:00" + "time": "2023-10-16T08:16:19+00:00" }, { "name": "utopia-php/messaging", @@ -6019,5 +6020,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 8dca081495..7d0e015270 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -4,13 +4,18 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Event; use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Utopia\Response\Model\Platform; +use Exception; +use Utopia\Audit\Audit; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; +use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -55,15 +60,13 @@ class Databases extends Action $document = new Document($payload['document'] ?? []); $database = new Document($payload['database'] ?? []); - if ($collection->isEmpty()) { - throw new \Exception('Missing collection'); - } - - if ($document->isEmpty()) { - throw new \Exception('Missing document'); + if ($database->isEmpty()) { + throw new Exception('Missing database'); } match (strval($type)) { + DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject), + DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject), DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), @@ -86,6 +89,12 @@ class Databases extends Action */ private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($attribute->isEmpty()) { + throw new Exception('Missing attribute'); + } $projectId = $project->getId(); @@ -172,25 +181,7 @@ class Databases extends Action ); } } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $attribute, - project: $project, - ); - - Realtime::send( - projectId: 'console', - payload: $attribute->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $attribute, $project, $projectId, $events); } if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) { @@ -214,6 +205,13 @@ class Databases extends Action **/ private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($attribute->isEmpty()) { + throw new Exception('Missing attribute'); + } + $projectId = $project->getId(); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ @@ -283,25 +281,7 @@ class Databases extends Action ); } } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $attribute, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $attribute->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $attribute, $project, $projectId, $events); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -379,6 +359,13 @@ class Databases extends Action */ private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($index->isEmpty()) { + throw new Exception('Missing index'); + } + $projectId = $project->getId(); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ @@ -411,25 +398,7 @@ class Databases extends Action $index->setAttribute('status', 'failed') ); } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $index, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $index->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $index, $project, $projectId, $events); } $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); @@ -450,6 +419,13 @@ class Databases extends Action */ private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($index->isEmpty()) { + throw new Exception('Missing index'); + } + $projectId = $project->getId(); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ @@ -470,7 +446,6 @@ class Databases extends Action } catch (\Exception $e) { Console::error($e->getMessage()); - if ($e instanceof DatabaseException) { $index->setAttribute('error', $e->getMessage()); } @@ -480,27 +455,167 @@ class Databases extends Action $index->setAttribute('status', 'stuck') ); } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $index, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $index->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $index, $project, $projectId, $events); } $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } + + /** + * @param Document $database + * @param Document $project + * @param $dbForProject + * @return void + * @throws Exception + */ + protected function deleteDatabase(Document $database, Document $project, $dbForProject): void + { + $this->deleteByGroup('database_' . $database->getInternalId(), [], $dbForProject, function ($collection) use ($database, $project, $dbForProject) { + $this->deleteCollection($database, $collection, $project, $dbForProject); + }); + + $dbForProject->deleteCollection('database_' . $database->getInternalId()); + + $this->deleteAuditLogsByResource('database/' . $database->getId(), $project, $dbForProject); + } + + /** + * @param Document $database + * @param Document $collection + * @param Document $project + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws DatabaseException + * @throws Restricted + * @throws Structure + */ + protected function deleteCollection(Document $database, Document $collection, Document $project, Database $dbForProject): void + { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + + $collectionId = $collection->getId(); + $collectionInternalId = $collection->getInternalId(); + $databaseId = $database->getId(); + $databaseInternalId = $database->getInternalId(); + + $relationships = \array_filter( + $collection->getAttribute('attributes'), + fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP + ); + + foreach ($relationships as $relationship) { + if (!$relationship['twoWay']) { + continue; + } + $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); + $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); + $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); + $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); + } + + $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); + + $this->deleteByGroup('attributes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteByGroup('indexes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project, $dbForProject); + } + + /** + * @param string $resource + * @param Document $project + * @param Database $dbForProject + * @return void + * @throws Exception + */ + protected function deleteAuditLogsByResource(string $resource, Document $project, Database $dbForProject): void + { + $this->deleteByGroup(Audit::COLLECTION, [ + Query::equal('resource', [$resource]) + ], $dbForProject); + } + + /** + * @param string $collection collectionID + * @param array $queries + * @param Database $database + * @param callable|null $callback + * @return void + * @throws Exception + */ + protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + { + $count = 0; + $chunk = 0; + $limit = 50; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); + + foreach ($results as $document) { + if ($database->deleteDocument($document->getCollection(), $document->getId())) { + Console::success('Deleted document "' . $document->getId() . '" successfully'); + + if (\is_callable($callback)) { + $callback($document); + } + } else { + Console::error('Failed to delete document: ' . $document->getId()); + } + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); + } + + protected function trigger( + Document $database, + Document $collection, + Document $attribute, + Document $project, + string $projectId, + array $events + ): void { + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $events[0], + payload: $attribute, + project: $project, + ); + Realtime::send( + projectId: 'console', + payload: $attribute->getArrayCopy(), + events: $events, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'projectId' => $projectId, + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId() + ] + ); + } }