Merge pull request #6934 from appwrite/feat-delegate-deletes

Delegate custom deletes

app/controllers/api/databases.php

@@ -680,9 +680,9 @@ App::delete('/v1/databases/:databaseId')
     ->param('databaseId', '', new UID(), 'Database ID.')
     ->inject('response')
     ->inject('dbForProject')
+    ->inject('queueForDatabase')
     ->inject('queueForEvents')
-    ->inject('queueForDeletes')
-    ->action(function (string $databaseId, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) {
+    ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) {
 
         $database = $dbForProject->getDocument('databases', $databaseId);
 
@@ -697,9 +697,9 @@ App::delete('/v1/databases/:databaseId')
         $dbForProject->deleteCachedDocument('databases', $database->getId());
         $dbForProject->deleteCachedCollection('databases_' . $database->getInternalId());
 
-        $queueForDeletes
-            ->setType(DELETE_TYPE_DOCUMENT)
-            ->setDocument($database);
+        $queueForDatabase
+            ->setType(DATABASE_TYPE_DELETE_DATABASE)
+            ->setDatabase($database);
 
         $queueForEvents
             ->setParam('databaseId', $database->getId())
@@ -1058,10 +1058,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
     ->param('collectionId', '', new UID(), 'Collection ID.')
     ->inject('response')
     ->inject('dbForProject')
-    ->inject('mode')
+    ->inject('queueForDatabase')
     ->inject('queueForEvents')
-    ->inject('queueForDeletes')
-    ->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, string $mode, Event $queueForEvents, Delete $queueForDeletes) {
+    ->inject('mode')
+    ->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, string $mode) {
 
         $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId));
 
@@ -1081,9 +1081,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
 
         $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
 
-        $queueForDeletes
-            ->setType(DELETE_TYPE_DOCUMENT)
-            ->setDocument($collection);
+        $queueForDatabase
+            ->setType(DATABASE_TYPE_DELETE_COLLECTION)
+            ->setDatabase($database)
+            ->setCollection($collection);
 
         $queueForEvents
             ->setContext('database', $database)
@@ -2337,8 +2338,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
             ->setType(DATABASE_TYPE_DELETE_ATTRIBUTE)
             ->setCollection($collection)
             ->setDatabase($db)
-            ->setDocument($attribute)
-        ;
+            ->setDocument($attribute);
 
         // Select response model based on type and format
         $type = $attribute->getAttribute('type');
@@ -3524,10 +3524,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
     ->inject('requestTimestamp')
     ->inject('response')
     ->inject('dbForProject')
-    ->inject('queueForEvents')
     ->inject('queueForDeletes')
+    ->inject('queueForEvents')
     ->inject('mode')
-    ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes, string $mode) {
+    ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, string $mode) {
         $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId));
 
         $isAPIKey = Auth::isAppUser(Authorization::getRoles());

app/init.php

@@ -146,6 +146,8 @@ const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute';
 const DATABASE_TYPE_CREATE_INDEX = 'createIndex';
 const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute';
 const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex';
+const DATABASE_TYPE_DELETE_COLLECTION = 'deleteCollection';
+const DATABASE_TYPE_DELETE_DATABASE = 'deleteDatabase';
 // Build Worker Types
 const BUILD_TYPE_DEPLOYMENT = 'deployment';
 const BUILD_TYPE_RETRY = 'retry';

composer.lock

@@ -2463,22 +2463,23 @@
         },
         {
             "name": "utopia-php/logger",
-            "version": "0.3.1",
+            "version": "0.3.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/logger.git",
-                "reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc"
+                "reference": "9151b7d16eab18d4c37c34643041cc0f33ca4a6c"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/logger/zipball/de623f1ec1c672c795d113dd25c5bf212f7ef4fc",
-                "reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc",
+                "url": "https://api.github.com/repos/utopia-php/logger/zipball/9151b7d16eab18d4c37c34643041cc0f33ca4a6c",
+                "reference": "9151b7d16eab18d4c37c34643041cc0f33ca4a6c",
                 "shasum": ""
             },
             "require": {
                 "php": ">=8.0"
             },
             "require-dev": {
+                "laravel/pint": "1.2.*",
                 "phpstan/phpstan": "1.9.x-dev",
                 "phpunit/phpunit": "^9.3",
                 "vimeo/psalm": "4.0.1"
@@ -2510,9 +2511,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/logger/issues",
-                "source": "https://github.com/utopia-php/logger/tree/0.3.1"
+                "source": "https://github.com/utopia-php/logger/tree/0.3.2"
             },
-            "time": "2023-02-10T15:52:50+00:00"
+            "time": "2023-10-16T08:16:19+00:00"
         },
         {
             "name": "utopia-php/messaging",
@@ -6019,5 +6020,5 @@
     "platform-overrides": {
         "php": "8.0"
     },
-    "plugin-api-version": "2.2.0"
+    "plugin-api-version": "2.6.0"
 }

src/Appwrite/Platform/Workers/Databases.php

@@ -4,13 +4,18 @@ namespace Appwrite\Platform\Workers;
 
 use Appwrite\Event\Event;
 use Appwrite\Messaging\Adapter\Realtime;
+use Appwrite\Utopia\Response\Model\Platform;
+use Exception;
+use Utopia\Audit\Audit;
 use Utopia\CLI\Console;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Exception\Authorization;
 use Utopia\Database\Exception\Conflict;
+use Utopia\Database\Exception\Restricted;
 use Utopia\Database\Exception\Structure;
 use Utopia\Database\Exception as DatabaseException;
+use Utopia\Database\Query;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
 
@@ -55,15 +60,13 @@ class Databases extends Action
         $document = new Document($payload['document'] ?? []);
         $database = new Document($payload['database'] ?? []);
 
-        if ($collection->isEmpty()) {
-            throw new \Exception('Missing collection');
-        }
-
-        if ($document->isEmpty()) {
-            throw new \Exception('Missing document');
+        if ($database->isEmpty()) {
+            throw new Exception('Missing database');
         }
 
         match (strval($type)) {
+            DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject),
+            DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject),
             DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject),
             DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject),
             DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject),
@@ -86,6 +89,12 @@ class Databases extends Action
      */
     private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
     {
+        if ($collection->isEmpty()) {
+            throw new Exception('Missing collection');
+        }
+        if ($attribute->isEmpty()) {
+            throw new Exception('Missing attribute');
+        }
 
         $projectId = $project->getId();
 
@@ -172,25 +181,7 @@ class Databases extends Action
                 );
             }
         } finally {
-            $target = Realtime::fromPayload(
-            // Pass first, most verbose event pattern
-                event: $events[0],
-                payload: $attribute,
-                project: $project,
-            );
-
-            Realtime::send(
-                projectId: 'console',
-                payload: $attribute->getArrayCopy(),
-                events: $events,
-                channels: $target['channels'],
-                roles: $target['roles'],
-                options: [
-                    'projectId' => $projectId,
-                    'databaseId' => $database->getId(),
-                    'collectionId' => $collection->getId()
-                ]
-            );
+            $this->trigger($database, $collection, $attribute, $project, $projectId, $events);
         }
 
         if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) {
@@ -214,6 +205,13 @@ class Databases extends Action
      **/
     private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
     {
+        if ($collection->isEmpty()) {
+            throw new Exception('Missing collection');
+        }
+        if ($attribute->isEmpty()) {
+            throw new Exception('Missing attribute');
+        }
+
         $projectId = $project->getId();
 
         $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [
@@ -283,25 +281,7 @@ class Databases extends Action
                 );
             }
         } finally {
-            $target = Realtime::fromPayload(
-            // Pass first, most verbose event pattern
-                event: $events[0],
-                payload: $attribute,
-                project: $project
-            );
-
-            Realtime::send(
-                projectId: 'console',
-                payload: $attribute->getArrayCopy(),
-                events: $events,
-                channels: $target['channels'],
-                roles: $target['roles'],
-                options: [
-                    'projectId' => $projectId,
-                    'databaseId' => $database->getId(),
-                    'collectionId' => $collection->getId()
-                ]
-            );
+            $this->trigger($database, $collection, $attribute, $project, $projectId, $events);
         }
 
         // The underlying database removes/rebuilds indexes when attribute is removed
@@ -379,6 +359,13 @@ class Databases extends Action
      */
     private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
     {
+        if ($collection->isEmpty()) {
+            throw new Exception('Missing collection');
+        }
+        if ($index->isEmpty()) {
+            throw new Exception('Missing index');
+        }
+
         $projectId = $project->getId();
 
         $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [
@@ -411,25 +398,7 @@ class Databases extends Action
                 $index->setAttribute('status', 'failed')
             );
         } finally {
-            $target = Realtime::fromPayload(
-            // Pass first, most verbose event pattern
-                event: $events[0],
-                payload: $index,
-                project: $project
-            );
-
-            Realtime::send(
-                projectId: 'console',
-                payload: $index->getArrayCopy(),
-                events: $events,
-                channels: $target['channels'],
-                roles: $target['roles'],
-                options: [
-                    'projectId' => $projectId,
-                    'databaseId' => $database->getId(),
-                    'collectionId' => $collection->getId()
-                ]
-            );
+            $this->trigger($database, $collection, $index, $project, $projectId, $events);
         }
 
         $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId);
@@ -450,6 +419,13 @@ class Databases extends Action
      */
     private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
     {
+        if ($collection->isEmpty()) {
+            throw new Exception('Missing collection');
+        }
+        if ($index->isEmpty()) {
+            throw new Exception('Missing index');
+        }
+
         $projectId = $project->getId();
 
         $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [
@@ -470,7 +446,6 @@ class Databases extends Action
         } catch (\Exception $e) {
             Console::error($e->getMessage());
 
-
             if ($e instanceof DatabaseException) {
                 $index->setAttribute('error', $e->getMessage());
             }
@@ -480,27 +455,167 @@ class Databases extends Action
                 $index->setAttribute('status', 'stuck')
             );
         } finally {
-            $target = Realtime::fromPayload(
-            // Pass first, most verbose event pattern
-                event: $events[0],
-                payload: $index,
-                project: $project
-            );
-
-            Realtime::send(
-                projectId: 'console',
-                payload: $index->getArrayCopy(),
-                events: $events,
-                channels: $target['channels'],
-                roles: $target['roles'],
-                options: [
-                    'projectId' => $projectId,
-                    'databaseId' => $database->getId(),
-                    'collectionId' => $collection->getId()
-                ]
-            );
+            $this->trigger($database, $collection, $index, $project, $projectId, $events);
         }
 
         $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collection->getId());
     }
+
+    /**
+     * @param Document $database
+     * @param Document $project
+     * @param $dbForProject
+     * @return void
+     * @throws Exception
+     */
+    protected function deleteDatabase(Document $database, Document $project, $dbForProject): void
+    {
+        $this->deleteByGroup('database_' . $database->getInternalId(), [], $dbForProject, function ($collection) use ($database, $project, $dbForProject) {
+            $this->deleteCollection($database, $collection, $project, $dbForProject);
+        });
+
+        $dbForProject->deleteCollection('database_' . $database->getInternalId());
+
+        $this->deleteAuditLogsByResource('database/' . $database->getId(), $project, $dbForProject);
+    }
+
+    /**
+     * @param Document $database
+     * @param Document $collection
+     * @param Document $project
+     * @param Database $dbForProject
+     * @return void
+     * @throws Authorization
+     * @throws Conflict
+     * @throws DatabaseException
+     * @throws Restricted
+     * @throws Structure
+     */
+    protected function deleteCollection(Document $database, Document $collection, Document $project, Database $dbForProject): void
+    {
+        if ($collection->isEmpty()) {
+            throw new Exception('Missing collection');
+        }
+
+        $collectionId = $collection->getId();
+        $collectionInternalId = $collection->getInternalId();
+        $databaseId = $database->getId();
+        $databaseInternalId = $database->getInternalId();
+
+        $relationships = \array_filter(
+            $collection->getAttribute('attributes'),
+            fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP
+        );
+
+        foreach ($relationships as $relationship) {
+            if (!$relationship['twoWay']) {
+                continue;
+            }
+            $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']);
+            $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']);
+            $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId());
+            $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId());
+        }
+
+        $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId());
+
+        $this->deleteByGroup('attributes', [
+            Query::equal('databaseInternalId', [$databaseInternalId]),
+            Query::equal('collectionInternalId', [$collectionInternalId])
+        ], $dbForProject);
+
+        $this->deleteByGroup('indexes', [
+            Query::equal('databaseInternalId', [$databaseInternalId]),
+            Query::equal('collectionInternalId', [$collectionInternalId])
+        ], $dbForProject);
+
+        $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project, $dbForProject);
+    }
+
+    /**
+     * @param string $resource
+     * @param Document $project
+     * @param Database $dbForProject
+     * @return void
+     * @throws Exception
+     */
+    protected function deleteAuditLogsByResource(string $resource, Document $project, Database $dbForProject): void
+    {
+        $this->deleteByGroup(Audit::COLLECTION, [
+            Query::equal('resource', [$resource])
+        ], $dbForProject);
+    }
+
+    /**
+     * @param string $collection collectionID
+     * @param array $queries
+     * @param Database $database
+     * @param callable|null $callback
+     * @return void
+     * @throws Exception
+     */
+    protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
+    {
+        $count = 0;
+        $chunk = 0;
+        $limit = 50;
+        $sum = $limit;
+
+        $executionStart = \microtime(true);
+
+        while ($sum === $limit) {
+            $chunk++;
+
+            $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
+
+            $sum = count($results);
+
+            Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');
+
+            foreach ($results as $document) {
+                if ($database->deleteDocument($document->getCollection(), $document->getId())) {
+                    Console::success('Deleted document "' . $document->getId() . '" successfully');
+
+                    if (\is_callable($callback)) {
+                        $callback($document);
+                    }
+                } else {
+                    Console::error('Failed to delete document: ' . $document->getId());
+                }
+                $count++;
+            }
+        }
+
+        $executionEnd = \microtime(true);
+
+        Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
+    }
+
+    protected function trigger(
+        Document $database,
+        Document $collection,
+        Document $attribute,
+        Document $project,
+        string $projectId,
+        array $events
+    ): void {
+        $target = Realtime::fromPayload(
+        // Pass first, most verbose event pattern
+            event: $events[0],
+            payload: $attribute,
+            project: $project,
+        );
+        Realtime::send(
+            projectId: 'console',
+            payload: $attribute->getArrayCopy(),
+            events: $events,
+            channels: $target['channels'],
+            roles: $target['roles'],
+            options: [
+                'projectId' => $projectId,
+                'databaseId' => $database->getId(),
+                'collectionId' => $collection->getId()
+            ]
+        );
+    }
 }