Merge pull request #7460 from appwrite/feat-maintenance-delete-expired-targets

Feat maintenance delete expired targets

app/config/collections.php

@@ -2036,6 +2036,17 @@ $commonCollections = [
                 'array' => false,
                 'filters' => [],
             ],
+            [
+                '$id' => ID::custom('expired'),
+                'type' => Database::VAR_BOOLEAN,
+                'format' => '',
+                'size' => 0,
+                'signed' => true,
+                'required' => false,
+                'default' => false,
+                'array' => false,
+                'filters' => [],
+            ],
         ],
         'indexes' => [
             [

app/init.php

@@ -174,6 +174,7 @@ const DELETE_TYPE_CACHE_BY_RESOURCE  = 'cacheByResource';
 const DELETE_TYPE_SCHEDULES = 'schedules';
 const DELETE_TYPE_TOPIC = 'topic';
 const DELETE_TYPE_TARGET = 'target';
+const DELETE_TYPE_EXPIRED_TARGETS = 'invalid_targets';
 // Mail Types
 const MAIL_TYPE_VERIFICATION = 'verification';
 const MAIL_TYPE_MAGIC_SESSION = 'magicSession';

src/Appwrite/Platform/Tasks/Maintenance.php

@@ -59,6 +59,7 @@ class Maintenance extends Action
             $this->renewCertificates($dbForConsole, $queueForCertificates);
             $this->notifyDeleteCache($cacheRetention, $queueForDeletes);
             $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
+            $this->notifyDeleteTargets($queueForDeletes);
         }, $interval);
     }
 
@@ -161,8 +162,7 @@ class Maintenance extends Action
 
     private function notifyDeleteCache($interval, Delete $queueForDeletes): void
     {
-
-        ($queueForDeletes)
+        $queueForDeletes
             ->setType(DELETE_TYPE_CACHE_BY_TIMESTAMP)
             ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
@@ -170,10 +170,16 @@ class Maintenance extends Action
 
     private function notifyDeleteSchedules($interval, Delete $queueForDeletes): void
     {
-
-        ($queueForDeletes)
+        $queueForDeletes
             ->setType(DELETE_TYPE_SCHEDULES)
             ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
+
+    private function notifyDeleteTargets(Delete $queueForDeletes): void
+    {
+        $queueForDeletes
+            ->setType(DELETE_TYPE_EXPIRED_TARGETS)
+            ->trigger();
+    }
 }

src/Appwrite/Platform/Tasks/ScheduleBase.php

@@ -73,6 +73,7 @@ abstract class ScheduleBase extends Action
                 '$id' => $schedule->getId(),
                 'resourceId' => $schedule->getAttribute('resourceId'),
                 'schedule' => $schedule->getAttribute('schedule'),
+                'active' => $schedule->getAttribute('active'),
                 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'),
                 'project' => $project, // TODO: @Meldiron Send only ID to worker to reduce memory usage here
                 'resource' => $resource, // TODO: @Meldiron Send only ID to worker to reduce memory usage here

src/Appwrite/Platform/Tasks/ScheduleMessages.php

@@ -33,6 +33,10 @@ class ScheduleMessages extends ScheduleBase
     protected function enqueueResources(Group $pools, Database $dbForConsole): void
     {
         foreach ($this->schedules as $schedule) {
+            if (!$schedule['active']) {
+                continue;
+            }
+
             $now = DateTime::now();
             $scheduledAt = DateTime::formatTz($schedule['schedule']);
 
@@ -44,24 +48,17 @@ class ScheduleMessages extends ScheduleBase
                 $queue = $pools->get('queue')->pop();
                 $connection = $queue->getResource();
                 $queueForMessaging = new Messaging($connection);
-                $queueForDeletes = new Delete($connection);
 
                 $queueForMessaging
                     ->setMessageId($schedule['resourceId'])
                     ->setProject($schedule['project'])
                     ->trigger();
 
-                $dbForConsole->updateDocument(
+                $dbForConsole->deleteDocument(
                     'schedules',
                     $schedule['$id'],
-                    new Document(['active' => false])
                 );
 
-                $queueForDeletes
-                    ->setType(DELETE_TYPE_SCHEDULES)
-                    ->setDocument($schedule)
-                    ->trigger();
-
                 $queue->reclaim();
 
                 unset($this->schedules[$schedule['resourceId']]);

src/Appwrite/Platform/Workers/Deletes.php

@@ -122,11 +122,9 @@ class Deletes extends Action
                         break;
                 }
                 break;
-
             case DELETE_TYPE_EXECUTIONS:
                 $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
                 break;
-
             case DELETE_TYPE_AUDIT:
                 if (!$project->isEmpty()) {
                     $this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
@@ -139,11 +137,9 @@ class Deletes extends Action
             case DELETE_TYPE_ABUSE:
                 $this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention);
                 break;
-
             case DELETE_TYPE_REALTIME:
                 $this->deleteRealtimeUsage($dbForConsole, $datetime);
                 break;
-
             case DELETE_TYPE_SESSIONS:
                 $this->deleteExpiredSessions($project, $getProjectDB);
                 break;
@@ -157,17 +153,19 @@ class Deletes extends Action
                 $this->deleteCacheByDate($project, $getProjectDB, $datetime);
                 break;
             case DELETE_TYPE_SCHEDULES:
-                $this->deleteSchedules($dbForConsole, $getProjectDB, $datetime, $document);
+                $this->deleteSchedules($dbForConsole, $getProjectDB, $datetime);
                 break;
             case DELETE_TYPE_TOPIC:
                 $this->deleteTopic($project, $getProjectDB, $document);
                 break;
             case DELETE_TYPE_TARGET:
-                $this->deleteTarget($project, $getProjectDB, $document);
+                $this->deleteTargetSubscribers($project, $getProjectDB, $document);
+                break;
+            case DELETE_TYPE_EXPIRED_TARGETS:
+                $this->deleteExpiredTargets($project, $getProjectDB);
                 break;
             default:
                 throw new \Exception('No delete operation for type: ' . \strval($type));
-                break;
         }
     }
 
@@ -183,13 +181,12 @@ class Deletes extends Action
      * @throws Structure
      * @throws DatabaseException
      */
-    private function deleteSchedules(Database $dbForConsole, callable $getProjectDB, string $datetime, ?Document $document = null): void
+    private function deleteSchedules(Database $dbForConsole, callable $getProjectDB, string $datetime): void
     {
         $this->listByGroup(
             'schedules',
             [
                 Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
-                Query::equal('resourceType', [$document->getAttribute('resourceType')]),
                 Query::lessThanEqual('resourceUpdatedAt', $datetime),
                 Query::equal('active', [false]),
             ],
@@ -230,17 +227,20 @@ class Deletes extends Action
      * @param Document $topic
      * @throws Exception
      */
-    protected function deleteTopic(Document $project, callable $getProjectDB, Document $topic)
+    private function deleteTopic(Document $project, callable $getProjectDB, Document $topic)
     {
         if ($topic->isEmpty()) {
             Console::error('Failed to delete subscribers. Topic not found');
             return;
         }
-        $dbForProject = $getProjectDB($project);
 
-        $this->deleteByGroup('subscribers', [
-            Query::equal('topicInternalId', [$topic->getInternalId()])
-        ], $dbForProject);
+        $this->deleteByGroup(
+            'subscribers',
+            [
+                Query::equal('topicInternalId', [$topic->getInternalId()])
+            ],
+            $getProjectDB($project)
+        );
     }
 
     /**
@@ -249,7 +249,7 @@ class Deletes extends Action
      * @param Document $target
      * @throws Exception
      */
-    protected function deleteTarget(Document $project, callable $getProjectDB, Document $target)
+    private function deleteTargetSubscribers(Document $project, callable $getProjectDB, Document $target)
     {
         /** @var Database */
         $dbForProject = $getProjectDB($project);
@@ -272,6 +272,27 @@ class Deletes extends Action
         );
     }
 
+    /**
+     * @param Document $project
+     * @param callable $getProjectDB
+     * @param Document $target
+     * @return void
+     * @throws Exception
+     */
+    private function deleteExpiredTargets(Document $project, callable $getProjectDB)
+    {
+        $this->deleteByGroup(
+            'targets',
+            [
+                Query::equal('expired', [true])
+            ],
+            $getProjectDB($project),
+            function (Document $target) use ($getProjectDB, $project) {
+                $this->deleteTargetSubscribers($project, $getProjectDB, $target);
+            }
+        );
+    }
+
     /**
      * @param Document $project
      * @param callable $getProjectDB
@@ -613,14 +634,14 @@ class Deletes extends Action
         ], $dbForProject);
 
         // Delete targets
-        $this->listByGroup(
+        $this->deleteByGroup(
             'targets',
             [
                 Query::equal('userInternalId', [$userInternalId])
             ],
             $dbForProject,
             function (Document $target) use ($getProjectDB, $project) {
-                $this->deleteTarget($project, $getProjectDB, $target);
+                $this->deleteTargetSubscribers($project, $getProjectDB, $target);
             }
         );
     }

src/Appwrite/Platform/Workers/Messaging.php

@@ -248,7 +248,11 @@ class Messaging extends Action
                                     ]);
 
                                     if ($target instanceof Document && !$target->isEmpty()) {
-                                        $dbForProject->deleteDocument('targets', $target->getId());
+                                        $dbForProject->updateDocument(
+                                            'targets',
+                                            $target->getId(),
+                                            $target->setAttribute('expired', true)
+                                        );
                                     }
                                 }
                             }