Merge pull request #4860 from appwrite/fix-deletes-worker-tasks

Fix: expired session, schedule deletes

.env

@@ -24,7 +24,7 @@ _APP_DB_SCHEMA=appwrite
 _APP_DB_USER=user
 _APP_DB_PASS=password
 _APP_DB_ROOT_PASS=rootsecretpassword
-_APP_CONNECTIONS_MAX=251
+_APP_CONNECTIONS_MAX=3100
 _APP_POOL_CLIENTS=14
 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
 _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite

app/workers/deletes.php

@@ -134,8 +134,7 @@ class DeletesV1 extends Worker
      */
     protected function deleteSchedules(string $datetime): void
     {
-
-        $this->deleteByGroup(
+        $this->listByGroup(
             'schedules',
             [
                 Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
@@ -145,7 +144,6 @@ class DeletesV1 extends Worker
             ],
             $this->getConsoleDB(),
             function (Document $document) {
-                Console::info('Querying schedule for function ' . $document->getAttribute('resourceId'));
                 $project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId'));
 
                 if ($project->isEmpty()) {
@@ -358,8 +356,15 @@ class DeletesV1 extends Worker
 
     protected function deleteExpiredSessions(): void
     {
-        $this->deleteForProjectIds(function (Document $project) use ($datetime) {
+        $consoleDB = $this->getConsoleDB();
+
+        $this->deleteForProjectIds(function (Document $project) use ($consoleDB) {
             $dbForProject = $this->getProjectDB($project);
+
+            $project = $consoleDB->getDocument('projects', $project->getId());
+            $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
+            $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
+
             // Delete Sessions
             $this->deleteByGroup('sessions', [
                 Query::lessThan('$createdAt', $expired)
@@ -569,6 +574,7 @@ class DeletesV1 extends Worker
      */
     protected function deleteForProjectIds(callable $callback): void
     {
+        // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
         $count = 0;
         $chunk = 0;
         $limit = 50;
@@ -632,6 +638,43 @@ class DeletesV1 extends Worker
         Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
     }
 
+    /**
+     * @param string $collection collectionID
+     * @param Query[] $queries
+     * @param Database $database
+     * @param callable $callback
+     */
+    protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
+    {
+        $count = 0;
+        $chunk = 0;
+        $limit = 50;
+        $results = [];
+        $sum = $limit;
+
+        $executionStart = \microtime(true);
+
+        while ($sum === $limit) {
+            $chunk++;
+
+            $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
+
+            $sum = count($results);
+
+            foreach ($results as $document) {
+                if (is_callable($callback)) {
+                    $callback($document);
+                }
+
+                $count++;
+            }
+        }
+
+        $executionEnd = \microtime(true);
+
+        Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
+    }
+
     /**
      * @param Document $document certificates document
      */

bin/patch-create-missing-schedules

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php patch-create-missing-schedules $@

docker-compose.yml

@@ -33,15 +33,6 @@ services:
       - 8080:80
       - 443:443
       - 9500:8080
-    ulimits:
-      nofile:
-        soft: 655350
-        hard: 655350
-    sysctls:
-      - net.core.somaxconn=1024
-      - net.ipv4.tcp_rmem=1024 4096 16384
-      - net.ipv4.tcp_wmem=1024 4096 16384
-      - net.ipv4.ip_local_port_range=1025 65535
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - appwrite-config:/storage/config:ro

src/Appwrite/Platform/Services/Tasks.php

@@ -8,6 +8,7 @@ use Appwrite\Platform\Tasks\Install;
 use Appwrite\Platform\Tasks\Maintenance;
 use Appwrite\Platform\Tasks\Migrate;
 use Appwrite\Platform\Tasks\Schedule;
+use Appwrite\Platform\Tasks\PatchCreateMissingSchedules;
 use Appwrite\Platform\Tasks\SDKs;
 use Appwrite\Platform\Tasks\Specs;
 use Appwrite\Platform\Tasks\SSL;
@@ -29,6 +30,7 @@ class Tasks extends Service
             ->addAction(Doctor::getName(), new Doctor())
             ->addAction(Install::getName(), new Install())
             ->addAction(Maintenance::getName(), new Maintenance())
+            ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules())
             ->addAction(Schedule::getName(), new Schedule())
             ->addAction(Migrate::getName(), new Migrate())
             ->addAction(SDKs::getName(), new SDKs())

src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php

@@ -0,0 +1,97 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\Platform\Action;
+use Utopia\CLI\Console;
+use Utopia\Database\DateTime;
+use Utopia\Database\Document;
+use Utopia\Database\Query;
+use Utopia\Database\Database;
+use Utopia\Database\ID;
+use Utopia\Database\Validator\Authorization;
+
+class PatchCreateMissingSchedules extends Action
+{
+    public static function getName(): string
+    {
+        return 'patch-create-missing-schedules';
+    }
+
+    public function __construct()
+    {
+        $this
+            ->desc('Ensure every function has a schedule')
+            ->inject('dbForConsole')
+            ->inject('getProjectDB')
+            ->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB));
+    }
+
+    /**
+     * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
+     */
+    public function action(Database $dbForConsole, callable $getProjectDB): void
+    {
+        Authorization::disable();
+        Authorization::setDefaultStatus(false);
+
+        Console::title('PatchCreateMissingSchedules V1');
+        Console::success(APP_NAME . ' PatchCreateMissingSchedules v1 has started');
+
+        $limit = 100;
+        $projectCursor = null;
+        while (true) {
+            $projectsQueries = [Query::limit($limit)];
+            if ($projectCursor !== null) {
+                $projectsQueries[] = Query::cursorAfter($projectCursor);
+            }
+            $projects = $dbForConsole->find('projects', $projectsQueries);
+
+            if (count($projects) === 0) {
+                break;
+            }
+
+            foreach ($projects as $project) {
+                Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")");
+                $dbForProject = $getProjectDB($project);
+                $functionCursor = null;
+
+                while (true) {
+                    $functionsQueries = [Query::limit($limit)];
+                    if ($functionCursor !== null) {
+                        $functionsQueries[] = Query::cursorAfter($functionCursor);
+                    }
+                    $functions = $dbForProject->find('functions', $functionsQueries);
+                    if (count($functions) === 0) {
+                        break;
+                    }
+
+                    foreach ($functions as $function) {
+                        $scheduleId = $function->getAttribute('scheduleId');
+                        $schedule = $dbForConsole->getDocument('schedules', $scheduleId);
+
+                        if ($schedule->isEmpty()) {
+                            $functionId = $function->getId();
+                            $schedule = $dbForConsole->createDocument('schedules', new Document([
+                                '$id' => ID::custom($scheduleId),
+                                'region' => $project->getAttribute('region', 'default'),
+                                'resourceType' => 'function',
+                                'resourceId' => $functionId,
+                                'resourceUpdatedAt' => DateTime::now(),
+                                'projectId' => $project->getId(),
+                                'schedule'  => $function->getAttribute('schedule'),
+                                'active' => !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')),
+                            ]));
+
+                            Console::success('Recreated schedule for function ' . $functionId);
+                        }
+                    }
+
+                    $functionCursor = $functions[array_key_last($functions)];
+                }
+            }
+
+            $projectCursor = $projects[array_key_last($projects)];
+        }
+    }
+}