Merge branch 'feat-db-pools' into fix-init-retry

.env

@@ -82,6 +82,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
 _APP_MAINTENANCE_RETENTION_ABUSE=86400
 _APP_MAINTENANCE_RETENTION_AUDIT=1209600
+_APP_MAINTENANCE_RETENTION_SCHEDULES= 86400
 _APP_USAGE_TIMESERIES_INTERVAL=2
 _APP_USAGE_DATABASE_INTERVAL=15
 _APP_USAGE_STATS=enabled

app/config/variables.php

@@ -920,6 +920,15 @@ return [
                 'required' => false,
                 'question' => '',
                 'filter' => ''
+            ],
+            [
+                'name' => '_APP_MAINTENANCE_RETENTION_SCHEDULES',
+                'description' => 'Schedules deletion interval ( in seconds ) ',
+                'introduction' => 'TBD',
+                'default' => '86400',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
             ]
         ],
     ],

app/controllers/api/functions.php

@@ -91,7 +91,7 @@ App::post('/v1/functions')
 
         $schedule = Authorization::skip(
             fn() => $dbForConsole->createDocument('schedules', new Document([
-                'region' => App::getEnv('_APP_REGION'), // Todo replace with projects region
+                'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
                 'resourceType' => 'function',
                 'resourceId' => $function->getId(),
                 'resourceUpdatedAt' => DateTime::now(),

app/init.php

@@ -156,6 +156,7 @@ const DELETE_TYPE_BUCKETS = 'buckets';
 const DELETE_TYPE_SESSIONS = 'sessions';
 const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp';
 const DELETE_TYPE_CACHE_BY_RESOURCE  = 'cacheByResource';
+const DELETE_TYPE_SCHEDULES = 'schedules';
 // Compression type
 const COMPRESSION_TYPE_NONE = 'none';
 const COMPRESSION_TYPE_GZIP = 'gzip';

app/views/install/compose.phtml

@@ -158,6 +158,7 @@ services:
       - _APP_MAINTENANCE_RETENTION_CACHE
       - _APP_MAINTENANCE_RETENTION_ABUSE
       - _APP_MAINTENANCE_RETENTION_AUDIT
+      - _APP_MAINTENANCE_RETENTION_SCHEDULES
       - _APP_SMS_PROVIDER
       - _APP_SMS_FROM
 

app/workers/builds.php

@@ -123,10 +123,12 @@ class BuildsV1 extends Worker
         /** Trigger Functions */
         $pools = $register->get('pools');
         $connection = $pools->get('queue')->pop();
+
         $functions = new Func($connection->getResource());
         $functions
             ->from($deploymentUpdate)
             ->trigger();
+
         $connection->reclaim();
 
         /** Trigger Realtime */

app/workers/deletes.php

@@ -114,6 +114,9 @@ class DeletesV1 extends Worker
             case DELETE_TYPE_CACHE_BY_TIMESTAMP:
                 $this->deleteCacheByDate();
                 break;
+            case DELETE_TYPE_SCHEDULES:
+                $this->deleteSchedules($this->args['datetime']);
+                break;
             default:
                 Console::error('No delete operation for type: ' . $type);
                 break;
@@ -124,6 +127,40 @@ class DeletesV1 extends Worker
     {
     }
 
+    /**
+     * @throws Exception
+     */
+    protected function deleteSchedules(string $datetime): void
+    {
+
+        $this->deleteByGroup(
+            'schedules',
+            [
+                Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
+                Query::equal('resourceType', ['function']),
+                Query::lessThanEqual('resourceUpdatedAt', $datetime),
+                Query::equal('active', [false]),
+            ],
+            $this->getConsoleDB(),
+            function (Document $document) {
+                Console::info('Querying schedule for function ' . $document->getAttribute('resourceId'));
+                $project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId'));
+
+                if ($project->isEmpty()) {
+                    Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId'));
+                    return;
+                }
+
+                $function = $this->getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId'));
+
+                if ($function->isEmpty()) {
+                    $this->getConsoleDB()->deleteDocument('schedules', $document->getId());
+                    Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId'));
+                }
+            }
+        );
+    }
+
     /**
      * @param string $resource
      */

composer.lock

@@ -5311,5 +5311,5 @@
     "platform-overrides": {
         "php": "8.0"
     },
-    "plugin-api-version": "2.1.0"
+    "plugin-api-version": "2.3.0"
 }

docker-compose.yml

@@ -601,6 +601,7 @@ services:
       - _APP_MAINTENANCE_RETENTION_CACHE
       - _APP_MAINTENANCE_RETENTION_ABUSE
       - _APP_MAINTENANCE_RETENTION_AUDIT
+      - _APP_MAINTENANCE_RETENTION_SCHEDULES
 
   appwrite-volume-sync:
     entrypoint: volume-sync

src/Appwrite/Event/Event.php

@@ -137,7 +137,7 @@ class Event
     }
 
     /**
-     * Get project for this event.
+     * Get user responsible for triggering this event.
      *
      * @return ?Document
      */

src/Appwrite/Platform/Tasks/Maintenance.php

@@ -118,6 +118,15 @@ class Maintenance extends Action
                 ->trigger();
         }
 
+        function notifyDeleteSchedules($interval)
+        {
+
+            (new Delete())
+                ->setType(DELETE_TYPE_SCHEDULES)
+                ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
+                ->trigger();
+        }
+
         // # of days in seconds (1 day = 86400s)
         $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
         $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
@@ -126,8 +135,9 @@ class Maintenance extends Action
         $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours
         $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days
         $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
+        $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
 
-        Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention, $dbForConsole) {
+        Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention, $schedulesDeletionRetention, $dbForConsole) {
             $time = DateTime::now();
 
             Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
@@ -139,8 +149,7 @@ class Maintenance extends Action
             notifyDeleteExpiredSessions();
             renewCertificates($dbForConsole);
             notifyDeleteCache($cacheRetention);
-
+            notifyDeleteSchedules($schedulesDeletionRetention);
-            // TODO: @Meldiron Every probably 24h, look for schedules with active=false, that doesnt have function anymore. Dlete such schedule
         }, $interval);
     }
 }

src/Appwrite/Platform/Tasks/Schedule.php

@@ -81,7 +81,7 @@ class Schedule extends Action
                 $paginationQueries[] = Query::cursorAfter($latestDocument);
             }
             $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
-                Query::equal('region', [App::getEnv('_APP_REGION')]),
+                Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
                 Query::equal('resourceType', ['function']),
                 Query::equal('active', [true]),
             ]));
@@ -123,7 +123,7 @@ class Schedule extends Action
                             $paginationQueries[] =  Query::cursorAfter($latestDocument);
                         }
                         $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
-                            Query::equal('region', [App::getEnv('_APP_REGION')]),
+                            Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
                             Query::equal('resourceType', ['function']),
                             Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate),
                         ]));