diff --git a/.env b/.env index 390e838d8..b94fa24f2 100644 --- a/.env +++ b/.env @@ -90,3 +90,4 @@ _APP_LOGGING_CONFIG= _APP_REGION=default _APP_DOCKER_HUB_USERNAME= _APP_DOCKER_HUB_PASSWORD= +_APP_MAINTENANCE_RETENTION_SCHEDULES=86400 diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 86981b007..520d29ccd 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -91,7 +91,7 @@ App::post('/v1/functions') $schedule = Authorization::skip( fn() => $dbForConsole->createDocument('schedules', new Document([ - 'region' => App::getEnv('_APP_REGION'), // Todo replace with projects region + 'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region 'resourceType' => 'function', 'resourceId' => $function->getId(), 'resourceUpdatedAt' => DateTime::now(), diff --git a/app/init.php b/app/init.php index af3b05965..22d67031f 100644 --- a/app/init.php +++ b/app/init.php @@ -156,6 +156,7 @@ const DELETE_TYPE_BUCKETS = 'buckets'; const DELETE_TYPE_SESSIONS = 'sessions'; const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp'; const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource'; +const DELETE_TYPE_SCHEDULES = 'schedules'; // Compression type const COMPRESSION_TYPE_NONE = 'none'; const COMPRESSION_TYPE_GZIP = 'gzip'; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 3e0df8ce3..93e64d4fc 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -114,6 +114,9 @@ class DeletesV1 extends Worker case DELETE_TYPE_CACHE_BY_TIMESTAMP: $this->deleteCacheByDate(); break; + case DELETE_TYPE_SCHEDULES: + $this->deleteSchedules($this->args['datetime']); + break; default: Console::error('No delete operation for type: ' . $type); break; @@ -124,6 +127,47 @@ class DeletesV1 extends Worker { } + /** + * @throws Exception + */ + protected function deleteSchedules(string $datetime): void + { + + $sum = $limit = 50; + $offset = 0; + /** @var Document[] $functions */ + while ($sum >= $limit) { + $schedules = $this->getConsoleDB()->find('schedules', [ + Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), + Query::equal('resourceType', ['function']), + Query::lessThanEqual('resourceUpdatedAt', $datetime), + Query::equal('active', [false]), + Query::limit($limit), + Query::offset($offset), + ]); + + $sum = \count($schedules); + $offset = $offset + $limit; + + foreach ($schedules as $schedule) { + Console::info('Querying schedule for function ' . $schedule->getAttribute('resourceId')); + $project = $this->getConsoleDB()->getDocument('projects', $schedule->getAttribute('projectId')); + + if ($project->isEmpty()) { + Console::success('Deleting schedule for function ' . $schedule->getAttribute('resourceId')); + continue; + } + + $function = $this->getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId')); + + if ($function->isEmpty()) { + $this->getConsoleDB()->deleteDocument('schedules', $schedule->getId()); + Console::success('Deleting schedule for function ' . $schedule->getAttribute('resourceId')); + } + } + } + } + /** * @param string $resource */ diff --git a/docker-compose.yml b/docker-compose.yml index 22ed89bed..7ba310060 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -592,6 +592,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_SCHEDULES appwrite-volume-sync: entrypoint: volume-sync diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 307f50261..5b41bae4b 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -118,6 +118,15 @@ class Maintenance extends Action ->trigger(); } + function notifyDeleteSchedules($interval) + { + + (new Delete()) + ->setType(DELETE_TYPE_SCHEDULES) + ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) + ->trigger(); + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); @@ -126,8 +135,9 @@ class Maintenance extends Action $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days + $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '360'); //Todo One day gap??? - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention, $dbForConsole) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention, $schedulesDeletionRetention, $dbForConsole) { $time = DateTime::now(); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); @@ -139,8 +149,7 @@ class Maintenance extends Action notifyDeleteExpiredSessions(); renewCertificates($dbForConsole); notifyDeleteCache($cacheRetention); - - // TODO: @Meldiron Every probably 24h, look for schedules with active=false, that doesnt have function anymore. Dlete such schedule + notifyDeleteSchedules($schedulesDeletionRetention); }, $interval); } } diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index dc86f9293..24654a47d 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -81,7 +81,7 @@ class Schedule extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [App::getEnv('_APP_REGION')]), + Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', ['function']), Query::equal('active', [true]), ])); @@ -123,7 +123,7 @@ class Schedule extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } $results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [App::getEnv('_APP_REGION')]), + Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', ['function']), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ]));