CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-14 08:44:49 +12:00
Code Issues Projects Releases Wiki Activity

Deleted functions

Browse source 
schedules clean up
This commit is contained in:
shimon 2022-11-16 14:51:43 +02:00
parent 390f407eb5
commit c67852a7fb
7 changed files with 62 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.env
View file

@ -90,3 +90,4 @@ _APP_LOGGING_CONFIG=
_APP_REGION=default
_APP_DOCKER_HUB_USERNAME=
_APP_DOCKER_HUB_PASSWORD=
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400

2
app/controllers/api/functions.php
View file

@ -91,7 +91,7 @@ App::post('/v1/functions')
$schedule = Authorization::skip(
fn() => $dbForConsole->createDocument('schedules', new Document([
'region' => App::getEnv('_APP_REGION'), // Todo replace with projects region
'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
'resourceType' => 'function',
'resourceId' => $function->getId(),
'resourceUpdatedAt' => DateTime::now(),

1
app/init.php
View file

@ -156,6 +156,7 @@ const DELETE_TYPE_BUCKETS = 'buckets';
const DELETE_TYPE_SESSIONS = 'sessions';
const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp';
const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource';
const DELETE_TYPE_SCHEDULES = 'schedules';
// Compression type
const COMPRESSION_TYPE_NONE = 'none';
const COMPRESSION_TYPE_GZIP = 'gzip';

44
app/workers/deletes.php
View file

@ -114,6 +114,9 @@ class DeletesV1 extends Worker
case DELETE_TYPE_CACHE_BY_TIMESTAMP:
$this->deleteCacheByDate();
break;
case DELETE_TYPE_SCHEDULES:
$this->deleteSchedules($this->args['datetime']);
break;
default:
Console::error('No delete operation for type: ' . $type);
break;
@ -124,6 +127,47 @@ class DeletesV1 extends Worker
{
}
/**
* @throws Exception
*/
protected function deleteSchedules(string $datetime): void
{
$sum = $limit = 50;
$offset = 0;
/** @var Document[] $functions */
while ($sum >= $limit) {
$schedules = $this->getConsoleDB()->find('schedules', [
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::lessThanEqual('resourceUpdatedAt', $datetime),
Query::equal('active', [false]),
Query::limit($limit),
Query::offset($offset),
]);
$sum = \count($schedules);
$offset = $offset + $limit;
foreach ($schedules as $schedule) {
Console::info('Querying schedule for function ' . $schedule->getAttribute('resourceId'));
$project = $this->getConsoleDB()->getDocument('projects', $schedule->getAttribute('projectId'));
if ($project->isEmpty()) {
Console::success('Deleting schedule for function ' . $schedule->getAttribute('resourceId'));
continue;
}
$function = $this->getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId'));
if ($function->isEmpty()) {
$this->getConsoleDB()->deleteDocument('schedules', $schedule->getId());
Console::success('Deleting schedule for function ' . $schedule->getAttribute('resourceId'));
}
}
}
}
/**
* @param string $resource
*/

1
docker-compose.yml
View file

@ -592,6 +592,7 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-volume-sync:
entrypoint: volume-sync

15
src/Appwrite/Platform/Tasks/Maintenance.php
View file

@ -118,6 +118,15 @@ class Maintenance extends Action
->trigger();
}
function notifyDeleteSchedules($interval)
{
(new Delete())
->setType(DELETE_TYPE_SCHEDULES)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
// # of days in seconds (1 day = 86400s)
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
$executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
@ -126,8 +135,9 @@ class Maintenance extends Action
$usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours
$usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '360'); //Todo One day gap???
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention, $dbForConsole) {
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention, $schedulesDeletionRetention, $dbForConsole) {
$time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
@ -139,8 +149,7 @@ class Maintenance extends Action
notifyDeleteExpiredSessions();
renewCertificates($dbForConsole);
notifyDeleteCache($cacheRetention);
// TODO: @Meldiron Every probably 24h, look for schedules with active=false, that doesnt have function anymore. Dlete such schedule
notifyDeleteSchedules($schedulesDeletionRetention);
}, $interval);
}
}

4
src/Appwrite/Platform/Tasks/Schedule.php
View file

@ -81,7 +81,7 @@ class Schedule extends Action
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [App::getEnv('_APP_REGION')]),
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::equal('active', [true]),
]));
@ -123,7 +123,7 @@ class Schedule extends Action
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [App::getEnv('_APP_REGION')]),
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate),
]));