CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-03 05:31:38 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'refactor-usage-sn' of https://github.com/appwrite/appwrite into refactor-usage-sn

Browse source
This commit is contained in:
Torsten Dittmann 2023-12-12 18:48:26 +01:00
parent 063db1e2bf fc504e3a68
commit cf87cfab3a
4 changed files with 44 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
app/worker.php
View file

@ -21,6 +21,7 @@ use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Service;
@ -102,6 +103,24 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
};
}, ['pools', 'dbForConsole', 'cache']);
Server::setResource('getProjectAbuseRetention', function () {
return function (Document $project) {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
};
});
Server::setResource('getProjectAuditRetention', function () {
return function (Document $project) {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
};
});
Server::setResource('getProjectExecutionRetention', function () {
return function (Document $project) {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
};
});
Server::setResource('cache', function (Registry $register) {
$pools = $register->get('pools');
$list = Config::getParam('pools-cache', []);

20
src/Appwrite/Platform/Tasks/Maintenance.php
View file

@ -36,20 +36,17 @@ class Maintenance extends Action
// # of days in seconds (1 day = 86400s)
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
$executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
$auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600');
$abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
$usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
$time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
$this->notifyDeleteExecutionLogs($executionLogsRetention, $queueForDeletes);
$this->notifyDeleteAbuseLogs($abuseLogsRetention, $queueForDeletes);
$this->notifyDeleteAuditLogs($auditLogRetention, $queueForDeletes);
$this->notifyDeleteExecutionLogs($queueForDeletes);
$this->notifyDeleteAbuseLogs($queueForDeletes);
$this->notifyDeleteAuditLogs($queueForDeletes);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->notifyDeleteConnections($queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
@ -59,27 +56,24 @@ class Maintenance extends Action
}, $interval);
}
private function notifyDeleteExecutionLogs(int $interval, Delete $queueForDeletes): void
private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
{
($queueForDeletes)
->setType(DELETE_TYPE_EXECUTIONS)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
private function notifyDeleteAbuseLogs(int $interval, Delete $queueForDeletes): void
private function notifyDeleteAbuseLogs(Delete $queueForDeletes): void
{
($queueForDeletes)
->setType(DELETE_TYPE_ABUSE)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
private function notifyDeleteAuditLogs(int $interval, Delete $queueForDeletes): void
private function notifyDeleteAuditLogs(Delete $queueForDeletes): void
{
($queueForDeletes)
->setType(DELETE_TYPE_AUDIT)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}

36
src/Appwrite/Platform/Workers/Deletes.php
View file

@ -45,14 +45,17 @@ class Deletes extends Action
->inject('getFunctionsDevice')
->inject('getBuildsDevice')
->inject('getCacheDevice')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice));
->inject('getProjectAbuseRetention')
->inject('getProjectExecutionRetention')
->inject('getProjectAuditRetention')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $getProjectAbuseRetention, $getProjectExecutionRetention, $getProjectAuditRetention));
}
/**
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice): void
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention): void
{
$payload = $message->getPayload() ?? [];
@ -114,12 +117,12 @@ class Deletes extends Action
break;
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($dbForConsole, $getProjectDB, $datetime);
$this->deleteExecutionLogs($dbForConsole, $getProjectDB, $getProjectExecutionRetention);
break;
case DELETE_TYPE_AUDIT:
if (!empty($datetime)) {
$this->deleteAuditLogs($dbForConsole, $getProjectDB, $datetime);
$this->deleteAuditLogs($dbForConsole, $getProjectDB, $getProjectAuditRetention);
}
if (!$document->isEmpty()) {
@ -127,7 +130,7 @@ class Deletes extends Action
}
break;
case DELETE_TYPE_ABUSE:
$this->deleteAbuseLogs($dbForConsole, $getProjectDB, $datetime);
$this->deleteAbuseLogs($dbForConsole, $getProjectDB, $getProjectAbuseRetention);
break;
case DELETE_TYPE_REALTIME:
@ -543,10 +546,11 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, string $datetime): void
private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, callable $getProjectExecutionRetention): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectExecutionRetention) {
$dbForProject = $getProjectDB($project);
$datetime = $getProjectExecutionRetention($project);
// Delete Executions
$this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
@ -597,15 +601,12 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAbuseRetention): void
{
if (empty($datetime)) {
throw new Exception('Failed to delete audit logs. No datetime provided');
}
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAbuseRetention) {
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$datetime = $getProjectAbuseRetention($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($datetime);
@ -622,15 +623,12 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAuditRetention): void
{
if (empty($datetime)) {
throw new Exception('Failed to delete audit logs. No datetime provided');
}
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAuditRetention) {
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$datetime = $getProjectAuditRetention($project);
$audit = new Audit($dbForProject);
$status = $audit->cleanup($datetime);
if (!$status) {

2
src/Appwrite/Utopia/Database/Validator/Queries/Base.php
View file

@ -25,7 +25,7 @@ class Base extends Queries
public function __construct(string $collection, array $allowedAttributes)
{
$config = Config::getParam('collections', []);
$collections = array_merge($config['console'], $config['projects'], $config['buckets'], $config['databases']);
$collections = array_merge($config['projects'], $config['buckets'], $config['databases'], $config['console']);
$collection = $collections[$collection];
// array for constant lookup time
$allowedAttributesLookup = [];