From 104b8bb36654cb60c83d3fd131a89f2504cb9ca0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 12 Dec 2023 08:07:24 +0000 Subject: [PATCH 1/3] provide retention time as queue server resource --- app/worker.php | 19 ++++++++++++ src/Appwrite/Platform/Workers/Deletes.php | 36 +++++++++++------------ 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/app/worker.php b/app/worker.php index 59119c692f..539b756869 100644 --- a/app/worker.php +++ b/app/worker.php @@ -21,6 +21,7 @@ use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\Platform\Service; @@ -102,6 +103,24 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso }; }, ['pools', 'dbForConsole', 'cache']); +Server::setResource('getProjectAbuseRetention', function () { + return function (Document $project) { + return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400)); + }; +}); + +Server::setResource('getProjectAuditRetention', function () { + return function (Document $project) { + return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); + }; +}); + +Server::setResource('getProjectExecutionRetention', function () { + return function (Document $project) { + return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); + }; +}); + Server::setResource('cache', function (Registry $register) { $pools = $register->get('pools'); $list = Config::getParam('pools-cache', []); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 38ff56ce44..4f23db87d1 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -45,14 +45,17 @@ class Deletes extends Action ->inject('getFunctionsDevice') ->inject('getBuildsDevice') ->inject('getCacheDevice') - ->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice)); + ->inject('getProjectAbuseRetention') + ->inject('getProjectExecutionRetention') + ->inject('getProjectAuditRetention') + ->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $getProjectAbuseRetention, $getProjectExecutionRetention, $getProjectAuditRetention)); } /** * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention): void { $payload = $message->getPayload() ?? []; @@ -114,12 +117,12 @@ class Deletes extends Action break; case DELETE_TYPE_EXECUTIONS: - $this->deleteExecutionLogs($dbForConsole, $getProjectDB, $datetime); + $this->deleteExecutionLogs($dbForConsole, $getProjectDB, $getProjectExecutionRetention); break; case DELETE_TYPE_AUDIT: if (!empty($datetime)) { - $this->deleteAuditLogs($dbForConsole, $getProjectDB, $datetime); + $this->deleteAuditLogs($dbForConsole, $getProjectDB, $getProjectAuditRetention); } if (!$document->isEmpty()) { @@ -127,7 +130,7 @@ class Deletes extends Action } break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($dbForConsole, $getProjectDB, $datetime); + $this->deleteAbuseLogs($dbForConsole, $getProjectDB, $getProjectAbuseRetention); break; case DELETE_TYPE_REALTIME: @@ -543,10 +546,11 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, string $datetime): void + private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, callable $getProjectExecutionRetention): void { - $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) { + $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectExecutionRetention) { $dbForProject = $getProjectDB($project); + $datetime = $getProjectExecutionRetention($project); // Delete Executions $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime) @@ -597,15 +601,12 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void + private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAbuseRetention): void { - if (empty($datetime)) { - throw new Exception('Failed to delete audit logs. No datetime provided'); - } - - $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) { + $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAbuseRetention) { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); + $datetime = $getProjectAbuseRetention($project); $timeLimit = new TimeLimit("", 0, 1, $dbForProject); $abuse = new Abuse($timeLimit); $status = $abuse->cleanup($datetime); @@ -622,15 +623,12 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void + private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAuditRetention): void { - if (empty($datetime)) { - throw new Exception('Failed to delete audit logs. No datetime provided'); - } - - $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) { + $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAuditRetention) { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); + $datetime = $getProjectAuditRetention($project); $audit = new Audit($dbForProject); $status = $audit->cleanup($datetime); if (!$status) { From 9ac57e2e7a6eb8e535deeb22054ced55879cb4ce Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 12 Dec 2023 08:11:01 +0000 Subject: [PATCH 2/3] remove unused retention --- src/Appwrite/Platform/Tasks/Maintenance.php | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 82a62ffed1..ecc2b26fd1 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -36,20 +36,17 @@ class Maintenance extends Action // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); - $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); - $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); - $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); $usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) { + Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) { $time = DateTime::now(); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); - $this->notifyDeleteExecutionLogs($executionLogsRetention, $queueForDeletes); - $this->notifyDeleteAbuseLogs($abuseLogsRetention, $queueForDeletes); - $this->notifyDeleteAuditLogs($auditLogRetention, $queueForDeletes); + $this->notifyDeleteExecutionLogs($queueForDeletes); + $this->notifyDeleteAbuseLogs($queueForDeletes); + $this->notifyDeleteAuditLogs($queueForDeletes); $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes); $this->notifyDeleteConnections($queueForDeletes); $this->notifyDeleteExpiredSessions($queueForDeletes); @@ -59,27 +56,24 @@ class Maintenance extends Action }, $interval); } - private function notifyDeleteExecutionLogs(int $interval, Delete $queueForDeletes): void + private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void { ($queueForDeletes) ->setType(DELETE_TYPE_EXECUTIONS) - ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) ->trigger(); } - private function notifyDeleteAbuseLogs(int $interval, Delete $queueForDeletes): void + private function notifyDeleteAbuseLogs(Delete $queueForDeletes): void { ($queueForDeletes) ->setType(DELETE_TYPE_ABUSE) - ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) ->trigger(); } - private function notifyDeleteAuditLogs(int $interval, Delete $queueForDeletes): void + private function notifyDeleteAuditLogs(Delete $queueForDeletes): void { ($queueForDeletes) ->setType(DELETE_TYPE_AUDIT) - ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) ->trigger(); } From fc504e3a68ece12901cbf63ad8cf456568054968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 12 Dec 2023 12:27:04 +0100 Subject: [PATCH 3/3] Change collections priority for pricing override --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index 3eea7b7b7e..d06111f60c 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -25,7 +25,7 @@ class Base extends Queries public function __construct(string $collection, array $allowedAttributes) { $config = Config::getParam('collections', []); - $collections = array_merge($config['console'], $config['projects'], $config['buckets'], $config['databases']); + $collections = array_merge($config['projects'], $config['buckets'], $config['databases'], $config['console']); $collection = $collections[$collection]; // array for constant lookup time $allowedAttributesLookup = [];