Merge branch 'refactor-usage-sn' of https://github.com/appwrite/appwrite into refactor-usage-sn
This commit is contained in:
commit
cf87cfab3a
4 changed files with 44 additions and 33 deletions
|
@ -21,6 +21,7 @@ use Utopia\Cache\Cache;
|
||||||
use Utopia\CLI\Console;
|
use Utopia\CLI\Console;
|
||||||
use Utopia\Config\Config;
|
use Utopia\Config\Config;
|
||||||
use Utopia\Database\Database;
|
use Utopia\Database\Database;
|
||||||
|
use Utopia\Database\DateTime;
|
||||||
use Utopia\Database\Document;
|
use Utopia\Database\Document;
|
||||||
use Utopia\Database\Validator\Authorization;
|
use Utopia\Database\Validator\Authorization;
|
||||||
use Utopia\Platform\Service;
|
use Utopia\Platform\Service;
|
||||||
|
@ -102,6 +103,24 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
|
||||||
};
|
};
|
||||||
}, ['pools', 'dbForConsole', 'cache']);
|
}, ['pools', 'dbForConsole', 'cache']);
|
||||||
|
|
||||||
|
Server::setResource('getProjectAbuseRetention', function () {
|
||||||
|
return function (Document $project) {
|
||||||
|
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
Server::setResource('getProjectAuditRetention', function () {
|
||||||
|
return function (Document $project) {
|
||||||
|
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
Server::setResource('getProjectExecutionRetention', function () {
|
||||||
|
return function (Document $project) {
|
||||||
|
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
Server::setResource('cache', function (Registry $register) {
|
Server::setResource('cache', function (Registry $register) {
|
||||||
$pools = $register->get('pools');
|
$pools = $register->get('pools');
|
||||||
$list = Config::getParam('pools-cache', []);
|
$list = Config::getParam('pools-cache', []);
|
||||||
|
|
|
@ -36,20 +36,17 @@ class Maintenance extends Action
|
||||||
|
|
||||||
// # of days in seconds (1 day = 86400s)
|
// # of days in seconds (1 day = 86400s)
|
||||||
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
|
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
|
||||||
$executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
|
|
||||||
$auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600');
|
|
||||||
$abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
|
|
||||||
$usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
|
$usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
|
||||||
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
|
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
|
||||||
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
|
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
|
||||||
|
|
||||||
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
|
Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
|
||||||
$time = DateTime::now();
|
$time = DateTime::now();
|
||||||
|
|
||||||
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
|
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
|
||||||
$this->notifyDeleteExecutionLogs($executionLogsRetention, $queueForDeletes);
|
$this->notifyDeleteExecutionLogs($queueForDeletes);
|
||||||
$this->notifyDeleteAbuseLogs($abuseLogsRetention, $queueForDeletes);
|
$this->notifyDeleteAbuseLogs($queueForDeletes);
|
||||||
$this->notifyDeleteAuditLogs($auditLogRetention, $queueForDeletes);
|
$this->notifyDeleteAuditLogs($queueForDeletes);
|
||||||
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
|
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
|
||||||
$this->notifyDeleteConnections($queueForDeletes);
|
$this->notifyDeleteConnections($queueForDeletes);
|
||||||
$this->notifyDeleteExpiredSessions($queueForDeletes);
|
$this->notifyDeleteExpiredSessions($queueForDeletes);
|
||||||
|
@ -59,27 +56,24 @@ class Maintenance extends Action
|
||||||
}, $interval);
|
}, $interval);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function notifyDeleteExecutionLogs(int $interval, Delete $queueForDeletes): void
|
private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
|
||||||
{
|
{
|
||||||
($queueForDeletes)
|
($queueForDeletes)
|
||||||
->setType(DELETE_TYPE_EXECUTIONS)
|
->setType(DELETE_TYPE_EXECUTIONS)
|
||||||
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
|
|
||||||
->trigger();
|
->trigger();
|
||||||
}
|
}
|
||||||
|
|
||||||
private function notifyDeleteAbuseLogs(int $interval, Delete $queueForDeletes): void
|
private function notifyDeleteAbuseLogs(Delete $queueForDeletes): void
|
||||||
{
|
{
|
||||||
($queueForDeletes)
|
($queueForDeletes)
|
||||||
->setType(DELETE_TYPE_ABUSE)
|
->setType(DELETE_TYPE_ABUSE)
|
||||||
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
|
|
||||||
->trigger();
|
->trigger();
|
||||||
}
|
}
|
||||||
|
|
||||||
private function notifyDeleteAuditLogs(int $interval, Delete $queueForDeletes): void
|
private function notifyDeleteAuditLogs(Delete $queueForDeletes): void
|
||||||
{
|
{
|
||||||
($queueForDeletes)
|
($queueForDeletes)
|
||||||
->setType(DELETE_TYPE_AUDIT)
|
->setType(DELETE_TYPE_AUDIT)
|
||||||
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
|
|
||||||
->trigger();
|
->trigger();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -45,14 +45,17 @@ class Deletes extends Action
|
||||||
->inject('getFunctionsDevice')
|
->inject('getFunctionsDevice')
|
||||||
->inject('getBuildsDevice')
|
->inject('getBuildsDevice')
|
||||||
->inject('getCacheDevice')
|
->inject('getCacheDevice')
|
||||||
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice));
|
->inject('getProjectAbuseRetention')
|
||||||
|
->inject('getProjectExecutionRetention')
|
||||||
|
->inject('getProjectAuditRetention')
|
||||||
|
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $getProjectAbuseRetention, $getProjectExecutionRetention, $getProjectAuditRetention));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
* @throws Throwable
|
* @throws Throwable
|
||||||
*/
|
*/
|
||||||
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice): void
|
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention): void
|
||||||
{
|
{
|
||||||
$payload = $message->getPayload() ?? [];
|
$payload = $message->getPayload() ?? [];
|
||||||
|
|
||||||
|
@ -114,12 +117,12 @@ class Deletes extends Action
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case DELETE_TYPE_EXECUTIONS:
|
case DELETE_TYPE_EXECUTIONS:
|
||||||
$this->deleteExecutionLogs($dbForConsole, $getProjectDB, $datetime);
|
$this->deleteExecutionLogs($dbForConsole, $getProjectDB, $getProjectExecutionRetention);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case DELETE_TYPE_AUDIT:
|
case DELETE_TYPE_AUDIT:
|
||||||
if (!empty($datetime)) {
|
if (!empty($datetime)) {
|
||||||
$this->deleteAuditLogs($dbForConsole, $getProjectDB, $datetime);
|
$this->deleteAuditLogs($dbForConsole, $getProjectDB, $getProjectAuditRetention);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!$document->isEmpty()) {
|
if (!$document->isEmpty()) {
|
||||||
|
@ -127,7 +130,7 @@ class Deletes extends Action
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case DELETE_TYPE_ABUSE:
|
case DELETE_TYPE_ABUSE:
|
||||||
$this->deleteAbuseLogs($dbForConsole, $getProjectDB, $datetime);
|
$this->deleteAbuseLogs($dbForConsole, $getProjectDB, $getProjectAbuseRetention);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case DELETE_TYPE_REALTIME:
|
case DELETE_TYPE_REALTIME:
|
||||||
|
@ -543,10 +546,11 @@ class Deletes extends Action
|
||||||
* @return void
|
* @return void
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, string $datetime): void
|
private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, callable $getProjectExecutionRetention): void
|
||||||
{
|
{
|
||||||
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
|
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectExecutionRetention) {
|
||||||
$dbForProject = $getProjectDB($project);
|
$dbForProject = $getProjectDB($project);
|
||||||
|
$datetime = $getProjectExecutionRetention($project);
|
||||||
// Delete Executions
|
// Delete Executions
|
||||||
$this->deleteByGroup('executions', [
|
$this->deleteByGroup('executions', [
|
||||||
Query::lessThan('$createdAt', $datetime)
|
Query::lessThan('$createdAt', $datetime)
|
||||||
|
@ -597,15 +601,12 @@ class Deletes extends Action
|
||||||
* @return void
|
* @return void
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
|
private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAbuseRetention): void
|
||||||
{
|
{
|
||||||
if (empty($datetime)) {
|
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAbuseRetention) {
|
||||||
throw new Exception('Failed to delete audit logs. No datetime provided');
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
|
|
||||||
$projectId = $project->getId();
|
$projectId = $project->getId();
|
||||||
$dbForProject = $getProjectDB($project);
|
$dbForProject = $getProjectDB($project);
|
||||||
|
$datetime = $getProjectAbuseRetention($project);
|
||||||
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
|
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
|
||||||
$abuse = new Abuse($timeLimit);
|
$abuse = new Abuse($timeLimit);
|
||||||
$status = $abuse->cleanup($datetime);
|
$status = $abuse->cleanup($datetime);
|
||||||
|
@ -622,15 +623,12 @@ class Deletes extends Action
|
||||||
* @return void
|
* @return void
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
|
private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAuditRetention): void
|
||||||
{
|
{
|
||||||
if (empty($datetime)) {
|
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAuditRetention) {
|
||||||
throw new Exception('Failed to delete audit logs. No datetime provided');
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
|
|
||||||
$projectId = $project->getId();
|
$projectId = $project->getId();
|
||||||
$dbForProject = $getProjectDB($project);
|
$dbForProject = $getProjectDB($project);
|
||||||
|
$datetime = $getProjectAuditRetention($project);
|
||||||
$audit = new Audit($dbForProject);
|
$audit = new Audit($dbForProject);
|
||||||
$status = $audit->cleanup($datetime);
|
$status = $audit->cleanup($datetime);
|
||||||
if (!$status) {
|
if (!$status) {
|
||||||
|
|
|
@ -25,7 +25,7 @@ class Base extends Queries
|
||||||
public function __construct(string $collection, array $allowedAttributes)
|
public function __construct(string $collection, array $allowedAttributes)
|
||||||
{
|
{
|
||||||
$config = Config::getParam('collections', []);
|
$config = Config::getParam('collections', []);
|
||||||
$collections = array_merge($config['console'], $config['projects'], $config['buckets'], $config['databases']);
|
$collections = array_merge($config['projects'], $config['buckets'], $config['databases'], $config['console']);
|
||||||
$collection = $collections[$collection];
|
$collection = $collections[$collection];
|
||||||
// array for constant lookup time
|
// array for constant lookup time
|
||||||
$allowedAttributesLookup = [];
|
$allowedAttributesLookup = [];
|
||||||
|
|
Loading…
Reference in a new issue