Merge branch 'refactor-usage-sn' of github.com:appwrite/appwrite into refactor-hamster-usage
Conflicts: src/Appwrite/Platform/Tasks/CalcTierStats.php
This commit is contained in:
commit
4e24e3cdc2
22 changed files with 325 additions and 280 deletions
2
.gitmodules
vendored
2
.gitmodules
vendored
|
@ -1,4 +1,4 @@
|
|||
[submodule "app/console"]
|
||||
path = app/console
|
||||
url = https://github.com/appwrite/console
|
||||
branch = billing
|
||||
branch = 3.3.4
|
|
@ -18,6 +18,63 @@ $auth = Config::getParam('auth', []);
|
|||
*/
|
||||
|
||||
$commonCollections = [
|
||||
'cache' => [
|
||||
'$collection' => Database::METADATA,
|
||||
'$id' => 'cache',
|
||||
'name' => 'Cache',
|
||||
'attributes' => [
|
||||
[
|
||||
'$id' => 'resource',
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => 255,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => 'accessedAt',
|
||||
'type' => Database::VAR_DATETIME,
|
||||
'format' => '',
|
||||
'size' => 0,
|
||||
'signed' => false,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => ['datetime'],
|
||||
],
|
||||
[
|
||||
'$id' => 'signature',
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => 255,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
],
|
||||
'indexes' => [
|
||||
[
|
||||
'$id' => '_key_accessedAt',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['accessedAt'],
|
||||
'lengths' => [],
|
||||
'orders' => [],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_resource',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['resource'],
|
||||
'lengths' => [],
|
||||
'orders' => [],
|
||||
],
|
||||
],
|
||||
],
|
||||
|
||||
'users' => [
|
||||
'$collection' => ID::custom(Database::METADATA),
|
||||
'$id' => ID::custom('users'),
|
||||
|
@ -1270,10 +1327,10 @@ $commonCollections = [
|
|||
]
|
||||
],
|
||||
|
||||
'stats' => [
|
||||
'stats_v2' => [
|
||||
'$collection' => ID::custom(Database::METADATA),
|
||||
'$id' => ID::custom('stats'),
|
||||
'name' => 'Stats',
|
||||
'$id' => ID::custom('stats_v2'),
|
||||
'name' => 'stats_v2',
|
||||
'attributes' => [
|
||||
[
|
||||
'$id' => ID::custom('metric'),
|
||||
|
@ -2872,63 +2929,6 @@ $projectCollections = array_merge([
|
|||
],
|
||||
],
|
||||
|
||||
'cache' => [
|
||||
'$collection' => Database::METADATA,
|
||||
'$id' => 'cache',
|
||||
'name' => 'Cache',
|
||||
'attributes' => [
|
||||
[
|
||||
'$id' => 'resource',
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => 255,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => 'accessedAt',
|
||||
'type' => Database::VAR_DATETIME,
|
||||
'format' => '',
|
||||
'size' => 0,
|
||||
'signed' => false,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => ['datetime'],
|
||||
],
|
||||
[
|
||||
'$id' => 'signature',
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => 255,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
],
|
||||
'indexes' => [
|
||||
[
|
||||
'$id' => '_key_accessedAt',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['accessedAt'],
|
||||
'lengths' => [],
|
||||
'orders' => [],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_resource',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['resource'],
|
||||
'lengths' => [],
|
||||
'orders' => [],
|
||||
],
|
||||
],
|
||||
],
|
||||
|
||||
'variables' => [
|
||||
'$collection' => Database::METADATA,
|
||||
'$id' => 'variables',
|
||||
|
|
|
@ -1,15 +1,9 @@
|
|||
<p>{{hello}}</p>
|
||||
|
||||
<br>
|
||||
|
||||
<p>{{hello}},</p>
|
||||
<p>{{body}}</p>
|
||||
|
||||
<a href="{{redirect}}" target="_blank">{{redirect}}</a>
|
||||
|
||||
<p><a href="{{redirect}}" target="_blank">{{redirect}}</a></p>
|
||||
<p>{{footer}}</p>
|
||||
|
||||
<br>
|
||||
|
||||
<p>{{thanks}}</p>
|
||||
|
||||
<p>{{signature}}</p>
|
||||
<p style="margin-bottom: 32px">
|
||||
{{thanks}},
|
||||
<br/>
|
||||
{{signature}}
|
||||
</p>
|
|
@ -1 +1 @@
|
|||
Subproject commit 3b6dd23ff5bf0633508ae7ef2c09640e3da4468b
|
||||
Subproject commit b3a0348b5516c969506370726c784c0fe1f3c62a
|
|
@ -45,6 +45,7 @@ use Utopia\Validator\WhiteList;
|
|||
use Appwrite\Auth\Validator\PasswordHistory;
|
||||
use Appwrite\Auth\Validator\PasswordDictionary;
|
||||
use Appwrite\Auth\Validator\PersonalData;
|
||||
use Appwrite\Hooks\Hooks;
|
||||
|
||||
$oauthDefaultSuccess = '/auth/oauth2/success';
|
||||
$oauthDefaultFailure = '/auth/oauth2/failure';
|
||||
|
@ -76,7 +77,8 @@ App::post('/v1/account')
|
|||
->inject('project')
|
||||
->inject('dbForProject')
|
||||
->inject('queueForEvents')
|
||||
->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) {
|
||||
->inject('hooks')
|
||||
->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
|
||||
|
||||
$email = \strtolower($email);
|
||||
if ('console' === $project->getId()) {
|
||||
|
@ -117,6 +119,8 @@ App::post('/v1/account')
|
|||
}
|
||||
}
|
||||
|
||||
$hooks->trigger('passwordValidator', [$project, $password, $user]);
|
||||
|
||||
$passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
|
||||
$password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
|
||||
try {
|
||||
|
|
|
@ -3563,7 +3563,7 @@ App::get('/v1/databases/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -3571,7 +3571,7 @@ App::get('/v1/databases/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
@ -3647,7 +3647,7 @@ App::get('/v1/databases/:databaseId/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -3655,7 +3655,7 @@ App::get('/v1/databases/:databaseId/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
@ -3733,7 +3733,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -3741,7 +3741,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -484,7 +484,7 @@ App::get('/v1/functions/:functionId/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -492,7 +492,7 @@ App::get('/v1/functions/:functionId/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
@ -576,7 +576,7 @@ App::get('/v1/functions/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -584,7 +584,7 @@ App::get('/v1/functions/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
@ -1730,6 +1730,13 @@ App::post('/v1/functions/:functionId/executions')
|
|||
->setAttribute('responseStatusCode', 500)
|
||||
->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
|
||||
Console::error($th->getMessage());
|
||||
} finally {
|
||||
$queueForUsage
|
||||
->addMetric(METRIC_EXECUTIONS, 1)
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
|
||||
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
|
||||
;
|
||||
}
|
||||
|
||||
if ($function->getAttribute('logging')) {
|
||||
|
@ -1737,15 +1744,6 @@ App::post('/v1/functions/:functionId/executions')
|
|||
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
|
||||
}
|
||||
|
||||
$queueForUsage
|
||||
->addMetric(METRIC_EXECUTIONS, 1)
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
|
||||
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000))// per project
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000))// per function
|
||||
;
|
||||
|
||||
|
||||
|
||||
$roles = Authorization::getRoles();
|
||||
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
|
||||
$isAppUser = Auth::isAppUser($roles);
|
||||
|
|
|
@ -73,7 +73,7 @@ App::get('/v1/project/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) {
|
||||
foreach ($metrics['total'] as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -81,7 +81,7 @@ App::get('/v1/project/usage')
|
|||
}
|
||||
|
||||
foreach ($metrics['period'] as $metric) {
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::greaterThanEqual('time', $firstDay),
|
||||
|
@ -116,7 +116,7 @@ App::get('/v1/project/usage')
|
|||
$id = $function->getId();
|
||||
$name = $function->getAttribute('name');
|
||||
$metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS);
|
||||
$value = $dbForProject->findOne('stats', [
|
||||
$value = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -132,7 +132,7 @@ App::get('/v1/project/usage')
|
|||
$id = $bucket->getId();
|
||||
$name = $bucket->getAttribute('name');
|
||||
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
|
||||
$value = $dbForProject->findOne('stats', [
|
||||
$value = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
|
|
@ -1491,7 +1491,7 @@ App::get('/v1/storage/usage')
|
|||
$total = [];
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -1499,7 +1499,7 @@ App::get('/v1/storage/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
@ -1576,7 +1576,7 @@ App::get('/v1/storage/:bucketId/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -1584,7 +1584,7 @@ App::get('/v1/storage/:bucketId/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -1238,7 +1238,7 @@ App::get('/v1/users/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $count => $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -1246,7 +1246,7 @@ App::get('/v1/users/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -72,6 +72,7 @@ use Ahc\Jwt\JWTException;
|
|||
use Appwrite\Event\Build;
|
||||
use Appwrite\Event\Certificate;
|
||||
use Appwrite\Event\Func;
|
||||
use Appwrite\Hooks\Hooks;
|
||||
use MaxMind\Db\Reader;
|
||||
use PHPMailer\PHPMailer\PHPMailer;
|
||||
use Swoole\Database\PDOProxy;
|
||||
|
@ -828,6 +829,9 @@ $register->set('passwordsDictionary', function () {
|
|||
$register->set('promiseAdapter', function () {
|
||||
return new Swoole();
|
||||
});
|
||||
$register->set('hooks', function () {
|
||||
return new Hooks();
|
||||
});
|
||||
/*
|
||||
* Localization
|
||||
*/
|
||||
|
@ -867,6 +871,10 @@ App::setResource('logger', function ($register) {
|
|||
return $register->get('logger');
|
||||
}, ['register']);
|
||||
|
||||
App::setResource('hooks', function ($register) {
|
||||
return $register->get('hooks');
|
||||
}, ['register']);
|
||||
|
||||
App::setResource('loggerBreadcrumbs', function () {
|
||||
return [];
|
||||
});
|
||||
|
|
|
@ -72,6 +72,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
|
|||
return $adapter;
|
||||
}, ['cache', 'register', 'message', 'dbForConsole']);
|
||||
|
||||
Server::setResource('project', function (Message $message, Database $dbForConsole) {
|
||||
$payload = $message->getPayload() ?? [];
|
||||
$project = new Document($payload['project'] ?? []);
|
||||
|
||||
if ($project->getId() === 'console') {
|
||||
return $project;
|
||||
}
|
||||
return $dbForConsole->getDocument('projects', $project->getId());
|
||||
;
|
||||
}, ['message', 'dbForConsole']);
|
||||
|
||||
Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
|
||||
$databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
|
||||
|
||||
|
@ -103,22 +114,16 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
|
|||
};
|
||||
}, ['pools', 'dbForConsole', 'cache']);
|
||||
|
||||
Server::setResource('getProjectAbuseRetention', function () {
|
||||
return function (Document $project) {
|
||||
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
|
||||
};
|
||||
Server::setResource('abuseRetention', function () {
|
||||
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
|
||||
});
|
||||
|
||||
Server::setResource('getProjectAuditRetention', function () {
|
||||
return function (Document $project) {
|
||||
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
|
||||
};
|
||||
Server::setResource('auditRetention', function () {
|
||||
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
|
||||
});
|
||||
|
||||
Server::setResource('getProjectExecutionRetention', function () {
|
||||
return function (Document $project) {
|
||||
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
|
||||
};
|
||||
Server::setResource('executionRetention', function () {
|
||||
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
|
||||
});
|
||||
|
||||
Server::setResource('cache', function (Registry $register) {
|
||||
|
|
82
composer.lock
generated
82
composer.lock
generated
|
@ -2768,16 +2768,16 @@
|
|||
},
|
||||
{
|
||||
"name": "nikic/php-parser",
|
||||
"version": "v4.17.1",
|
||||
"version": "v4.18.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/nikic/PHP-Parser.git",
|
||||
"reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d"
|
||||
"reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d",
|
||||
"reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d",
|
||||
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/1bcbb2179f97633e98bbbc87044ee2611c7d7999",
|
||||
"reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
|
@ -2818,9 +2818,9 @@
|
|||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/nikic/PHP-Parser/issues",
|
||||
"source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1"
|
||||
"source": "https://github.com/nikic/PHP-Parser/tree/v4.18.0"
|
||||
},
|
||||
"time": "2023-08-13T19:53:39+00:00"
|
||||
"time": "2023-12-10T21:03:43+00:00"
|
||||
},
|
||||
{
|
||||
"name": "phar-io/manifest",
|
||||
|
@ -3103,29 +3103,29 @@
|
|||
},
|
||||
{
|
||||
"name": "phpspec/prophecy",
|
||||
"version": "v1.17.0",
|
||||
"version": "v1.18.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/phpspec/prophecy.git",
|
||||
"reference": "15873c65b207b07765dbc3c95d20fdf4a320cbe2"
|
||||
"reference": "d4f454f7e1193933f04e6500de3e79191648ed0c"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/phpspec/prophecy/zipball/15873c65b207b07765dbc3c95d20fdf4a320cbe2",
|
||||
"reference": "15873c65b207b07765dbc3c95d20fdf4a320cbe2",
|
||||
"url": "https://api.github.com/repos/phpspec/prophecy/zipball/d4f454f7e1193933f04e6500de3e79191648ed0c",
|
||||
"reference": "d4f454f7e1193933f04e6500de3e79191648ed0c",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"doctrine/instantiator": "^1.2 || ^2.0",
|
||||
"php": "^7.2 || 8.0.* || 8.1.* || 8.2.*",
|
||||
"php": "^7.2 || 8.0.* || 8.1.* || 8.2.* || 8.3.*",
|
||||
"phpdocumentor/reflection-docblock": "^5.2",
|
||||
"sebastian/comparator": "^3.0 || ^4.0",
|
||||
"sebastian/recursion-context": "^3.0 || ^4.0"
|
||||
"sebastian/comparator": "^3.0 || ^4.0 || ^5.0",
|
||||
"sebastian/recursion-context": "^3.0 || ^4.0 || ^5.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpspec/phpspec": "^6.0 || ^7.0",
|
||||
"phpstan/phpstan": "^1.9",
|
||||
"phpunit/phpunit": "^8.0 || ^9.0"
|
||||
"phpunit/phpunit": "^8.0 || ^9.0 || ^10.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
|
@ -3158,6 +3158,7 @@
|
|||
"keywords": [
|
||||
"Double",
|
||||
"Dummy",
|
||||
"dev",
|
||||
"fake",
|
||||
"mock",
|
||||
"spy",
|
||||
|
@ -3165,9 +3166,9 @@
|
|||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/phpspec/prophecy/issues",
|
||||
"source": "https://github.com/phpspec/prophecy/tree/v1.17.0"
|
||||
"source": "https://github.com/phpspec/prophecy/tree/v1.18.0"
|
||||
},
|
||||
"time": "2023-02-02T15:41:36+00:00"
|
||||
"time": "2023-12-07T16:22:33+00:00"
|
||||
},
|
||||
{
|
||||
"name": "phpstan/phpdoc-parser",
|
||||
|
@ -4654,16 +4655,16 @@
|
|||
},
|
||||
{
|
||||
"name": "squizlabs/php_codesniffer",
|
||||
"version": "3.7.2",
|
||||
"version": "3.8.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/squizlabs/PHP_CodeSniffer.git",
|
||||
"reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879"
|
||||
"url": "https://github.com/PHPCSStandards/PHP_CodeSniffer.git",
|
||||
"reference": "5805f7a4e4958dbb5e944ef1e6edae0a303765e7"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/ed8e00df0a83aa96acf703f8c2979ff33341f879",
|
||||
"reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879",
|
||||
"url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/5805f7a4e4958dbb5e944ef1e6edae0a303765e7",
|
||||
"reference": "5805f7a4e4958dbb5e944ef1e6edae0a303765e7",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
|
@ -4673,7 +4674,7 @@
|
|||
"php": ">=5.4.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0"
|
||||
"phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0 || ^8.0 || ^9.0"
|
||||
},
|
||||
"bin": [
|
||||
"bin/phpcs",
|
||||
|
@ -4692,22 +4693,45 @@
|
|||
"authors": [
|
||||
{
|
||||
"name": "Greg Sherwood",
|
||||
"role": "lead"
|
||||
"role": "Former lead"
|
||||
},
|
||||
{
|
||||
"name": "Juliette Reinders Folmer",
|
||||
"role": "Current lead"
|
||||
},
|
||||
{
|
||||
"name": "Contributors",
|
||||
"homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer/graphs/contributors"
|
||||
}
|
||||
],
|
||||
"description": "PHP_CodeSniffer tokenizes PHP, JavaScript and CSS files and detects violations of a defined set of coding standards.",
|
||||
"homepage": "https://github.com/squizlabs/PHP_CodeSniffer",
|
||||
"homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer",
|
||||
"keywords": [
|
||||
"phpcs",
|
||||
"standards",
|
||||
"static analysis"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/squizlabs/PHP_CodeSniffer/issues",
|
||||
"source": "https://github.com/squizlabs/PHP_CodeSniffer",
|
||||
"wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki"
|
||||
"issues": "https://github.com/PHPCSStandards/PHP_CodeSniffer/issues",
|
||||
"security": "https://github.com/PHPCSStandards/PHP_CodeSniffer/security/policy",
|
||||
"source": "https://github.com/PHPCSStandards/PHP_CodeSniffer",
|
||||
"wiki": "https://github.com/PHPCSStandards/PHP_CodeSniffer/wiki"
|
||||
},
|
||||
"time": "2023-02-22T23:07:41+00:00"
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://github.com/PHPCSStandards",
|
||||
"type": "github"
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/jrfnl",
|
||||
"type": "github"
|
||||
},
|
||||
{
|
||||
"url": "https://opencollective.com/php_codesniffer",
|
||||
"type": "open_collective"
|
||||
}
|
||||
],
|
||||
"time": "2023-12-08T12:32:31+00:00"
|
||||
},
|
||||
{
|
||||
"name": "swoole/ide-helper",
|
||||
|
@ -5104,5 +5128,5 @@
|
|||
"platform-overrides": {
|
||||
"php": "8.0"
|
||||
},
|
||||
"plugin-api-version": "2.2.0"
|
||||
"plugin-api-version": "2.6.0"
|
||||
}
|
||||
|
|
26
src/Appwrite/Hooks/Hooks.php
Normal file
26
src/Appwrite/Hooks/Hooks.php
Normal file
|
@ -0,0 +1,26 @@
|
|||
<?php
|
||||
|
||||
namespace Appwrite\Hooks;
|
||||
|
||||
class Hooks
|
||||
{
|
||||
/**
|
||||
* @var callable[] $hooks
|
||||
*/
|
||||
private static array $hooks = [];
|
||||
|
||||
public static function add(string $name, callable $action)
|
||||
{
|
||||
self::$hooks[$name] = $action;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param mixed[] $params
|
||||
*/
|
||||
public function trigger(string $name, array $params = [])
|
||||
{
|
||||
if (isset(self::$hooks[$name])) {
|
||||
call_user_func_array(self::$hooks[$name], $params);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -91,6 +91,8 @@ class CreateInfMetric extends Action
|
|||
->get($db)
|
||||
->reclaim();
|
||||
}
|
||||
|
||||
Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId());
|
||||
}
|
||||
|
||||
$sum = \count($projects);
|
||||
|
@ -118,11 +120,9 @@ class CreateInfMetric extends Action
|
|||
try {
|
||||
$id = \md5("_inf_{$metric}");
|
||||
|
||||
$dbForProject->deleteDocument('stats', $id);
|
||||
$dbForProject->deleteDocument('stats_v2', $id);
|
||||
|
||||
echo "_inf_{$metric} , $value \n";
|
||||
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
$dbForProject->createDocument('stats_v2', new Document([
|
||||
'$id' => $id,
|
||||
'metric' => $metric,
|
||||
'period' => 'inf',
|
||||
|
@ -144,7 +144,7 @@ class CreateInfMetric extends Action
|
|||
protected function getFromMetric(database $dbForProject, string $metric): int|float
|
||||
{
|
||||
|
||||
return $dbForProject->sum('stats', 'value', [
|
||||
return $dbForProject->sum('stats_v2', 'value', [
|
||||
Query::equal('metric', [
|
||||
$metric,
|
||||
]),
|
||||
|
|
|
@ -240,7 +240,7 @@ class Hamster extends Action
|
|||
$limit = $periodValue['limit'];
|
||||
$period = $periodValue['period'];
|
||||
|
||||
$requestDocs = $dbForProject->find('stats', [
|
||||
$requestDocs = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -44,18 +44,51 @@ class Maintenance extends Action
|
|||
$time = DateTime::now();
|
||||
|
||||
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
|
||||
$this->notifyDeleteExecutionLogs($queueForDeletes);
|
||||
$this->notifyDeleteAbuseLogs($queueForDeletes);
|
||||
$this->notifyDeleteAuditLogs($queueForDeletes);
|
||||
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
|
||||
|
||||
$this->foreachProject($dbForConsole, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
|
||||
$queueForDeletes->setProject($project);
|
||||
|
||||
$this->notifyDeleteExecutionLogs($queueForDeletes);
|
||||
$this->notifyDeleteAbuseLogs($queueForDeletes);
|
||||
$this->notifyDeleteAuditLogs($queueForDeletes);
|
||||
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
|
||||
$this->notifyDeleteExpiredSessions($queueForDeletes);
|
||||
});
|
||||
|
||||
$this->notifyDeleteConnections($queueForDeletes);
|
||||
$this->notifyDeleteExpiredSessions($queueForDeletes);
|
||||
$this->renewCertificates($dbForConsole, $queueForCertificates);
|
||||
$this->notifyDeleteCache($cacheRetention, $queueForDeletes);
|
||||
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
|
||||
}, $interval);
|
||||
}
|
||||
|
||||
protected function foreachProject(Database $dbForConsole, callable $callback): void
|
||||
{
|
||||
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
$limit = 50;
|
||||
$sum = $limit;
|
||||
$executionStart = \microtime(true);
|
||||
|
||||
while ($sum === $limit) {
|
||||
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
|
||||
|
||||
$chunk++;
|
||||
|
||||
/** @var string[] $projectIds */
|
||||
$sum = count($projects);
|
||||
|
||||
foreach ($projects as $project) {
|
||||
$callback($project);
|
||||
$count++;
|
||||
}
|
||||
}
|
||||
|
||||
$executionEnd = \microtime(true);
|
||||
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
|
||||
}
|
||||
|
||||
private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
|
||||
{
|
||||
($queueForDeletes)
|
||||
|
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
namespace Appwrite\Platform\Workers;
|
||||
|
||||
use Appwrite\Auth\Auth;
|
||||
use Executor\Executor;
|
||||
use Throwable;
|
||||
use Utopia\Abuse\Abuse;
|
||||
|
@ -45,17 +46,17 @@ class Deletes extends Action
|
|||
->inject('getFunctionsDevice')
|
||||
->inject('getBuildsDevice')
|
||||
->inject('getCacheDevice')
|
||||
->inject('getProjectAbuseRetention')
|
||||
->inject('getProjectExecutionRetention')
|
||||
->inject('getProjectAuditRetention')
|
||||
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $getProjectAbuseRetention, $getProjectExecutionRetention, $getProjectAuditRetention));
|
||||
->inject('abuseRetention')
|
||||
->inject('executionRetention')
|
||||
->inject('auditRetention')
|
||||
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $abuseRetention, $executionRetention, $auditRetention));
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws Exception
|
||||
* @throws Throwable
|
||||
*/
|
||||
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention): void
|
||||
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention): void
|
||||
{
|
||||
$payload = $message->getPayload() ?? [];
|
||||
|
||||
|
@ -117,12 +118,12 @@ class Deletes extends Action
|
|||
break;
|
||||
|
||||
case DELETE_TYPE_EXECUTIONS:
|
||||
$this->deleteExecutionLogs($dbForConsole, $getProjectDB, $getProjectExecutionRetention);
|
||||
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
|
||||
break;
|
||||
|
||||
case DELETE_TYPE_AUDIT:
|
||||
if (!empty($datetime)) {
|
||||
$this->deleteAuditLogs($dbForConsole, $getProjectDB, $getProjectAuditRetention);
|
||||
if (!$project->isEmpty()) {
|
||||
$this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
|
||||
}
|
||||
|
||||
if (!$document->isEmpty()) {
|
||||
|
@ -130,7 +131,7 @@ class Deletes extends Action
|
|||
}
|
||||
break;
|
||||
case DELETE_TYPE_ABUSE:
|
||||
$this->deleteAbuseLogs($dbForConsole, $getProjectDB, $getProjectAbuseRetention);
|
||||
$this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention);
|
||||
break;
|
||||
|
||||
case DELETE_TYPE_REALTIME:
|
||||
|
@ -138,10 +139,10 @@ class Deletes extends Action
|
|||
break;
|
||||
|
||||
case DELETE_TYPE_SESSIONS:
|
||||
$this->deleteExpiredSessions($dbForConsole, $getProjectDB);
|
||||
$this->deleteExpiredSessions($project, $getProjectDB);
|
||||
break;
|
||||
case DELETE_TYPE_USAGE:
|
||||
$this->deleteUsageStats($dbForConsole, $getProjectDB, $hourlyUsageRetentionDatetime);
|
||||
$this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
|
||||
break;
|
||||
case DELETE_TYPE_CACHE_BY_RESOURCE:
|
||||
$this->deleteCacheByResource($project, $getProjectDB, $resource);
|
||||
|
@ -340,16 +341,14 @@ class Deletes extends Action
|
|||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
private function deleteUsageStats(Database $dbForConsole, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
|
||||
private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
|
||||
{
|
||||
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $hourlyUsageRetentionDatetime) {
|
||||
$dbForProject = $getProjectDB($project);
|
||||
// Delete Usage stats
|
||||
$this->deleteByGroup('stats', [
|
||||
Query::lessThan('time', $hourlyUsageRetentionDatetime),
|
||||
Query::equal('period', ['1h']),
|
||||
], $dbForProject);
|
||||
});
|
||||
$dbForProject = $getProjectDB($project);
|
||||
// Delete Usage stats
|
||||
$this->deleteByGroup('stats_v2', [
|
||||
Query::lessThan('time', $hourlyUsageRetentionDatetime),
|
||||
Query::equal('period', ['1h']),
|
||||
], $dbForProject);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -546,16 +545,13 @@ class Deletes extends Action
|
|||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, callable $getProjectExecutionRetention): void
|
||||
private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
|
||||
{
|
||||
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectExecutionRetention) {
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$datetime = $getProjectExecutionRetention($project);
|
||||
// Delete Executions
|
||||
$this->deleteByGroup('executions', [
|
||||
Query::lessThan('$createdAt', $datetime)
|
||||
], $dbForProject);
|
||||
});
|
||||
$dbForProject = $getProjectDB($project);
|
||||
// Delete Executions
|
||||
$this->deleteByGroup('executions', [
|
||||
Query::lessThan('$createdAt', $datetime)
|
||||
], $dbForProject);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -564,20 +560,16 @@ class Deletes extends Action
|
|||
* @return void
|
||||
* @throws Exception|Throwable
|
||||
*/
|
||||
private function deleteExpiredSessions(Database $dbForConsole, callable $getProjectDB): void
|
||||
private function deleteExpiredSessions(Document $project, callable $getProjectDB): void
|
||||
{
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
|
||||
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
|
||||
|
||||
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($dbForConsole, $getProjectDB) {
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$project = $dbForConsole->getDocument('projects', $project->getId());
|
||||
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
|
||||
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
|
||||
|
||||
// Delete Sessions
|
||||
$this->deleteByGroup('sessions', [
|
||||
Query::lessThan('$createdAt', $expired)
|
||||
], $dbForProject);
|
||||
});
|
||||
// Delete Sessions
|
||||
$this->deleteByGroup('sessions', [
|
||||
Query::lessThan('$createdAt', $expired)
|
||||
], $dbForProject);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -601,19 +593,16 @@ class Deletes extends Action
|
|||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAbuseRetention): void
|
||||
private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void
|
||||
{
|
||||
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAbuseRetention) {
|
||||
$projectId = $project->getId();
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$datetime = $getProjectAbuseRetention($project);
|
||||
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
|
||||
$abuse = new Abuse($timeLimit);
|
||||
$status = $abuse->cleanup($datetime);
|
||||
if (!$status) {
|
||||
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
|
||||
}
|
||||
});
|
||||
$projectId = $project->getId();
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
|
||||
$abuse = new Abuse($timeLimit);
|
||||
$status = $abuse->cleanup($abuseRetention);
|
||||
if (!$status) {
|
||||
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -623,18 +612,15 @@ class Deletes extends Action
|
|||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAuditRetention): void
|
||||
private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void
|
||||
{
|
||||
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAuditRetention) {
|
||||
$projectId = $project->getId();
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$datetime = $getProjectAuditRetention($project);
|
||||
$audit = new Audit($dbForProject);
|
||||
$status = $audit->cleanup($datetime);
|
||||
if (!$status) {
|
||||
throw new Exception('Failed to delete Audit logs for project' . $projectId);
|
||||
}
|
||||
});
|
||||
$projectId = $project->getId();
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$audit = new Audit($dbForProject);
|
||||
$status = $audit->cleanup($auditRetention);
|
||||
if (!$status) {
|
||||
throw new Exception('Failed to delete Audit logs for project' . $projectId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -869,39 +855,6 @@ class Deletes extends Action
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Database $dbForConsole
|
||||
* @param callable $callback
|
||||
* @throws Exception
|
||||
*/
|
||||
private function deleteForProjectIds(database $dbForConsole, callable $callback): void
|
||||
{
|
||||
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
$limit = 50;
|
||||
$sum = $limit;
|
||||
$executionStart = \microtime(true);
|
||||
|
||||
while ($sum === $limit) {
|
||||
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
|
||||
|
||||
$chunk++;
|
||||
|
||||
/** @var string[] $projectIds */
|
||||
$sum = count($projects);
|
||||
|
||||
Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
|
||||
foreach ($projects as $project) {
|
||||
$callback($project);
|
||||
$count++;
|
||||
}
|
||||
}
|
||||
|
||||
$executionEnd = \microtime(true);
|
||||
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $collection collectionID
|
||||
* @param array $queries
|
||||
|
|
|
@ -421,6 +421,16 @@ class Functions extends Action
|
|||
|
||||
$error = $th->getMessage();
|
||||
$errorCode = $th->getCode();
|
||||
} finally {
|
||||
/** Trigger usage queue */
|
||||
$queueForUsage
|
||||
->setProject($project)
|
||||
->addMetric(METRIC_EXECUTIONS, 1)
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
|
||||
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
|
||||
->trigger()
|
||||
;
|
||||
}
|
||||
|
||||
if ($function->getAttribute('logging')) {
|
||||
|
@ -472,15 +482,5 @@ class Functions extends Action
|
|||
if (!empty($error)) {
|
||||
throw new Exception($error, $errorCode);
|
||||
}
|
||||
|
||||
/** Trigger usage queue */
|
||||
$queueForUsage
|
||||
->setProject($project)
|
||||
->addMetric(METRIC_EXECUTIONS, 1)
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
|
||||
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
|
||||
->trigger()
|
||||
;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,7 +6,7 @@ use Exception;
|
|||
use Utopia\App;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\DSN\DSN;
|
||||
use Utopia\Messaging\Messages\Sms;
|
||||
use Utopia\Messaging\Messages\SMS;
|
||||
use Utopia\Messaging\Adapters\SMS\Mock;
|
||||
use Utopia\Messaging\Adapters\SMS\Msg91;
|
||||
use Utopia\Messaging\Adapters\SMS\Telesign;
|
||||
|
|
|
@ -121,8 +121,8 @@ class Usage extends Action
|
|||
}
|
||||
break;
|
||||
case $document->getCollection() === 'databases': // databases
|
||||
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
|
||||
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
|
||||
$collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
|
||||
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
|
||||
if (!empty($collections['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_COLLECTIONS,
|
||||
|
@ -140,7 +140,7 @@ class Usage extends Action
|
|||
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
|
||||
$parts = explode('_', $document->getCollection());
|
||||
$databaseInternalId = $parts[1] ?? 0;
|
||||
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
|
||||
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
|
||||
|
||||
if (!empty($documents['value'])) {
|
||||
$metrics[] = [
|
||||
|
@ -155,8 +155,8 @@ class Usage extends Action
|
|||
break;
|
||||
|
||||
case $document->getCollection() === 'buckets':
|
||||
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
|
||||
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
|
||||
$files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
|
||||
$storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
|
||||
|
||||
if (!empty($files['value'])) {
|
||||
$metrics[] = [
|
||||
|
@ -174,13 +174,13 @@ class Usage extends Action
|
|||
break;
|
||||
|
||||
case $document->getCollection() === 'functions':
|
||||
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
|
||||
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
|
||||
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
|
||||
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
|
||||
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
|
||||
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
|
||||
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
|
||||
$deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
|
||||
$deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
|
||||
$builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
|
||||
$buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
|
||||
$buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
|
||||
$executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
|
||||
$executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
|
||||
|
||||
if (!empty($deployments['value'])) {
|
||||
$metrics[] = [
|
||||
|
|
|
@ -67,7 +67,7 @@ class UsageHook extends Usage
|
|||
$id = \md5("{$time}_{$period}_{$key}");
|
||||
|
||||
try {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
$dbForProject->createDocument('stats_v2', new Document([
|
||||
'$id' => $id,
|
||||
'period' => $period,
|
||||
'time' => $time,
|
||||
|
@ -78,14 +78,14 @@ class UsageHook extends Usage
|
|||
} catch (Duplicate $th) {
|
||||
if ($value < 0) {
|
||||
$dbForProject->decreaseDocumentAttribute(
|
||||
'stats',
|
||||
'stats_v2',
|
||||
$id,
|
||||
'value',
|
||||
abs($value)
|
||||
);
|
||||
} else {
|
||||
$dbForProject->increaseDocumentAttribute(
|
||||
'stats',
|
||||
'stats_v2',
|
||||
$id,
|
||||
'value',
|
||||
$value
|
||||
|
|
Loading…
Reference in a new issue