1
0
Fork 0
mirror of synced 2024-10-01 17:58:02 +13:00

Merge branch 'refactor-usage-sn' of github.com:appwrite/appwrite into refactor-hamster-usage

 Conflicts:
	src/Appwrite/Platform/Tasks/CalcTierStats.php
This commit is contained in:
shimon 2023-12-21 14:25:06 +02:00
commit 4e24e3cdc2
22 changed files with 325 additions and 280 deletions

2
.gitmodules vendored
View file

@ -1,4 +1,4 @@
[submodule "app/console"]
path = app/console
url = https://github.com/appwrite/console
branch = billing
branch = 3.3.4

View file

@ -18,6 +18,63 @@ $auth = Config::getParam('auth', []);
*/
$commonCollections = [
'cache' => [
'$collection' => Database::METADATA,
'$id' => 'cache',
'name' => 'Cache',
'attributes' => [
[
'$id' => 'resource',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'accessedAt',
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => 'signature',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => '_key_accessedAt',
'type' => Database::INDEX_KEY,
'attributes' => ['accessedAt'],
'lengths' => [],
'orders' => [],
],
[
'$id' => '_key_resource',
'type' => Database::INDEX_KEY,
'attributes' => ['resource'],
'lengths' => [],
'orders' => [],
],
],
],
'users' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('users'),
@ -1270,10 +1327,10 @@ $commonCollections = [
]
],
'stats' => [
'stats_v2' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('stats'),
'name' => 'Stats',
'$id' => ID::custom('stats_v2'),
'name' => 'stats_v2',
'attributes' => [
[
'$id' => ID::custom('metric'),
@ -2872,63 +2929,6 @@ $projectCollections = array_merge([
],
],
'cache' => [
'$collection' => Database::METADATA,
'$id' => 'cache',
'name' => 'Cache',
'attributes' => [
[
'$id' => 'resource',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'accessedAt',
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => 'signature',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => '_key_accessedAt',
'type' => Database::INDEX_KEY,
'attributes' => ['accessedAt'],
'lengths' => [],
'orders' => [],
],
[
'$id' => '_key_resource',
'type' => Database::INDEX_KEY,
'attributes' => ['resource'],
'lengths' => [],
'orders' => [],
],
],
],
'variables' => [
'$collection' => Database::METADATA,
'$id' => 'variables',

View file

@ -1,15 +1,9 @@
<p>{{hello}}</p>
<br>
<p>{{hello}},</p>
<p>{{body}}</p>
<a href="{{redirect}}" target="_blank">{{redirect}}</a>
<p><a href="{{redirect}}" target="_blank">{{redirect}}</a></p>
<p>{{footer}}</p>
<br>
<p>{{thanks}}</p>
<p>{{signature}}</p>
<p style="margin-bottom: 32px">
{{thanks}},
<br/>
{{signature}}
</p>

@ -1 +1 @@
Subproject commit 3b6dd23ff5bf0633508ae7ef2c09640e3da4468b
Subproject commit b3a0348b5516c969506370726c784c0fe1f3c62a

View file

@ -45,6 +45,7 @@ use Utopia\Validator\WhiteList;
use Appwrite\Auth\Validator\PasswordHistory;
use Appwrite\Auth\Validator\PasswordDictionary;
use Appwrite\Auth\Validator\PersonalData;
use Appwrite\Hooks\Hooks;
$oauthDefaultSuccess = '/auth/oauth2/success';
$oauthDefaultFailure = '/auth/oauth2/failure';
@ -76,7 +77,8 @@ App::post('/v1/account')
->inject('project')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) {
->inject('hooks')
->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
$email = \strtolower($email);
if ('console' === $project->getId()) {
@ -117,6 +119,8 @@ App::post('/v1/account')
}
}
$hooks->trigger('passwordValidator', [$project, $password, $user]);
$passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
$password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
try {

View file

@ -3563,7 +3563,7 @@ App::get('/v1/databases/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -3571,7 +3571,7 @@ App::get('/v1/databases/usage')
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
@ -3647,7 +3647,7 @@ App::get('/v1/databases/:databaseId/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -3655,7 +3655,7 @@ App::get('/v1/databases/:databaseId/usage')
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
@ -3733,7 +3733,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -3741,7 +3741,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),

View file

@ -484,7 +484,7 @@ App::get('/v1/functions/:functionId/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -492,7 +492,7 @@ App::get('/v1/functions/:functionId/usage')
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
@ -576,7 +576,7 @@ App::get('/v1/functions/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -584,7 +584,7 @@ App::get('/v1/functions/usage')
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
@ -1730,6 +1730,13 @@ App::post('/v1/functions/:functionId/executions')
->setAttribute('responseStatusCode', 500)
->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
Console::error($th->getMessage());
} finally {
$queueForUsage
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
;
}
if ($function->getAttribute('logging')) {
@ -1737,15 +1744,6 @@ App::post('/v1/functions/:functionId/executions')
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
}
$queueForUsage
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000))// per function
;
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles);

View file

@ -73,7 +73,7 @@ App::get('/v1/project/usage')
Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) {
foreach ($metrics['total'] as $metric) {
$result = $dbForProject->findOne('stats', [
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -81,7 +81,7 @@ App::get('/v1/project/usage')
}
foreach ($metrics['period'] as $metric) {
$results = $dbForProject->find('stats', [
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::greaterThanEqual('time', $firstDay),
@ -116,7 +116,7 @@ App::get('/v1/project/usage')
$id = $function->getId();
$name = $function->getAttribute('name');
$metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS);
$value = $dbForProject->findOne('stats', [
$value = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -132,7 +132,7 @@ App::get('/v1/project/usage')
$id = $bucket->getId();
$name = $bucket->getAttribute('name');
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
$value = $dbForProject->findOne('stats', [
$value = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);

View file

@ -1491,7 +1491,7 @@ App::get('/v1/storage/usage')
$total = [];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -1499,7 +1499,7 @@ App::get('/v1/storage/usage')
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
@ -1576,7 +1576,7 @@ App::get('/v1/storage/:bucketId/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -1584,7 +1584,7 @@ App::get('/v1/storage/:bucketId/usage')
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),

View file

@ -1238,7 +1238,7 @@ App::get('/v1/users/usage')
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $count => $metric) {
$result = $dbForProject->findOne('stats', [
$result = $dbForProject->findOne('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
@ -1246,7 +1246,7 @@ App::get('/v1/users/usage')
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
$results = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),

View file

@ -72,6 +72,7 @@ use Ahc\Jwt\JWTException;
use Appwrite\Event\Build;
use Appwrite\Event\Certificate;
use Appwrite\Event\Func;
use Appwrite\Hooks\Hooks;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy;
@ -828,6 +829,9 @@ $register->set('passwordsDictionary', function () {
$register->set('promiseAdapter', function () {
return new Swoole();
});
$register->set('hooks', function () {
return new Hooks();
});
/*
* Localization
*/
@ -867,6 +871,10 @@ App::setResource('logger', function ($register) {
return $register->get('logger');
}, ['register']);
App::setResource('hooks', function ($register) {
return $register->get('hooks');
}, ['register']);
App::setResource('loggerBreadcrumbs', function () {
return [];
});

View file

@ -72,6 +72,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
return $adapter;
}, ['cache', 'register', 'message', 'dbForConsole']);
Server::setResource('project', function (Message $message, Database $dbForConsole) {
$payload = $message->getPayload() ?? [];
$project = new Document($payload['project'] ?? []);
if ($project->getId() === 'console') {
return $project;
}
return $dbForConsole->getDocument('projects', $project->getId());
;
}, ['message', 'dbForConsole']);
Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
$databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
@ -103,22 +114,16 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
};
}, ['pools', 'dbForConsole', 'cache']);
Server::setResource('getProjectAbuseRetention', function () {
return function (Document $project) {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
};
Server::setResource('abuseRetention', function () {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
});
Server::setResource('getProjectAuditRetention', function () {
return function (Document $project) {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
};
Server::setResource('auditRetention', function () {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
});
Server::setResource('getProjectExecutionRetention', function () {
return function (Document $project) {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
};
Server::setResource('executionRetention', function () {
return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
});
Server::setResource('cache', function (Registry $register) {

82
composer.lock generated
View file

@ -2768,16 +2768,16 @@
},
{
"name": "nikic/php-parser",
"version": "v4.17.1",
"version": "v4.18.0",
"source": {
"type": "git",
"url": "https://github.com/nikic/PHP-Parser.git",
"reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d"
"reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d",
"reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d",
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/1bcbb2179f97633e98bbbc87044ee2611c7d7999",
"reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999",
"shasum": ""
},
"require": {
@ -2818,9 +2818,9 @@
],
"support": {
"issues": "https://github.com/nikic/PHP-Parser/issues",
"source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1"
"source": "https://github.com/nikic/PHP-Parser/tree/v4.18.0"
},
"time": "2023-08-13T19:53:39+00:00"
"time": "2023-12-10T21:03:43+00:00"
},
{
"name": "phar-io/manifest",
@ -3103,29 +3103,29 @@
},
{
"name": "phpspec/prophecy",
"version": "v1.17.0",
"version": "v1.18.0",
"source": {
"type": "git",
"url": "https://github.com/phpspec/prophecy.git",
"reference": "15873c65b207b07765dbc3c95d20fdf4a320cbe2"
"reference": "d4f454f7e1193933f04e6500de3e79191648ed0c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phpspec/prophecy/zipball/15873c65b207b07765dbc3c95d20fdf4a320cbe2",
"reference": "15873c65b207b07765dbc3c95d20fdf4a320cbe2",
"url": "https://api.github.com/repos/phpspec/prophecy/zipball/d4f454f7e1193933f04e6500de3e79191648ed0c",
"reference": "d4f454f7e1193933f04e6500de3e79191648ed0c",
"shasum": ""
},
"require": {
"doctrine/instantiator": "^1.2 || ^2.0",
"php": "^7.2 || 8.0.* || 8.1.* || 8.2.*",
"php": "^7.2 || 8.0.* || 8.1.* || 8.2.* || 8.3.*",
"phpdocumentor/reflection-docblock": "^5.2",
"sebastian/comparator": "^3.0 || ^4.0",
"sebastian/recursion-context": "^3.0 || ^4.0"
"sebastian/comparator": "^3.0 || ^4.0 || ^5.0",
"sebastian/recursion-context": "^3.0 || ^4.0 || ^5.0"
},
"require-dev": {
"phpspec/phpspec": "^6.0 || ^7.0",
"phpstan/phpstan": "^1.9",
"phpunit/phpunit": "^8.0 || ^9.0"
"phpunit/phpunit": "^8.0 || ^9.0 || ^10.0"
},
"type": "library",
"extra": {
@ -3158,6 +3158,7 @@
"keywords": [
"Double",
"Dummy",
"dev",
"fake",
"mock",
"spy",
@ -3165,9 +3166,9 @@
],
"support": {
"issues": "https://github.com/phpspec/prophecy/issues",
"source": "https://github.com/phpspec/prophecy/tree/v1.17.0"
"source": "https://github.com/phpspec/prophecy/tree/v1.18.0"
},
"time": "2023-02-02T15:41:36+00:00"
"time": "2023-12-07T16:22:33+00:00"
},
{
"name": "phpstan/phpdoc-parser",
@ -4654,16 +4655,16 @@
},
{
"name": "squizlabs/php_codesniffer",
"version": "3.7.2",
"version": "3.8.0",
"source": {
"type": "git",
"url": "https://github.com/squizlabs/PHP_CodeSniffer.git",
"reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879"
"url": "https://github.com/PHPCSStandards/PHP_CodeSniffer.git",
"reference": "5805f7a4e4958dbb5e944ef1e6edae0a303765e7"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/ed8e00df0a83aa96acf703f8c2979ff33341f879",
"reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879",
"url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/5805f7a4e4958dbb5e944ef1e6edae0a303765e7",
"reference": "5805f7a4e4958dbb5e944ef1e6edae0a303765e7",
"shasum": ""
},
"require": {
@ -4673,7 +4674,7 @@
"php": ">=5.4.0"
},
"require-dev": {
"phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0"
"phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0 || ^8.0 || ^9.0"
},
"bin": [
"bin/phpcs",
@ -4692,22 +4693,45 @@
"authors": [
{
"name": "Greg Sherwood",
"role": "lead"
"role": "Former lead"
},
{
"name": "Juliette Reinders Folmer",
"role": "Current lead"
},
{
"name": "Contributors",
"homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer/graphs/contributors"
}
],
"description": "PHP_CodeSniffer tokenizes PHP, JavaScript and CSS files and detects violations of a defined set of coding standards.",
"homepage": "https://github.com/squizlabs/PHP_CodeSniffer",
"homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer",
"keywords": [
"phpcs",
"standards",
"static analysis"
],
"support": {
"issues": "https://github.com/squizlabs/PHP_CodeSniffer/issues",
"source": "https://github.com/squizlabs/PHP_CodeSniffer",
"wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki"
"issues": "https://github.com/PHPCSStandards/PHP_CodeSniffer/issues",
"security": "https://github.com/PHPCSStandards/PHP_CodeSniffer/security/policy",
"source": "https://github.com/PHPCSStandards/PHP_CodeSniffer",
"wiki": "https://github.com/PHPCSStandards/PHP_CodeSniffer/wiki"
},
"time": "2023-02-22T23:07:41+00:00"
"funding": [
{
"url": "https://github.com/PHPCSStandards",
"type": "github"
},
{
"url": "https://github.com/jrfnl",
"type": "github"
},
{
"url": "https://opencollective.com/php_codesniffer",
"type": "open_collective"
}
],
"time": "2023-12-08T12:32:31+00:00"
},
{
"name": "swoole/ide-helper",
@ -5104,5 +5128,5 @@
"platform-overrides": {
"php": "8.0"
},
"plugin-api-version": "2.2.0"
"plugin-api-version": "2.6.0"
}

View file

@ -0,0 +1,26 @@
<?php
namespace Appwrite\Hooks;
class Hooks
{
/**
* @var callable[] $hooks
*/
private static array $hooks = [];
public static function add(string $name, callable $action)
{
self::$hooks[$name] = $action;
}
/**
* @param mixed[] $params
*/
public function trigger(string $name, array $params = [])
{
if (isset(self::$hooks[$name])) {
call_user_func_array(self::$hooks[$name], $params);
}
}
}

View file

@ -91,6 +91,8 @@ class CreateInfMetric extends Action
->get($db)
->reclaim();
}
Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId());
}
$sum = \count($projects);
@ -118,11 +120,9 @@ class CreateInfMetric extends Action
try {
$id = \md5("_inf_{$metric}");
$dbForProject->deleteDocument('stats', $id);
$dbForProject->deleteDocument('stats_v2', $id);
echo "_inf_{$metric} , $value \n";
$dbForProject->createDocument('stats', new Document([
$dbForProject->createDocument('stats_v2', new Document([
'$id' => $id,
'metric' => $metric,
'period' => 'inf',
@ -144,7 +144,7 @@ class CreateInfMetric extends Action
protected function getFromMetric(database $dbForProject, string $metric): int|float
{
return $dbForProject->sum('stats', 'value', [
return $dbForProject->sum('stats_v2', 'value', [
Query::equal('metric', [
$metric,
]),

View file

@ -240,7 +240,7 @@ class Hamster extends Action
$limit = $periodValue['limit'];
$period = $periodValue['period'];
$requestDocs = $dbForProject->find('stats', [
$requestDocs = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),

View file

@ -44,18 +44,51 @@ class Maintenance extends Action
$time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
$this->notifyDeleteExecutionLogs($queueForDeletes);
$this->notifyDeleteAbuseLogs($queueForDeletes);
$this->notifyDeleteAuditLogs($queueForDeletes);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->foreachProject($dbForConsole, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$queueForDeletes->setProject($project);
$this->notifyDeleteExecutionLogs($queueForDeletes);
$this->notifyDeleteAbuseLogs($queueForDeletes);
$this->notifyDeleteAuditLogs($queueForDeletes);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
});
$this->notifyDeleteConnections($queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
$this->renewCertificates($dbForConsole, $queueForCertificates);
$this->notifyDeleteCache($cacheRetention, $queueForDeletes);
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
}, $interval);
}
protected function foreachProject(Database $dbForConsole, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
{
($queueForDeletes)

View file

@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Auth\Auth;
use Executor\Executor;
use Throwable;
use Utopia\Abuse\Abuse;
@ -45,17 +46,17 @@ class Deletes extends Action
->inject('getFunctionsDevice')
->inject('getBuildsDevice')
->inject('getCacheDevice')
->inject('getProjectAbuseRetention')
->inject('getProjectExecutionRetention')
->inject('getProjectAuditRetention')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $getProjectAbuseRetention, $getProjectExecutionRetention, $getProjectAuditRetention));
->inject('abuseRetention')
->inject('executionRetention')
->inject('auditRetention')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $abuseRetention, $executionRetention, $auditRetention));
}
/**
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention): void
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention): void
{
$payload = $message->getPayload() ?? [];
@ -117,12 +118,12 @@ class Deletes extends Action
break;
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($dbForConsole, $getProjectDB, $getProjectExecutionRetention);
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
break;
case DELETE_TYPE_AUDIT:
if (!empty($datetime)) {
$this->deleteAuditLogs($dbForConsole, $getProjectDB, $getProjectAuditRetention);
if (!$project->isEmpty()) {
$this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
}
if (!$document->isEmpty()) {
@ -130,7 +131,7 @@ class Deletes extends Action
}
break;
case DELETE_TYPE_ABUSE:
$this->deleteAbuseLogs($dbForConsole, $getProjectDB, $getProjectAbuseRetention);
$this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention);
break;
case DELETE_TYPE_REALTIME:
@ -138,10 +139,10 @@ class Deletes extends Action
break;
case DELETE_TYPE_SESSIONS:
$this->deleteExpiredSessions($dbForConsole, $getProjectDB);
$this->deleteExpiredSessions($project, $getProjectDB);
break;
case DELETE_TYPE_USAGE:
$this->deleteUsageStats($dbForConsole, $getProjectDB, $hourlyUsageRetentionDatetime);
$this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($project, $getProjectDB, $resource);
@ -340,16 +341,14 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteUsageStats(Database $dbForConsole, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $hourlyUsageRetentionDatetime) {
$dbForProject = $getProjectDB($project);
// Delete Usage stats
$this->deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),
], $dbForProject);
});
$dbForProject = $getProjectDB($project);
// Delete Usage stats
$this->deleteByGroup('stats_v2', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),
], $dbForProject);
}
/**
@ -546,16 +545,13 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, callable $getProjectExecutionRetention): void
private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectExecutionRetention) {
$dbForProject = $getProjectDB($project);
$datetime = $getProjectExecutionRetention($project);
// Delete Executions
$this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
], $dbForProject);
});
$dbForProject = $getProjectDB($project);
// Delete Executions
$this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
], $dbForProject);
}
/**
@ -564,20 +560,16 @@ class Deletes extends Action
* @return void
* @throws Exception|Throwable
*/
private function deleteExpiredSessions(Database $dbForConsole, callable $getProjectDB): void
private function deleteExpiredSessions(Document $project, callable $getProjectDB): void
{
$dbForProject = $getProjectDB($project);
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($dbForConsole, $getProjectDB) {
$dbForProject = $getProjectDB($project);
$project = $dbForConsole->getDocument('projects', $project->getId());
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
// Delete Sessions
$this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired)
], $dbForProject);
});
// Delete Sessions
$this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired)
], $dbForProject);
}
/**
@ -601,19 +593,16 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAbuseRetention): void
private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAbuseRetention) {
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$datetime = $getProjectAbuseRetention($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($datetime);
if (!$status) {
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
}
});
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($abuseRetention);
if (!$status) {
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
}
}
/**
@ -623,18 +612,15 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAuditRetention): void
private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAuditRetention) {
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$datetime = $getProjectAuditRetention($project);
$audit = new Audit($dbForProject);
$status = $audit->cleanup($datetime);
if (!$status) {
throw new Exception('Failed to delete Audit logs for project' . $projectId);
}
});
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$audit = new Audit($dbForProject);
$status = $audit->cleanup($auditRetention);
if (!$status) {
throw new Exception('Failed to delete Audit logs for project' . $projectId);
}
}
/**
@ -869,39 +855,6 @@ class Deletes extends Action
}
}
/**
* @param Database $dbForConsole
* @param callable $callback
* @throws Exception
*/
private function deleteForProjectIds(database $dbForConsole, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
/**
* @param string $collection collectionID
* @param array $queries

View file

@ -421,6 +421,16 @@ class Functions extends Action
$error = $th->getMessage();
$errorCode = $th->getCode();
} finally {
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
}
if ($function->getAttribute('logging')) {
@ -472,15 +482,5 @@ class Functions extends Action
if (!empty($error)) {
throw new Exception($error, $errorCode);
}
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
}
}

View file

@ -6,7 +6,7 @@ use Exception;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\DSN\DSN;
use Utopia\Messaging\Messages\Sms;
use Utopia\Messaging\Messages\SMS;
use Utopia\Messaging\Adapters\SMS\Mock;
use Utopia\Messaging\Adapters\SMS\Msg91;
use Utopia\Messaging\Adapters\SMS\Telesign;

View file

@ -121,8 +121,8 @@ class Usage extends Action
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
$collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
@ -140,7 +140,7 @@ class Usage extends Action
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
@ -155,8 +155,8 @@ class Usage extends Action
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
$files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
@ -174,13 +174,13 @@ class Usage extends Action
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
$deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [

View file

@ -67,7 +67,7 @@ class UsageHook extends Usage
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
$dbForProject->createDocument('stats_v2', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
@ -78,14 +78,14 @@ class UsageHook extends Usage
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
'stats_v2',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
'stats_v2',
$id,
'value',
$value