diff --git a/.travis.yml b/.travis.yml index a90890423c..9279ed15bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,10 @@ arch: os: linux +# Small change +vm: + size: large + language: shell notifications: @@ -35,6 +39,12 @@ install: script: - docker ps -a +# Tests should fail if any container is in exited status +- ALL_UP=`docker ps -aq --filter "status=exited"` +- > + if [[ "$ALL_UP" != "" ]]; then + exit 1 + fi - docker-compose logs appwrite - docker-compose logs mariadb - docker-compose logs appwrite-worker-functions diff --git a/Dockerfile b/Dockerfile index 611bcfaf34..ebf2dd8b1f 100755 --- a/Dockerfile +++ b/Dockerfile @@ -225,6 +225,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/schedule && \ diff --git a/app/cli.php b/app/cli.php index ee6fe1a801..99a77d6382 100644 --- a/app/cli.php +++ b/app/cli.php @@ -1,6 +1,6 @@ task('version') diff --git a/app/config/collections2.php b/app/config/collections2.php index 8079bdd7c8..9ba9635457 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1728,6 +1728,91 @@ $collections = [ ], ], ], + 'stats' => [ + '$collection' => Database::METADATA, + '$id' => 'stats', + 'name' => 'Stats', + 'attributes' => [ + [ + '$id' => 'metric', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'value', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'time', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'period', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 4, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'type', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 1, + 'signed' => false, + 'required' => true, + 'default' => 0, // 0 -> count, 1 -> sum + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_time', + 'type' => Database::INDEX_KEY, + 'attributes' => ['time'], + 'lengths' => [], + 'orders' => [Database::ORDER_DESC], + ], + [ + '$id' => '_key_metric', + 'type' => Database::INDEX_KEY, + 'attributes' => ['metric'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_metric_period', + 'type' => Database::INDEX_KEY, + 'attributes' => ['metric', 'period'], + 'lengths' => [], + 'orders' => [Database::ORDER_DESC], + ], + ], + ] ]; return $collections; diff --git a/app/config/variables.php b/app/config/variables.php index df7346cd59..132e279c2c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -149,6 +149,15 @@ return [ 'required' => false, 'question' => '', 'filter' => '' + ], + [ + 'name' => '_APP_USAGE_AGGREGATION_INTERVAL', + 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to mariadb from InfluxDB. The default value is 30 seconds.', + 'introduction' => '0.10.0', + 'default' => '30', + 'required' => false, + 'question' => '', + 'filter' => '' ] ], ], diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index d0d9046be7..bc34f81dda 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -52,12 +52,14 @@ App::post('/v1/account') ->inject('project') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); if ('console' === $project->getId()) { @@ -120,6 +122,9 @@ App::post('/v1/account') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.create', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($user, Response::MODEL_USER); }); @@ -147,13 +152,15 @@ App::post('/v1/account/sessions') ->inject('locale') ->inject('geodb') ->inject('audits') - ->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits) { + ->inject('usage') + ->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); $protocol = $request->getProtocol(); @@ -227,6 +234,11 @@ App::post('/v1/account/sessions') ->setAttribute('countryName', $countryName) ; + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.create', 1) + ->setParam('provider', 'email') + ; $response->dynamic($session, Response::MODEL_SESSION); }); @@ -357,7 +369,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('geodb') ->inject('audits') ->inject('events') - ->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events) use ($oauthDefaultSuccess) { + ->inject('usage') + ->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events, $usage) use ($oauthDefaultSuccess) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ @@ -365,6 +378,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') /** @var Utopia\Database\Database $dbForInternal */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); @@ -545,6 +559,11 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $events->setParam('eventData', $response->output($session, Response::MODEL_SESSION)); + $usage + ->setParam('users.sessions.create', 1) + ->setParam('projectId', $project->getId()) + ->setParam('provider', 'oauth2-'.$provider) + ; if (!Config::getParam('domainVerification')) { $response ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) @@ -595,7 +614,8 @@ App::post('/v1/account/sessions/anonymous') ->inject('dbForInternal') ->inject('geodb') ->inject('audits') - ->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits) { + ->inject('usage') + ->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Locale\Locale $locale */ @@ -604,6 +624,7 @@ App::post('/v1/account/sessions/anonymous') /** @var Utopia\Database\Database $dbForInternal */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); @@ -686,6 +707,11 @@ App::post('/v1/account/sessions/anonymous') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.sessions.create', 1) + ->setParam('provider', 'anonymous') + ; + if (!Config::getParam('domainVerification')) { $response ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) @@ -771,10 +797,15 @@ App::get('/v1/account') ->label('sdk.response.model', Response::MODEL_USER) ->inject('response') ->inject('user') - ->action(function ($response, $user) { + ->inject('usage') + ->action(function ($response, $user, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ + /** @var Appwrite\Stats\Stats $usage */ + $usage + ->setParam('users.read', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -791,12 +822,17 @@ App::get('/v1/account/prefs') ->label('sdk.response.model', Response::MODEL_PREFERENCES) ->inject('response') ->inject('user') - ->action(function ($response, $user) { + ->inject('usage') + ->action(function ($response, $user, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ + /** @var Appwrite\Stats\Stats $usage */ $prefs = $user->getAttribute('prefs', new \stdClass()); + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -814,10 +850,12 @@ App::get('/v1/account/sessions') ->inject('response') ->inject('user') ->inject('locale') - ->action(function ($response, $user, $locale) { + ->inject('usage') + ->action(function ($response, $user, $locale, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ + /** @var Appwrite\Stats\Stats $usage */ $sessions = $user->getAttribute('sessions', []); $current = Auth::sessionVerify($sessions, Auth::$secret); @@ -831,6 +869,9 @@ App::get('/v1/account/sessions') $sessions[$key] = $session; } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'sessions' => $sessions, 'sum' => count($sessions), @@ -853,13 +894,15 @@ App::get('/v1/account/logs') ->inject('locale') ->inject('geodb') ->inject('dbForInternal') - ->action(function ($response, $user, $locale, $geodb, $dbForInternal) { + ->inject('usage') + ->action(function ($response, $user, $locale, $geodb, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $audit = new Audit($dbForInternal); @@ -906,6 +949,9 @@ App::get('/v1/account/logs') } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST); }); @@ -925,11 +971,13 @@ App::get('/v1/account/sessions/:sessionId') ->inject('user') ->inject('locale') ->inject('dbForInternal') - ->action(function ($sessionId, $response, $user, $locale, $dbForInternal) { + ->inject('usage') + ->action(function ($sessionId, $response, $user, $locale, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $sessions = $user->getAttribute('sessions', []); $sessionId = ($sessionId === 'current') @@ -948,6 +996,10 @@ App::get('/v1/account/sessions/:sessionId') ->setAttribute('countryName', $countryName) ; + $usage + ->setParam('users.read', 1) + ; + return $response->dynamic($session, Response::MODEL_SESSION); } } @@ -972,11 +1024,13 @@ App::patch('/v1/account/name') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($name, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($name, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('name', $name)); @@ -986,6 +1040,10 @@ App::patch('/v1/account/name') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; + $response->dynamic($user, Response::MODEL_USER); }); @@ -1007,11 +1065,13 @@ App::patch('/v1/account/password') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ // Check old password only if its an existing user. if ($user->getAttribute('passwordUpdate') !== 0 && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'))) { // Double check user password @@ -1029,6 +1089,9 @@ App::patch('/v1/account/password') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1050,11 +1113,13 @@ App::patch('/v1/account/email') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($email, $password, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($email, $password, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $isAnonymousUser = is_null($user->getAttribute('email')) && is_null($user->getAttribute('password')); // Check if request is from an anonymous account for converting @@ -1082,6 +1147,9 @@ App::patch('/v1/account/email') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1102,11 +1170,13 @@ App::patch('/v1/account/prefs') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($prefs, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($prefs, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs)); @@ -1115,6 +1185,9 @@ App::patch('/v1/account/prefs') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1135,13 +1208,15 @@ App::delete('/v1/account') ->inject('dbForInternal') ->inject('audits') ->inject('events') - ->action(function ($request, $response, $user, $dbForInternal, $audits, $events) { + ->inject('usage') + ->action(function ($request, $response, $user, $dbForInternal, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', false)); @@ -1171,6 +1246,9 @@ App::delete('/v1/account') ; } + $usage + ->setParam('users.delete', 1) + ; $response ->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) @@ -1198,7 +1276,8 @@ App::delete('/v1/account/sessions/:sessionId') ->inject('locale') ->inject('audits') ->inject('events') - ->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events) { + ->inject('usage') + ->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ @@ -1206,6 +1285,7 @@ App::delete('/v1/account/sessions/:sessionId') /** @var Utopia\Locale\Locale $locale */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $sessionId = ($sessionId === 'current') @@ -1252,6 +1332,10 @@ App::delete('/v1/account/sessions/:sessionId') ->setParam('eventData', $response->output($session, Response::MODEL_SESSION)) ; + $usage + ->setParam('users.sessions.delete', 1) + ->setParam('users.update', 1) + ; return $response->noContent(); } } @@ -1278,7 +1362,8 @@ App::delete('/v1/account/sessions') ->inject('locale') ->inject('audits') ->inject('events') - ->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events) { + ->inject('usage') + ->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ @@ -1286,6 +1371,7 @@ App::delete('/v1/account/sessions') /** @var Utopia\Locale\Locale $locale */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $sessions = $user->getAttribute('sessions', []); @@ -1321,13 +1407,19 @@ App::delete('/v1/account/sessions') $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('sessions', [])); + $numOfSessions = count($sessions); + $events ->setParam('eventData', $response->output(new Document([ 'sessions' => $sessions, - 'sum' => count($sessions), + 'sum' => $numOfSessions, ]), Response::MODEL_SESSION_LIST)) ; + $usage + ->setParam('users.sessions.delete', $numOfSessions) + ->setParam('users.update', 1) + ; $response->noContent(); }); @@ -1355,7 +1447,8 @@ App::post('/v1/account/recovery') ->inject('mails') ->inject('audits') ->inject('events') - ->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events) { + ->inject('usage') + ->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -1364,6 +1457,7 @@ App::post('/v1/account/recovery') /** @var Appwrite\Event\Event $mails */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles); $isAppUser = Auth::isAppUser(Authorization::$roles); @@ -1431,6 +1525,9 @@ App::post('/v1/account/recovery') ->setParam('resource', 'user/' . $profile->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($recovery, Response::MODEL_TOKEN); }); @@ -1456,10 +1553,12 @@ App::put('/v1/account/recovery') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ if ($password !== $passwordAgain) { throw new Exception('Passwords must match', 400); @@ -1506,6 +1605,9 @@ App::put('/v1/account/recovery') ->setParam('resource', 'user/' . $profile->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($recovery, Response::MODEL_TOKEN); }); @@ -1533,7 +1635,8 @@ App::post('/v1/account/verification') ->inject('audits') ->inject('events') ->inject('mails') - ->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails) { + ->inject('usage') + ->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ @@ -1543,6 +1646,7 @@ App::post('/v1/account/verification') /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $mails */ + /** @var Appwrite\Stats\Stats $usage */ $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles); $isAppUser = Auth::isAppUser(Authorization::$roles); @@ -1600,6 +1704,9 @@ App::post('/v1/account/verification') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($verification, Response::MODEL_TOKEN); }); @@ -1624,11 +1731,13 @@ App::put('/v1/account/verification') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $profile = $dbForInternal->getDocument('users', $userId); @@ -1666,5 +1775,8 @@ App::put('/v1/account/verification') ->setParam('resource', 'user/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($verification, Response::MODEL_TOKEN); }); diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 5560d931d9..2b37f454dc 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -28,13 +28,15 @@ use Utopia\Database\Exception\Structure as StructureException; use Appwrite\Utopia\Response; use Appwrite\Database\Validator\CustomId; use DeviceDetector\DeviceDetector; +use Utopia\Database\Validator\Authorization; -$attributesCallback = function ($collectionId, $attribute, $response, $dbForInternal, $database, $audits) { +$attributesCallback = function ($collectionId, $attribute, $response, $dbForInternal, $database, $audits, $usage) { /** @var Utopia\Database\Document $attribute*/ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $attributeId = $attribute->getId(); $type = $attribute->getAttribute('type', ''); @@ -105,6 +107,8 @@ $attributesCallback = function ($collectionId, $attribute, $response, $dbForInte ->setParam('document', $attribute) ; + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.attributes.create') ->setParam('resource', 'collection/'.$collection->getId()) @@ -136,10 +140,12 @@ App::post('/v1/database/collections') ->inject('dbForInternal') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collectionId = $collectionId == 'unique()' ? $dbForExternal->getId() : $collectionId; @@ -166,6 +172,8 @@ App::post('/v1/database/collections') ->setParam('data', $collection->getArrayCopy()) ; + $usage->setParam('database.collections.create', 1); + $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -188,7 +196,8 @@ App::get('/v1/database/collections') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -206,6 +215,8 @@ App::get('/v1/database/collections') } } + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'collections' => $dbForInternal->find('collections', $queries, $limit, $offset, [], [$orderType], $afterCollection ?? null), 'sum' => $dbForInternal->count('collections', $queries, APP_LIMIT_COUNT), @@ -226,7 +237,8 @@ App::get('/v1/database/collections/:collectionId') ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -236,9 +248,191 @@ App::get('/v1/database/collections/:collectionId') throw new Exception('Collection not found', 404); } + $usage->setParam('database.collections.read', 1); + $response->dynamic($collection, Response::MODEL_COLLECTION); }); +App::get('/v1/database/usage') + ->desc('Get usage stats for the database') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'database') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_DATABASE) + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + 'database.documents.count', + 'database.collections.count', + 'database.collections.create', + 'database.collections.read', + 'database.collections.update', + 'database.collections.delete', + 'database.documents.create', + 'database.documents.read', + 'database.documents.update', + 'database.documents.delete' + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'documents.count' => $stats["database.documents.count"], + 'collections.count' => $stats["database.collections.count"], + 'documents.create' => $stats["database.documents.create"], + 'documents.read' => $stats["database.documents.read"], + 'documents.update' => $stats["database.documents.update"], + 'documents.delete' => $stats["database.documents.delete"], + 'collections.create' => $stats["database.collections.create"], + 'collections.read' => $stats["database.collections.read"], + 'collections.update' => $stats["database.collections.update"], + 'collections.delete' => $stats["database.collections.delete"], + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_DATABASE); + }); + +App::get('/v1/database/:collectionId/usage') + ->desc('Get usage stats for a collection') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'database') + ->label('sdk.method', 'getCollectionUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION) + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->param('collectionId', '', new UID(), 'Collection unique ID.') + ->inject('response') + ->inject('dbForInternal') + ->inject('dbForExternal') + ->action(function ($range, $collectionId, $response, $dbForInternal, $dbForExternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $collection = $dbForExternal->getCollection($collectionId); + + if ($collection->isEmpty()) { + throw new Exception('Collection not found', 404); + } + + $usage = []; + if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + "database.collections.$collectionId.documents.count", + "database.collections.$collectionId.documents.create", + "database.collections.$collectionId.documents.read", + "database.collections.$collectionId.documents.update", + "database.collections.$collectionId.documents.delete", + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'documents.count' => $stats["database.collections.$collectionId.documents.count"], + 'documents.create' => $stats["database.collections.$collectionId.documents.create"], + 'documents.read' => $stats["database.collections.$collectionId.documents.read"], + 'documents.update' => $stats["database.collections.$collectionId.documents.update"], + 'documents.delete' => $stats["database.collections.$collectionId.documents.delete"] + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_COLLECTION); + }); + App::get('/v1/database/collections/:collectionId/logs') ->desc('List Collection Logs') ->groups(['api', 'database']) @@ -355,10 +549,12 @@ App::put('/v1/database/collections/:collectionId') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -386,6 +582,8 @@ App::put('/v1/database/collections/:collectionId') throw new Exception('Bad structure. '.$exception->getMessage(), 400); } + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.collections.update') ->setParam('resource', 'collection/'.$collection->getId()) @@ -413,12 +611,14 @@ App::delete('/v1/database/collections/:collectionId') ->inject('events') ->inject('audits') ->inject('deletes') - ->action(function ($collectionId, $response, $dbForInternal, $dbForExternal, $events, $audits, $deletes) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForInternal, $dbForExternal, $events, $audits, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $audits */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -426,12 +626,17 @@ App::delete('/v1/database/collections/:collectionId') throw new Exception('Collection not found', 404); } - $dbForExternal->deleteCollection($collectionId); // TDOD move to DB worker - if (!$dbForInternal->deleteDocument('collections', $collectionId)) { throw new Exception('Failed to remove collection from DB', 500); } + $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $collection) + ; + + $usage->setParam('database.collections.delete', 1); + $events ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) ; @@ -467,12 +672,13 @@ App::post('/v1/database/collections/:collectionId/attributes/string') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ - /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback($collectionId, new Document([ '$id' => $attributeId, @@ -481,7 +687,7 @@ App::post('/v1/database/collections/:collectionId/attributes/string') 'required' => $required, 'default' => $default, 'array' => $array, - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/email') @@ -505,11 +711,13 @@ App::post('/v1/database/collections/:collectionId/attributes/email') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback($collectionId, new Document([ '$id' => $attributeId, @@ -519,7 +727,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') 'default' => $default, 'array' => $array, 'format' => 'email', - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/ip') @@ -543,11 +751,13 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback($collectionId, new Document([ '$id' => $attributeId, @@ -557,7 +767,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') 'default' => $default, 'array' => $array, 'format' => 'ip', - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/url') @@ -581,11 +791,13 @@ App::post('/v1/database/collections/:collectionId/attributes/url') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback($collectionId, new Document([ '$id' => $attributeId, @@ -595,7 +807,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') 'default' => $default, 'array' => $array, 'format' => 'url', - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/integer') @@ -621,11 +833,13 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback($collectionId, new Document([ '$id' => $attributeId, @@ -639,7 +853,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') 'min' => (is_null($min)) ? PHP_INT_MIN : \intval($min), 'max' => (is_null($max)) ? PHP_INT_MAX : \intval($max), ], - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/float') @@ -665,11 +879,13 @@ App::post('/v1/database/collections/:collectionId/attributes/float') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback($collectionId, new Document([ '$id' => $attributeId, @@ -683,7 +899,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') 'min' => (is_null($min)) ? PHP_FLOAT_MIN : \floatval($min), 'max' => (is_null($max)) ? PHP_FLOAT_MAX : \floatval($max), ], - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/boolean') @@ -707,11 +923,13 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback($collectionId, new Document([ '$id' => $attributeId, @@ -720,7 +938,7 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') 'required' => $required, 'default' => $default, 'array' => $array, - ]), $response, $dbForInternal, $database, $audits); + ]), $response, $dbForInternal, $database, $audits, $usage); }); App::get('/v1/database/collections/:collectionId/attributes') @@ -737,7 +955,8 @@ App::get('/v1/database/collections/:collectionId/attributes') ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -755,6 +974,8 @@ App::get('/v1/database/collections/:collectionId/attributes') ])]); }, $attributes); + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'sum' => \count($attributes), 'attributes' => $attributes @@ -776,7 +997,8 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ->param('attributeId', '', new Key(), 'Attribute ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $attributeId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -798,6 +1020,8 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') $attribute = new Document([\array_merge($attributes[$attributeIndex], [ 'collectionId' => $collectionId, ])]); + + $usage->setParam('database.collections.read', 1); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); }); @@ -820,12 +1044,14 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') ->inject('database') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $attributeId, $response, $dbForInternal, $database, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $response, $dbForInternal, $database, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -848,6 +1074,8 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') ->setParam('document', $attribute) ; + $usage->setParam('database.collections.update', 1); + $events ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)) ; @@ -882,11 +1110,13 @@ App::post('/v1/database/collections/:collectionId/indexes') ->inject('dbForInternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForInternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForInternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $audits */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -951,6 +1181,8 @@ App::post('/v1/database/collections/:collectionId/indexes') ->setParam('document', $index) ; + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.indexes.create') ->setParam('resource', 'collection/'.$collection->getId()) @@ -975,7 +1207,8 @@ App::get('/v1/database/collections/:collectionId/indexes') ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -993,6 +1226,8 @@ App::get('/v1/database/collections/:collectionId/indexes') ])]); }, $indexes); + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'sum' => \count($indexes), 'attributes' => $indexes, @@ -1014,7 +1249,8 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') ->param('indexId', null, new Key(), 'Index ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($collectionId, $indexId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -1036,6 +1272,8 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') $index = new Document([\array_merge($indexes[$indexIndex], [ 'collectionId' => $collectionId, ])]); + + $usage->setParam('database.collections.read', 1); $response->dynamic($index, Response::MODEL_INDEX); }); @@ -1058,12 +1296,14 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') ->inject('database') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $indexId, $response, $dbForInternal, $database, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $response, $dbForInternal, $database, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -1086,6 +1326,8 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') ->setParam('document', $index) ; + $usage->setParam('database.collections.update', 1); + $events ->setParam('payload', $response->output($index, Response::MODEL_INDEX)) ; @@ -1121,12 +1363,14 @@ App::post('/v1/database/collections/:collectionId/documents') ->inject('dbForExternal') ->inject('user') ->inject('audits') - ->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $user, $audits) { + ->inject('usage') + ->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $user, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -1159,6 +1403,11 @@ App::post('/v1/database/collections/:collectionId/documents') throw new Exception('Document already exists', 409); } + $usage + ->setParam('database.documents.create', 1) + ->setParam('collectionId', $collectionId) + ; + $audits ->setParam('event', 'database.documents.create') ->setParam('resource', 'document/'.$document->getId()) @@ -1190,10 +1439,12 @@ App::get('/v1/database/collections/:collectionId/documents') ->inject('response') ->inject('dbForInternal') ->inject('dbForExternal') - ->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForInternal, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForInternal, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -1220,6 +1471,11 @@ App::get('/v1/database/collections/:collectionId/documents') } } + $usage + ->setParam('database.documents.read', 1) + ->setParam('collectionId', $collectionId) + ; + $response->dynamic(new Document([ 'sum' => $dbForExternal->count($collectionId, $queries, APP_LIMIT_COUNT), 'documents' => $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null), @@ -1242,7 +1498,8 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') ->inject('response') ->inject('dbForInternal') ->inject('dbForExternal') - ->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $$dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ @@ -1259,6 +1516,11 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('No document found', 404); } + $usage + ->setParam('database.documents.read', 1) + ->setParam('collectionId', $collectionId) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1283,11 +1545,13 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') ->inject('dbForInternal') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -1330,6 +1594,11 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') catch (StructureException $exception) { throw new Exception($exception->getMessage(), 400); } + + $usage + ->setParam('database.documents.update', 1) + ->setParam('collectionId', $collectionId) + ; $audits ->setParam('event', 'database.documents.update') @@ -1358,11 +1627,13 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') ->inject('dbForExternal') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForInternal->getDocument('collections', $collectionId); @@ -1378,6 +1649,11 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $dbForExternal->deleteDocument($collectionId, $documentId); + $usage + ->setParam('database.documents.delete', 1) + ->setParam('collectionId', $collectionId) + ; + $events ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)) ; diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index fbd27c47f3..2a67c6d4c6 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -146,6 +146,9 @@ App::get('/v1/functions/:functionId/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'functions') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS) ->param('functionId', '', new UID(), 'Function unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) ->inject('response') @@ -164,99 +167,62 @@ App::get('/v1/functions/:functionId/usage') throw new Exception('Function not found', 404); } + $usage = []; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), - 'group' => '30m', + 'period' => '30m', + 'limit' => 48, ], '7d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 7, ], '30d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 30, ], '90d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 90, ], ]; + + $metrics = [ + "functions.$functionId.executions", + "functions.$functionId.failures", + "functions.$functionId.compute" + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $client = $register->get('influxdb'); - - $executions = []; - $failures = []; - $compute = []; - - if ($client) { - $start = $period[$range]['start']->format(DateTime::RFC3339); - $end = $period[$range]['end']->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Executions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $executions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Failures - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' AND "functionStatus"=\'failed\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $failures[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Compute - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_time" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $compute[] = [ - 'value' => round((!empty($point['value'])) ? $point['value'] / 1000 : 0, 2), // minutes - 'date' => \strtotime($point['time']), - ]; - } - } - - $response->json([ + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ 'range' => $range, - 'executions' => [ - 'data' => $executions, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $executions)), - ], - 'failures' => [ - 'data' => $failures, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $failures)), - ], - 'compute' => [ - 'data' => $compute, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $compute)), - ], + 'functions.executions' => $stats["functions.$functionId.executions"], + 'functions.failures' => $stats["functions.$functionId.failures"], + 'functions.compute' => $stats["functions.$functionId.compute"] ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS); }); App::put('/v1/functions/:functionId') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index b83dff6d9e..3c7dbb8e5e 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -6,10 +6,14 @@ use Appwrite\Network\Validator\CNAME; use Appwrite\Network\Validator\Domain as DomainValidator; use Appwrite\Network\Validator\URL; use Appwrite\Utopia\Response; +use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; +use Utopia\Audit\Audit; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Domains\Domain; use Utopia\Exception; @@ -19,13 +23,11 @@ use Utopia\Validator\Integer; use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\Audit\Audit; -use Utopia\Abuse\Adapters\TimeLimit; App::init(function ($project) { /** @var Utopia\Database\Document $project */ - if($project->getId() !== 'console') { + if ($project->getId() !== 'console') { throw new Exception('Access to this API is forbidden.', 401); } }, ['project'], 'projects'); @@ -69,7 +71,7 @@ App::post('/v1/projects') if ($team->isEmpty()) { throw new Exception('Team not found', 404); } - + $auth = Config::getParam('auth', []); $auths = ['limit' => 0]; foreach ($auth as $index => $method) { @@ -215,22 +217,25 @@ App::get('/v1/projects/:projectId') }); App::get('/v1/projects/:projectId/usage') - ->desc('Get Project') + ->desc('Get usage stats for a project') ->groups(['api', 'projects']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForConsole') - ->inject('projectDB') + ->inject('dbForInternal') ->inject('register') - ->action(function ($projectId, $range, $response, $dbForConsole, $projectDB, $register) { + ->action(function ($projectId, $range, $response, $dbForConsole, $dbForInternal, $register) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ - /** @var Appwrite\Database\Database $projectDB */ + /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Registry\Registry $register */ $project = $dbForConsole->getDocument('projects', $projectId); @@ -239,172 +244,72 @@ App::get('/v1/projects/:projectId/usage') throw new Exception('Project not found', 404); } + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ '24h' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), - 'group' => '30m', + 'period' => '30m', + 'limit' => 48, ], '7d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 7, ], '30d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 30, ], '90d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + 'period' => '1d', + 'limit' => 90, ], ]; - $client = $register->get('influxdb'); + $dbForInternal->setNamespace('project_' . $projectId . '_internal'); - $requests = []; - $network = []; - $functions = []; + $metrics = [ + 'requests', + 'network', + 'executions', + 'users.count', + 'database.documents.count', + 'database.collections.count', + 'storage.total' + ]; - if ($client) { - $start = $period[$range]['start']->format(DateTime::RFC3339); - $end = $period[$range]['end']->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); + $stats = []; - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - } - } else { - $requests = []; - $network = []; - $functions = []; - } - - // Users - - $projectDB->getCollection([ - 'limit' => 0, - 'offset' => 0, - 'filters' => [ - '$collection=users', - ], - ]); - - $usersTotal = $projectDB->getSum(); - - // Documents - - $collections = $projectDB->getCollection([ - 'limit' => 100, - 'offset' => 0, - 'filters' => [ - '$collection=collections', - ], - ]); - - $collectionsTotal = $projectDB->getSum(); - - $documents = []; - - foreach ($collections as $collection) { - $result = $projectDB->getCollection([ - 'limit' => 0, - 'offset' => 0, - 'filters' => [ - '$collection=' . $collection['$id'], - ], + $usage = new Document([ + 'range' => $range, + 'requests' => $stats['requests'], + 'network' => $stats['network'], + 'functions' => $stats['executions'], + 'documents' => $stats['database.documents.count'], + 'collections' => $stats['database.collections.count'], + 'users' => $stats['users.count'], + 'storage' => $stats['storage.total'] ]); - - $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; } - $response->json([ - 'range' => $range, - 'requests' => [ - 'data' => $requests, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $requests)), - ], - 'network' => [ - 'data' => \array_map(function ($value) {return ['value' => \round($value['value'] / 1000000, 2), 'date' => $value['date']];}, $network), // convert bytes to mb - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $network)), - ], - 'functions' => [ - 'data' => $functions, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $functions)), - ], - 'collections' => [ - 'data' => $collections, - 'total' => $collectionsTotal, - ], - 'documents' => [ - 'data' => $documents, - 'total' => \array_sum(\array_map(function ($item) { - return $item['total']; - }, $documents)), - ], - 'users' => [ - 'data' => [], - 'total' => $usersTotal, - ], - 'storage' => [ - 'total' => $projectDB->getCount( - [ - 'attribute' => 'sizeOriginal', - 'filters' => [ - '$collection=files', - ], - ] - ) + - $projectDB->getCount( - [ - 'attribute' => 'size', - 'filters' => [ - '$collection=tags', - ], - ] - ), - ], - ]); + $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); }); App::patch('/v1/projects/:projectId') @@ -467,7 +372,7 @@ App::patch('/v1/projects/:projectId/service') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') - ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function($element) {return $element['optional'];})), true), 'Service name.') + ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function ($element) {return $element['optional'];})), true), 'Service name.') ->param('status', null, new Boolean(), 'Service status.') ->inject('response') ->inject('dbForConsole') diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index af58a010d6..7ea048c5b3 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -10,6 +10,7 @@ use Utopia\Validator\HexColor; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Filesystem; use Appwrite\ClamAV\Network; +use Utopia\Database\Validator\Authorization; use Appwrite\Database\Validator\CustomId; use Utopia\Database\Document; use Utopia\Database\Validator\UID; @@ -22,6 +23,7 @@ use Utopia\Image\Image; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Response; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Query; App::post('/v1/storage/files') @@ -54,7 +56,7 @@ App::post('/v1/storage/files') /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $request->getFiles('file'); @@ -150,6 +152,8 @@ App::post('/v1/storage/files') $usage ->setParam('storage', $sizeActual) + ->setParam('storage.files.create', 1) + ->setParam('bucketId', 'default') ; $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -175,9 +179,11 @@ App::get('/v1/storage/files') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, $search)] : []; @@ -189,6 +195,11 @@ App::get('/v1/storage/files') } } + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic(new Document([ 'files' => $dbForInternal->find('files', $queries, $limit, $offset, [], [$orderType], $afterFile ?? null), 'sum' => $dbForInternal->count('files', $queries, APP_LIMIT_COUNT), @@ -209,16 +220,21 @@ App::get('/v1/storage/files/:fileId') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); if (empty($file->getId())) { throw new Exception('File not found', 404); } - + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; $response->dynamic($file, Response::MODEL_FILE); }); @@ -249,11 +265,13 @@ App::get('/v1/storage/files/:fileId/preview') ->inject('response') ->inject('project') ->inject('dbForInternal') - ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $stats */ $storage = 'files'; @@ -366,6 +384,11 @@ App::get('/v1/storage/files/:fileId/preview') $cache->save($key, $data); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response ->setContentType($outputs[$output]) ->addHeader('Expires', $date) @@ -390,9 +413,11 @@ App::get('/v1/storage/files/:fileId/download') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -424,6 +449,11 @@ App::get('/v1/storage/files/:fileId/download') $source = $compressor->decompress($source); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($file->getAttribute('mimeType')) @@ -448,9 +478,11 @@ App::get('/v1/storage/files/:fileId/view') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); $mimes = Config::getParam('storage-mimes'); @@ -490,6 +522,11 @@ App::get('/v1/storage/files/:fileId/view') $output = $compressor->decompress($source); $fileName = $file->getAttribute('name', ''); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($contentType) @@ -520,7 +557,8 @@ App::put('/v1/storage/files/:fileId') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ @@ -542,6 +580,11 @@ App::put('/v1/storage/files/:fileId') ->setParam('resource', 'file/'.$file->getId()) ; + $usage + ->setParam('storage.files.update', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic($file, Response::MODEL_FILE); }); @@ -567,7 +610,7 @@ App::delete('/v1/storage/files/:fileId') /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -590,6 +633,8 @@ App::delete('/v1/storage/files/:fileId') $usage ->setParam('storage', $file->getAttribute('size', 0) * -1) + ->setParam('storage.files.delete', 1) + ->setParam('bucketId', 'default') ; $events @@ -597,4 +642,159 @@ App::delete('/v1/storage/files/:fileId') ; $response->noContent(); + }); + +App::get('/v1/storage/usage') + ->desc('Get usage stats for storage') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_STORAGE) + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + "storage.total", + "storage.files.count" + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'storage' => $stats['storage.total'], + 'files' => $stats['storage.files.count'] + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_STORAGE); + }); + +App::get('/v1/storage/:bucketId/usage') + ->desc('Get usage stats for a storage bucket') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getBucketUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($bucketId, $range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + // TODO: Check if the storage bucket exists else throw 404 + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + "storage.buckets.$bucketId.files.count", + "storage.buckets.$bucketId.files.create", + "storage.buckets.$bucketId.files.read", + "storage.buckets.$bucketId.files.update", + "storage.buckets.$bucketId.files.delete" + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'files.count' => $stats["storage.buckets.$bucketId.files.count"], + 'files.create' => $stats["storage.buckets.$bucketId.files.create"], + 'files.read' => $stats["storage.buckets.$bucketId.files.read"], + 'files.update' => $stats["storage.buckets.$bucketId.files.update"], + 'files.delete' => $stats["storage.buckets.$bucketId.files.delete"] + ]); + } + + $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS); }); \ No newline at end of file diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index ec9daf9801..c6c8207e21 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -17,7 +17,10 @@ use Utopia\Database\Exception\Duplicate; use Utopia\Database\Validator\UID; use DeviceDetector\DeviceDetector; use Appwrite\Database\Validator\CustomId; +use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; App::post('/v1/users') ->desc('Create User') @@ -37,9 +40,11 @@ App::post('/v1/users') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $email, $password, $name, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $email, $password, $name, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); @@ -66,6 +71,10 @@ App::post('/v1/users') throw new Exception('Account already exists', 409); } + $usage + ->setParam('users.create', 1) + ; + $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($user, Response::MODEL_USER); }); @@ -88,9 +97,11 @@ App::get('/v1/users') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ if (!empty($after)) { $afterUser = $dbForInternal->getDocument('users', $after); @@ -102,6 +113,10 @@ App::get('/v1/users') $results = $dbForInternal->find('users', [], $limit, $offset, [], [$orderType], $afterUser ?? null); $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'users' => $results, @@ -123,9 +138,11 @@ App::get('/v1/users/:userId') ->param('userId', '', new UID(), 'User unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -133,6 +150,9 @@ App::get('/v1/users/:userId') throw new Exception('User not found', 404); } + $usage + ->setParam('users.read', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -150,9 +170,11 @@ App::get('/v1/users/:userId/prefs') ->param('userId', '', new UID(), 'User unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -162,6 +184,9 @@ App::get('/v1/users/:userId/prefs') $prefs = $user->getAttribute('prefs', new \stdClass()); + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -180,10 +205,12 @@ App::get('/v1/users/:userId/sessions') ->inject('response') ->inject('dbForInternal') ->inject('locale') - ->action(function ($userId, $response, $dbForInternal, $locale) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $locale, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -203,6 +230,9 @@ App::get('/v1/users/:userId/sessions') $sessions[$key] = $session; } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'sessions' => $sessions, 'sum' => count($sessions), @@ -225,12 +255,14 @@ App::get('/v1/users/:userId/logs') ->inject('dbForInternal') ->inject('locale') ->inject('geodb') - ->action(function ($userId, $response, $dbForInternal, $locale, $geodb) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $locale, $geodb, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -312,6 +344,9 @@ App::get('/v1/users/:userId/logs') } } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST); }); @@ -331,9 +366,11 @@ App::patch('/v1/users/:userId/status') ->param('status', null, new Boolean(true), 'User Status. To activate the user pass `true` and to block the user pass `false`') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $status, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $status, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -343,6 +380,9 @@ App::patch('/v1/users/:userId/status') $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', (bool) $status)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -362,9 +402,11 @@ App::patch('/v1/users/:userId/verification') ->param('emailVerification', false, new Boolean(), 'User Email Verification Status.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $emailVerification, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $emailVerification, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -374,6 +416,9 @@ App::patch('/v1/users/:userId/verification') $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('emailVerification', $emailVerification)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -516,9 +561,11 @@ App::patch('/v1/users/:userId/prefs') ->param('prefs', '', new Assoc(), 'Prefs key-value JSON object.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $prefs, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $prefs, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -528,6 +575,9 @@ App::patch('/v1/users/:userId/prefs') $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -547,10 +597,12 @@ App::delete('/v1/users/:userId/sessions/:sessionId') ->inject('response') ->inject('dbForInternal') ->inject('events') - ->action(function ($userId, $sessionId, $response, $dbForInternal, $events) { + ->inject('usage') + ->action(function ($userId, $sessionId, $response, $dbForInternal, $events, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -577,6 +629,11 @@ App::delete('/v1/users/:userId/sessions/:sessionId') } } + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.delete', 1) + ; + $response->noContent(); }); @@ -595,10 +652,12 @@ App::delete('/v1/users/:userId/sessions') ->inject('response') ->inject('dbForInternal') ->inject('events') - ->action(function ($userId, $response, $dbForInternal, $events) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $events, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -618,6 +677,10 @@ App::delete('/v1/users/:userId/sessions') ->setParam('eventData', $response->output($user, Response::MODEL_USER)) ; + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.delete', 1) + ; $response->noContent(); }); @@ -637,11 +700,13 @@ App::delete('/v1/users/:userId') ->inject('dbForInternal') ->inject('events') ->inject('deletes') - ->action(function ($userId, $response, $dbForInternal, $events, $deletes) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $events, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $deletes */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -652,11 +717,6 @@ App::delete('/v1/users/:userId') if (!$dbForInternal->deleteDocument('users', $userId)) { throw new Exception('Failed to remove user from DB', 500); } - - // $dbForInternal->createDocument('users', new Document([ - // '$id' => $userId, - // '$read' => ['role:all'], - // ])); $deletes ->setParam('type', DELETE_TYPE_DOCUMENT) @@ -667,5 +727,96 @@ App::delete('/v1/users/:userId') ->setParam('eventData', $response->output($user, Response::MODEL_USER)) ; + $usage + ->setParam('users.delete', 1) + ; $response->noContent(); }); + +App::get('/v1/users/usage') + ->desc('Get usage stats for the users API') + ->groups(['api', 'users']) + ->label('scope', 'users.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'users') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_USERS) + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(function($value) { return "oauth-".$value; }, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true) + ->inject('response') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($range, $provider, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + "users.count", + "users.create", + "users.read", + "users.update", + "users.delete", + "users.sessions.create", + "users.sessions.$provider.create", + "users.sessions.delete" + ]; + + $stats = []; + + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); + + $usage = new Document([ + 'range' => $range, + 'users.count' => $stats["users.count"], + 'users.create' => $stats["users.create"], + 'users.read' => $stats["users.read"], + 'users.update' => $stats["users.update"], + 'users.delete' => $stats["users.delete"], + 'sessions.create' => $stats["users.sessions.create"], + 'sessions.provider.create' => $stats["users.sessions.$provider.create"], + 'sessions.delete' => $stats["users.sessions.delete"] + ]); + + } + + $response->dynamic($usage, Response::MODEL_USAGE_USERS); + }); \ No newline at end of file diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 3707eebfad..b7953da491 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -104,6 +104,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $events, $aud ->setParam('httpRequest', 1) ->setParam('httpUrl', $request->getHostname().$request->getURI()) ->setParam('httpMethod', $request->getMethod()) + ->setParam('httpPath', $route->getPath()) ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->setParam('storage', 0) diff --git a/app/init.php b/app/init.php index c5a15cc734..57b55da95e 100644 --- a/app/init.php +++ b/app/init.php @@ -17,6 +17,7 @@ ini_set('display_startup_errors', 1); ini_set('default_socket_timeout', -1); error_reporting(E_ALL); +use Appwrite\Extend\PDO; use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; @@ -88,6 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; +const DELETE_TYPE_USAGE = 'usage'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; @@ -345,6 +347,29 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__.'/db/DBIP/dbip-country-lite-2021-06.mmdb'); }); +$register->set('db', function () { // This is usually for our workers or CLI commands scope + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + + $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + )); + + return $pdo; +}); +$register->set('cache', function () { // This is usually for our workers or CLI commands scope + $redis = new Redis(); + $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); /* * Localization diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index eccdf61b17..e0ef1adc01 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -39,17 +39,29 @@ $cli ]); } + function notifyDeleteUsageStats(int $interval30m, int $interval1d) + { + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ + 'type' => DELETE_TYPE_USAGE, + 'timestamp1d' => time() - $interval1d, + 'timestamp30m' => time() - $interval30m, + ]); + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); + $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours + $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days - Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention){ + Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); + notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); }, $interval); }); \ No newline at end of file diff --git a/app/tasks/usage.php b/app/tasks/usage.php new file mode 100644 index 0000000000..4ff5e8cad9 --- /dev/null +++ b/app/tasks/usage.php @@ -0,0 +1,586 @@ +task('usage') + ->desc('Schedules syncing data from influxdb to Appwrite console db') + ->action(function () use ($register) { + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); + + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) + $periods = [ + [ + 'key' => '30m', + 'startTime' => '-24 hours', + ], + [ + 'key' => '1d', + 'startTime' => '-90 days', + ], + ]; + + // all the metrics that we are collecting at the moment + $globalMetrics = [ + 'requests' => [ + 'table' => 'appwrite_usage_requests_all', + ], + 'network' => [ + 'table' => 'appwrite_usage_network_all', + ], + 'executions' => [ + 'table' => 'appwrite_usage_executions_all', + ], + 'database.collections.create' => [ + 'table' => 'appwrite_usage_database_collections_create', + ], + 'database.collections.read' => [ + 'table' => 'appwrite_usage_database_collections_read', + ], + 'database.collections.update' => [ + 'table' => 'appwrite_usage_database_collections_update', + ], + 'database.collections.delete' => [ + 'table' => 'appwrite_usage_database_collections_delete', + ], + 'database.documents.create' => [ + 'table' => 'appwrite_usage_database_documents_create', + ], + 'database.documents.read' => [ + 'table' => 'appwrite_usage_database_documents_read', + ], + 'database.documents.update' => [ + 'table' => 'appwrite_usage_database_documents_update', + ], + 'database.documents.delete' => [ + 'table' => 'appwrite_usage_database_documents_delete', + ], + 'database.collections.collectionId.documents.create' => [ + 'table' => 'appwrite_usage_database_documents_create', + 'groupBy' => 'collectionId', + ], + 'database.collections.collectionId.documents.read' => [ + 'table' => 'appwrite_usage_database_documents_read', + 'groupBy' => 'collectionId', + ], + 'database.collections.collectionId.documents.update' => [ + 'table' => 'appwrite_usage_database_documents_update', + 'groupBy' => 'collectionId', + ], + 'database.collections.collectionId.documents.delete' => [ + 'table' => 'appwrite_usage_database_documents_delete', + 'groupBy' => 'collectionId', + ], + 'storage.buckets.bucketId.files.create' => [ + 'table' => 'appwrite_usage_storage_files_create', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.read' => [ + 'table' => 'appwrite_usage_storage_files_read', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.update' => [ + 'table' => 'appwrite_usage_storage_files_update', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.delete' => [ + 'table' => 'appwrite_usage_storage_files_delete', + 'groupBy' => 'bucketId', + ], + 'users.create' => [ + 'table' => 'appwrite_usage_users_create', + ], + 'users.read' => [ + 'table' => 'appwrite_usage_users_read', + ], + 'users.update' => [ + 'table' => 'appwrite_usage_users_update', + ], + 'users.delete' => [ + 'table' => 'appwrite_usage_users_delete', + ], + 'users.sessions.create' => [ + 'table' => 'appwrite_usage_users_sessions_create', + ], + 'users.sessions.provider.create' => [ + 'table' => 'appwrite_usage_users_sessions_create', + 'groupBy' => 'provider', + ], + 'users.sessions.delete' => [ + 'table' => 'appwrite_usage_users_sessions_delete', + ], + 'functions.functionId.executions' => [ + 'table' => 'appwrite_usage_executions_all', + 'groupBy' => 'functionId', + ], + 'functions.functionId.compute' => [ + 'table' => 'appwrite_usage_executions_time', + 'groupBy' => 'functionId', + ], + 'functions.functionId.failures' => [ + 'table' => 'appwrite_usage_executions_all', + 'groupBy' => 'functionId', + 'filters' => [ + 'functionStatus' => 'failed', + ], + ], + ]; + + // TODO Maybe move this to the setResource method, and reuse in the http.php file + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // connect to db + try { + $attempts++; + $db = $register->get('db'); + $redis = $register->get('cache'); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); + + // TODO use inject + $cacheAdapter = new Cache(new Redis($redis)); + $dbForProject = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole->setNamespace('project_console_internal'); + + $latestTime = []; + + Authorization::disable(); + + $iterations = 0; + Console::loop(function () use ($interval, $register, $dbForProject, $dbForConsole, $globalMetrics, $periods, &$latestTime, &$iterations) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); + + $loopStart = microtime(true); + + /** + * Aggregate InfluxDB every 30 seconds + * @var InfluxDB\Client $client + */ + $client = $register->get('influxdb'); + if ($client) { + $attempts = 0; + $max = 10; + $sleep = 1; + + $database = $client->selectDB('telegraf'); + do { // check if telegraf database is ready + $attempts++; + if(!in_array('telegraf', $client->listDatabases())) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } else { + break; // leave the do-while if successful + } + } while ($attempts < $max); + + // sync data + foreach ($globalMetrics as $metric => $options) { //for each metrics + foreach ($periods as $period) { // aggregate data for each period + $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); + if (!empty($latestTime[$metric][$period['key']])) { + $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); + } + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + + $table = $options['table']; //Which influxdb table to query for this metric + $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc + + $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status + if (!empty($filters)) { + $filters = ' AND ' . implode(' AND ', array_map(function ($filter, $value) { + return '"' . $filter . '"=\'' . $value . '\''; + }, array_keys($filters), array_values($filters))); + } + + $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\'' . (empty($filters) ? '' : $filters) . ' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); + + $points = $result->getPoints(); + foreach ($points as $point) { + $projectId = $point['projectId']; + + if (!empty($projectId) && $projectId != 'console') { + $dbForProject->setNamespace('project_' . $projectId . '_internal'); + $metricUpdated = $metric; + + if (!empty($groupBy)) { + $groupedBy = $point[$options['groupBy']] ?? ''; + if (empty($groupedBy)) { + continue; + } + $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); + } + + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric + $value = (!empty($point['value'])) ? $point['value'] : 0; + + try { + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period['key'], + 'time' => $time, + 'metric' => $metricUpdated, + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestTime[$metric][$period['key']] = $time; + } catch (\Exception $e) { // if projects are deleted this might fail + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); + } + } + } + } + } + } + + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + if ($iterations % 30 == 0) { // Every 15 minutes aggregate number of objects in database + + $latestProject = null; + + do { // Loop over all the projects + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // list projects + try { + $attempts++; + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed access console db: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); + + if (empty($projects)) { + continue; + } + + $latestProject = $projects[array_key_last($projects)]; + + foreach ($projects as $project) { + $projectId = $project->getId(); + + // Get total storage + $dbForProject->setNamespace('project_' . $projectId . '_internal'); + $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '1d', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } + + $collections = [ + 'users' => [ + 'namespace' => 'internal', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => 'internal', + 'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting + 'documents' => [ + 'namespace' => 'external', + ], + ], + ], + 'files' => [ + 'metricPrefix' => 'storage', + 'namespace' => 'internal', + ], + ]; + + foreach ($collections as $collection => $options) { + try { + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$projectId}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $subCollections = $options['subCollections'] ?? []; + + if (empty($subCollections)) { + continue; + } + + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + + do { // Loop over all the parent collection document for each sub collection + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections + + if (empty($parents)) { + continue; + } + + $latestParent = $parents[array_key_last($parents)]; + + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count + $dbForProject->setNamespace("project_{$projectId}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + + $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count + + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count"; + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + } + } + } while (!empty($parents)); + + /** + * Inserting project level counts for sub collections like database.documents.count + */ + foreach ($subCollectionCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count"; + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + } + } catch (\Exception$e) { + Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); + } + } + } + } while (!empty($projects)); + } + + $iterations++; + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); + }); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 14b5a0818e..28b314b18f 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -294,6 +294,26 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-usage: + image: /: + entrypoint: usage + container_name: appwrite-usage + restart: unless-stopped + networks: + - appwrite + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_AGGREGATION_INTERVAL appwrite-schedule: image: /: diff --git a/app/workers.php b/app/workers.php deleted file mode 100644 index 7dd48eaff8..0000000000 --- a/app/workers.php +++ /dev/null @@ -1,34 +0,0 @@ -set('db', function () { - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); - - $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - )); - - return $pdo; -}); - -$register->set('cache', function () { // Register cache connection - $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); - diff --git a/app/workers/audits.php b/app/workers/audits.php index b18a57ffd1..e18b9d532a 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\Audit\Audit; use Utopia\CLI\Console; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Audits V1 Worker'); Console::success(APP_NAME.' audits worker v1 has started'); diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 96148b9e71..0b64c087f4 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -9,7 +9,7 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Domains\Domain; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Certificates V1 Worker'); Console::success(APP_NAME.' certificates worker v1 has started'); diff --git a/app/workers/database.php b/app/workers/database.php index fd38246150..2366b37998 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -5,7 +5,7 @@ use Utopia\CLI\Console; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Database V1 Worker'); Console::success(APP_NAME.' database worker v1 has started'."\n"); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1894aaf76b..5b1072c135 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -11,7 +11,7 @@ use Utopia\Abuse\Adapters\TimeLimit; use Utopia\CLI\Console; use Utopia\Audit\Audit; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Deletes V1 Worker'); Console::success(APP_NAME.' deletes worker v1 has started'."\n"); @@ -44,6 +44,9 @@ class DeletesV1 extends Worker switch ($document->getCollection()) { // TODO@kodumbeats define these as constants somewhere + case 'collections': + $this->deleteCollection($document, $projectId); + break; case 'projects': $this->deleteProject($document); break; @@ -78,7 +81,10 @@ class DeletesV1 extends Worker $document = new Document($this->args['document']); $this->deleteCertificates($document); break; - + + case DELETE_TYPE_USAGE: + $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); + break; default: Console::error('No delete operation for type: '.$type); break; @@ -88,6 +94,51 @@ class DeletesV1 extends Worker public function shutdown(): void { } + + /** + * @param Document $document teams document + * @param string $projectId + */ + protected function deleteCollection(Document $document, string $projectId): void + { + $collectionId = $document->getId(); + + $dbForInternal = $this->getInternalDB($projectId); + $dbForExternal = $this->getExternalDB($projectId); + + $this->deleteByGroup('attributes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + + $this->deleteByGroup('indexes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + + $dbForExternal->deleteCollection($collectionId); + } + + /** + * @param int $timestamp1d + * @param int $timestamp30m + */ + protected function deleteUsageStats(int $timestamp1d, int $timestamp30m) { + $this->deleteForProjectIds(function($projectId) use ($timestamp1d, $timestamp30m) { + if (!($dbForInternal = $this->getInternalDB($projectId))) { + throw new Exception('Failed to get projectDB for project '.$projectId); + } + + // Delete Usage stats + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp1d]), + new Query('period', Query::TYPE_EQUAL, ['1d']), + ], $dbForInternal); + + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp30m]), + new Query('period', Query::TYPE_EQUAL, ['30m']), + ], $dbForInternal); + }); + } /** * @param Document $document teams document diff --git a/app/workers/functions.php b/app/workers/functions.php index bed0bda138..49b42b9492 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -13,7 +13,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Runtime::enableCoroutine(0); diff --git a/app/workers/mails.php b/app/workers/mails.php index 25abe54aa5..9071c00901 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -6,7 +6,7 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Locale\Locale; -require_once __DIR__ . '/../workers.php'; +require_once __DIR__ . '/../init.php'; Console::title('Mails V1 Worker'); Console::success(APP_NAME . ' mails worker v1 has started' . "\n"); diff --git a/app/workers/webhooks.php b/app/workers/webhooks.php index 00307bdda2..a73dbb6c79 100644 --- a/app/workers/webhooks.php +++ b/app/workers/webhooks.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\App; use Utopia\CLI\Console; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Webhooks V1 Worker'); Console::success(APP_NAME.' webhooks worker v1 has started'); diff --git a/bin/usage b/bin/usage new file mode 100755 index 0000000000..2709200ae4 --- /dev/null +++ b/bin/usage @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php usage $@ \ No newline at end of file diff --git a/composer.json b/composer.json index c51501f712..8ba7dc4b21 100644 --- a/composer.json +++ b/composer.json @@ -83,4 +83,4 @@ "php": "8.0" } } -} +} \ No newline at end of file diff --git a/composer.lock b/composer.lock index b6418d1482..4fc89ea245 100644 --- a/composer.lock +++ b/composer.lock @@ -3383,16 +3383,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.12.0", + "version": "v4.13.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143" + "reference": "50953a2691a922aa1769461637869a0a2faa3f53" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/6608f01670c3cc5079e18c1dab1104e002579143", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/50953a2691a922aa1769461637869a0a2faa3f53", + "reference": "50953a2691a922aa1769461637869a0a2faa3f53", "shasum": "" }, "require": { @@ -3433,9 +3433,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.12.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.13.0" }, - "time": "2021-07-21T10:44:31+00:00" + "time": "2021-09-20T12:20:58+00:00" }, { "name": "openlss/lib-array2xml", @@ -3712,16 +3712,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.4.0", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0" + "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/30f38bffc6f24293dadd1823936372dfa9e86e2f", + "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f", "shasum": "" }, "require": { @@ -3729,7 +3729,8 @@ "phpdocumentor/reflection-common": "^2.0" }, "require-dev": { - "ext-tokenizer": "*" + "ext-tokenizer": "*", + "psalm/phar": "^4.8" }, "type": "library", "extra": { @@ -3755,9 +3756,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.4.0" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.5.0" }, - "time": "2020-09-17T18:55:26+00:00" + "time": "2021-09-17T15:28:14+00:00" }, { "name": "phpspec/prophecy", @@ -3828,23 +3829,23 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.6", + "version": "9.2.7", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "f6293e1b30a2354e8428e004689671b83871edde" + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/f6293e1b30a2354e8428e004689671b83871edde", - "reference": "f6293e1b30a2354e8428e004689671b83871edde", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/d4c798ed8d51506800b441f7a13ecb0f76f12218", + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.10.2", + "nikic/php-parser": "^4.12.0", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -3893,7 +3894,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.6" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.7" }, "funding": [ { @@ -3901,7 +3902,7 @@ "type": "github" } ], - "time": "2021-03-28T07:26:59+00:00" + "time": "2021-09-17T05:39:03+00:00" }, { "name": "phpunit/php-file-iterator", @@ -6015,16 +6016,16 @@ }, { "name": "twig/twig", - "version": "v2.14.6", + "version": "v2.14.7", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260" + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/27e5cf2b05e3744accf39d4c68a3235d9966d260", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/8e202327ee1ed863629de9b18a5ec70ac614d88f", + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f", "shasum": "" }, "require": { @@ -6034,7 +6035,7 @@ }, "require-dev": { "psr/container": "^1.0", - "symfony/phpunit-bridge": "^4.4.9|^5.0.9" + "symfony/phpunit-bridge": "^4.4.9|^5.0.9|^6.0" }, "type": "library", "extra": { @@ -6078,7 +6079,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v2.14.6" + "source": "https://github.com/twigphp/Twig/tree/v2.14.7" }, "funding": [ { @@ -6090,7 +6091,7 @@ "type": "tidelift" } ], - "time": "2021-05-16T12:12:47+00:00" + "time": "2021-09-17T08:39:54+00:00" }, { "name": "vimeo/psalm", diff --git a/docker-compose.yml b/docker-compose.yml index ac12f20106..539799aa9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,7 +73,6 @@ services: - mariadb - redis # - clamav - - influxdb entrypoint: - php - -e @@ -112,8 +111,6 @@ services: - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD - _APP_USAGE_STATS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - _APP_STORAGE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_CONTAINERS @@ -349,6 +346,37 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-usage: + entrypoint: + - php + - -e + - /usr/src/code/app/cli.php + - usage + container_name: appwrite-usage + build: + context: . + args: + - DEBUG=false + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + - ./dev:/usr/local/dev + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_SYNC_INTERVAL + appwrite-schedule: entrypoint: schedule container_name: appwrite-schedule diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 353369eb65..5095c5395f 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -21,6 +21,28 @@ class OpenAPI3 extends Format return 'Open API 3'; } + /** + * Get Used Models + * + * Recursively get all used models + * + * @param object $model + * @param array $models + * + * @return void + */ + protected function getUsedModels($model, array &$usedModels) + { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) { + $usedModels[] = $model; + return; + } + if (!is_object($model)) return; + foreach ($model->getRules() as $rule) { + $this->getUsedModels($rule['type'], $usedModels); + } + } + /** * Parse * @@ -352,11 +374,7 @@ class OpenAPI3 extends Format $output['paths'][$url][\strtolower($route->getMethod())] = $temp; } foreach ($this->models as $model) { - foreach ($model->getRules() as $rule) { - if (!in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float'])) { - $usedModels[] = $rule['type']; - } - } + $this->getUsedModels($model, $usedModels); } foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels) && $model->getType() !== 'error') { diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index d357283a02..2a3be45d17 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -21,6 +21,28 @@ class Swagger2 extends Format return 'Swagger 2'; } + /** + * Get Used Models + * + * Recursively get all used models + * + * @param object $model + * @param array $models + * + * @return void + */ + protected function getUsedModels($model, array &$usedModels) + { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) { + $usedModels[] = $model; + return; + } + if (!is_object($model)) return; + foreach ($model->getRules() as $rule) { + $this->getUsedModels($rule['type'], $usedModels); + } + } + /** * Parse * @@ -354,15 +376,9 @@ class Swagger2 extends Format $output['paths'][$url][\strtolower($route->getMethod())] = $temp; } foreach ($this->models as $model) { - foreach ($model->getRules() as $rule) { - if ( - in_array($model->getType(), $usedModels) - && !in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float']) - ) { - $usedModels[] = $rule['type']; - } - } + $this->getUsedModels($model, $usedModels); } + foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels)) { continue; diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index b07c4c85c5..0cdd19fee3 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -94,7 +94,7 @@ class Stats $functionExecutionTime = $this->params['functionExecutionTime'] ?? 0; $functionStatus = $this->params['functionStatus'] ?? ''; - $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + $tags = ",projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) $this->statsd->setNamespace($this->namespace); @@ -112,7 +112,70 @@ class Stats $this->statsd->count('network.outbound' . $tags, $networkResponseSize); $this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize); + $dbMetrics = [ + 'database.collections.create', + 'database.collections.read', + 'database.collections.update', + 'database.collections.delete', + 'database.documents.create', + 'database.documents.read', + 'database.documents.update', + 'database.documents.delete', + ]; + + foreach ($dbMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); + $this->statsd->increment($metric . $tags); + } + } + + $storageMertics = [ + 'storage.files.create', + 'storage.files.read', + 'storage.files.update', + 'storage.files.delete', + ]; + + foreach ($storageMertics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->increment($metric . $tags); + } + } + + $usersMetrics = [ + 'users.create', + 'users.read', + 'users.update', + 'users.delete', + ]; + + foreach ($usersMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId}"; + $this->statsd->increment($metric . $tags); + } + } + + $sessionsMetrics = [ + 'users.sessions.create', + 'users.sessions.delete', + ]; + + foreach ($sessionsMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId},provider=". ($this->params['provider'] ?? ''); + $this->statsd->count($metric . $tags, $value); + } + } + if ($storage >= 1) { + $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->count('storage.all' . $tags, $storage); } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 3cd25b6032..329dc86432 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -33,6 +33,7 @@ use Appwrite\Utopia\Response\Model\Team; use Appwrite\Utopia\Response\Model\Locale; use Appwrite\Utopia\Response\Model\Log; use Appwrite\Utopia\Response\Model\Membership; +use Appwrite\Utopia\Response\Model\Metric; use Appwrite\Utopia\Response\Model\Permissions; use Appwrite\Utopia\Response\Model\Phone; use Appwrite\Utopia\Response\Model\Platform; @@ -43,6 +44,13 @@ use Appwrite\Utopia\Response\Model\Token; use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last +use Appwrite\Utopia\Response\Model\UsageBuckets; +use Appwrite\Utopia\Response\Model\UsageCollection; +use Appwrite\Utopia\Response\Model\UsageDatabase; +use Appwrite\Utopia\Response\Model\UsageFunctions; +use Appwrite\Utopia\Response\Model\UsageProject; +use Appwrite\Utopia\Response\Model\UsageStorage; +use Appwrite\Utopia\Response\Model\UsageUsers; use stdClass; /** @@ -56,8 +64,17 @@ class Response extends SwooleResponse const MODEL_LOG = 'log'; const MODEL_LOG_LIST = 'logList'; const MODEL_ERROR = 'error'; + const MODEL_METRIC = 'metric'; + const MODEL_METRIC_LIST = 'metricList'; const MODEL_ERROR_DEV = 'errorDev'; const MODEL_BASE_LIST = 'baseList'; + const MODEL_USAGE_DATABASE = 'usageDatabase'; + const MODEL_USAGE_COLLECTION = 'usageCollection'; + const MODEL_USAGE_USERS = 'usageUsers'; + const MODEL_USAGE_BUCKETS = 'usageBuckets'; + const MODEL_USAGE_STORAGE = 'usageStorage'; + const MODEL_USAGE_FUNCTIONS = 'usageFunctions'; + const MODEL_USAGE_PROJECT = 'usageProject'; // Database const MODEL_COLLECTION = 'collection'; @@ -125,6 +142,7 @@ class Response extends SwooleResponse // Deprecated const MODEL_PERMISSIONS = 'permissions'; const MODEL_RULE = 'rule'; + const MODEL_TASK = 'task'; // Tests (keep last) const MODEL_MOCK = 'mock'; @@ -176,6 +194,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Languages List', self::MODEL_LANGUAGE_LIST, 'languages', self::MODEL_LANGUAGE)) ->setModel(new BaseList('Currencies List', self::MODEL_CURRENCY_LIST, 'currencies', self::MODEL_CURRENCY)) ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) + ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false)) // Entities ->setModel(new Collection()) ->setModel(new Attribute()) @@ -204,6 +223,14 @@ class Response extends SwooleResponse ->setModel(new Language()) ->setModel(new Currency()) ->setModel(new Phone()) + ->setModel(new Metric()) + ->setModel(new UsageDatabase()) + ->setModel(new UsageCollection()) + ->setModel(new UsageUsers()) + ->setModel(new UsageStorage()) + ->setModel(new UsageBuckets()) + ->setModel(new UsageFunctions()) + ->setModel(new UsageProject()) // Verification // Recovery // Tests (keep last) diff --git a/src/Appwrite/Utopia/Response/Model/Metric.php b/src/Appwrite/Utopia/Response/Model/Metric.php new file mode 100644 index 0000000000..26038af081 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/Metric.php @@ -0,0 +1,47 @@ +addRule('value', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The value of this metric at the timestamp.', + 'default' => -1, + 'example' => 1, + ]) + ->addRule('timestamp', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The UNIX timestamp at which this metric was aggregated.', + 'default' => 0, + 'example' => 1592981250 + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'Metric'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_METRIC; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php new file mode 100644 index 0000000000..8c66ff47c4 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -0,0 +1,77 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('files.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of files in this bucket.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageBuckets'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_BUCKETS; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageCollection.php b/src/Appwrite/Utopia/Response/Model/UsageCollection.php new file mode 100644 index 0000000000..f2815831a2 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageCollection.php @@ -0,0 +1,77 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('documents.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageCollection'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_COLLECTION; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php new file mode 100644 index 0000000000..2ebe0b64c0 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php @@ -0,0 +1,112 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('documents.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of collections.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections delete.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageDatabase'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_DATABASE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageFunctions.php b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php new file mode 100644 index 0000000000..625bdab71c --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php @@ -0,0 +1,63 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('functions.executions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function executions.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions.failures', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution failures.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions.compute', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution duration.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageFunctions'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_FUNCTIONS; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php new file mode 100644 index 0000000000..c44af4cf6d --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -0,0 +1,91 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('requests', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of requests.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('network', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for consumed bandwidth.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function executions.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of documents.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of collections.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of users.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for the occupied storage size (in bytes).', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageProject'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_PROJECT; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageStorage.php b/src/Appwrite/Utopia/Response/Model/UsageStorage.php new file mode 100644 index 0000000000..4462a219a9 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageStorage.php @@ -0,0 +1,56 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for the occupied storage size (in bytes).', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of files.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'StorageUsage'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_STORAGE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsageUsers.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php new file mode 100644 index 0000000000..8813615062 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -0,0 +1,98 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('users.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of users.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('sessions.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('sessions.provider.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('sessions.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsageUsers'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USAGE_USERS; + } +} \ No newline at end of file diff --git a/tests/e2e/Scopes/SideConsole.php b/tests/e2e/Scopes/SideConsole.php new file mode 100644 index 0000000000..ada99a6e6a --- /dev/null +++ b/tests/e2e/Scopes/SideConsole.php @@ -0,0 +1,23 @@ + 'http://localhost', + 'cookie' => 'a_session_console='. $this->getRoot()['session'], + 'x-appwrite-mode' => 'admin' + ]; + } + + /** + * @return string + */ + public function getSide() + { + return 'console'; + } +} diff --git a/tests/e2e/Services/Database/DatabaseConsoleClientTest.php b/tests/e2e/Services/Database/DatabaseConsoleClientTest.php new file mode 100644 index 0000000000..625f26107b --- /dev/null +++ b/tests/e2e/Services/Database/DatabaseConsoleClientTest.php @@ -0,0 +1,125 @@ +client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => 'unique()', + 'name' => 'Movies', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals($movies['headers']['status-code'], 201); + $this->assertEquals($movies['body']['name'], 'Movies'); + + return ['moviesId' => $movies['body']['$id']]; + } + + public function testGetDatabaseUsage() + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 11); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['collections.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + $this->assertIsArray($response['body']['collections.create']); + $this->assertIsArray($response['body']['collections.read']); + $this->assertIsArray($response['body']['collections.update']); + $this->assertIsArray($response['body']['collections.delete']); + } + + + /** + * @depends testCreateCollection + */ + public function testGetCollectionUsage(array $data) + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + $response = $this->client->call(Client::METHOD_GET, '/database/randomCollectionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 404); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 6); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + } +} \ No newline at end of file diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 6c0a500756..4ae81cb03d 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -498,5 +498,13 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals(400, $tooMany['headers']['status-code']); $this->assertEquals('Index limit exceeded', $tooMany['body']['message']); + + $collection = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(204, $collection['headers']['status-code']); } } \ No newline at end of file diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php new file mode 100644 index 0000000000..ee15a81dcb --- /dev/null +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -0,0 +1,91 @@ +client->call(Client::METHOD_POST, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => 'unique()', + 'name' => 'Test', + 'execute' => ['user:'.$this->getUser()['$id']], + 'runtime' => 'php-8.0', + 'vars' => [ + 'funcKey1' => 'funcValue1', + 'funcKey2' => 'funcValue2', + 'funcKey3' => 'funcValue3', + ], + 'events' => [ + 'account.create', + 'account.delete', + ], + 'schedule' => '0 0 1 1 *', + 'timeout' => 10, + ]); + + $this->assertEquals(201, $function['headers']['status-code']); + + return [ + 'functionId' => $function['body']['$id'] + ]; + } + + /** + * @depends testCreateFunction + */ + public function testGetCollectionUsage(array $data) + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '232h' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/functions/randomFunctionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 4); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['functions.executions']); + $this->assertIsArray($response['body']['functions.failures']); + $this->assertIsArray($response['body']['functions.compute']); + } + +} \ No newline at end of file diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 4d83efb416..1842891492 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -215,24 +215,16 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(count($response['body']), 8); $this->assertNotEmpty($response['body']); - $this->assertArrayHasKey('collections', $response['body']); - $this->assertArrayHasKey('documents', $response['body']); - $this->assertArrayHasKey('network', $response['body']); - $this->assertArrayHasKey('requests', $response['body']); - $this->assertArrayHasKey('storage', $response['body']); - $this->assertArrayHasKey('users', $response['body']); - $this->assertIsArray($response['body']['collections']['data']); - $this->assertIsInt($response['body']['collections']['total']); - $this->assertIsArray($response['body']['documents']['data']); - $this->assertIsInt($response['body']['documents']['total']); - $this->assertIsArray($response['body']['network']['data']); - $this->assertIsInt($response['body']['network']['total']); - $this->assertIsArray($response['body']['requests']['data']); - $this->assertIsInt($response['body']['requests']['total']); - $this->assertIsInt($response['body']['storage']['total']); - $this->assertIsArray($response['body']['users']['data']); - $this->assertIsInt($response['body']['users']['total']); + $this->assertEquals('30d', $response['body']['range']); + $this->assertIsArray($response['body']['requests']); + $this->assertIsArray($response['body']['network']); + $this->assertIsArray($response['body']['functions']); + $this->assertIsArray($response['body']['documents']); + $this->assertIsArray($response['body']['collections']); + $this->assertIsArray($response['body']['users']); + $this->assertIsArray($response['body']['storage']); /** * Test for FAILURE diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index b2c9582960..4ec1a6ce9a 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -2,13 +2,91 @@ namespace Tests\E2E\Services\Storage; +use Tests\E2E\Client; use Tests\E2E\Scopes\Scope; -use Tests\E2E\Scopes\ProjectConsole; -use Tests\E2E\Scopes\SideClient; +use Tests\E2E\Scopes\ProjectCustom; +use Tests\E2E\Scopes\SideConsole; class StorageConsoleClientTest extends Scope { + use SideConsole; use StorageBase; - use ProjectConsole; - use SideClient; + use ProjectCustom; + + public function testGetStorageUsage() + { + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 3); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['storage']); + $this->assertIsArray($response['body']['files']); + } + + public function testGetStorageBucketUsage() + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + // TODO: Uncomment once we implement check for missing bucketId in the usage endpoint. + + // $response = $this->client->call(Client::METHOD_GET, '/storage/randomBucketId/usage', array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), [ + // 'range' => '24h' + // ]); + + // $this->assertEquals($response['headers']['status-code'], 404); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 6); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['files.count']); + $this->assertIsArray($response['body']['files.create']); + $this->assertIsArray($response['body']['files.read']); + $this->assertIsArray($response['body']['files.update']); + $this->assertIsArray($response['body']['files.delete']); + } } \ No newline at end of file diff --git a/tests/e2e/Services/Users/UsersConsoleClientTest.php b/tests/e2e/Services/Users/UsersConsoleClientTest.php new file mode 100644 index 0000000000..90a10a0c4f --- /dev/null +++ b/tests/e2e/Services/Users/UsersConsoleClientTest.php @@ -0,0 +1,83 @@ +client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h', + 'provider' => 'email' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h', + 'provider' => 'some-random-provider' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h', + 'provider' => 'email' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 9); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['users.count']); + $this->assertIsArray($response['body']['users.create']); + $this->assertIsArray($response['body']['users.read']); + $this->assertIsArray($response['body']['users.update']); + $this->assertIsArray($response['body']['users.delete']); + $this->assertIsArray($response['body']['sessions.create']); + $this->assertIsArray($response['body']['sessions.provider.create']); + $this->assertIsArray($response['body']['sessions.delete']); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 9); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['users.count']); + $this->assertIsArray($response['body']['users.create']); + $this->assertIsArray($response['body']['users.read']); + $this->assertIsArray($response['body']['users.update']); + $this->assertIsArray($response['body']['users.delete']); + $this->assertIsArray($response['body']['sessions.create']); + $this->assertIsArray($response['body']['sessions.provider.create']); + $this->assertIsArray($response['body']['sessions.delete']); + } +} \ No newline at end of file