CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-02 19:04:49 +12:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feat-database-indexing' into feat-delete-collection-attributes-indexes

Browse source
This commit is contained in:
kodumbeats 2021-09-12 09:07:46 -04:00
parent 8c3ca6808c a1d1bc935b
commit b4ad95747a
45 changed files with 2910 additions and 403 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Dockerfile
View file

@ -225,6 +225,7 @@ RUN mkdir -p /storage/uploads && \
# Executables
RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/usage && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/migrate && \
chmod +x /usr/local/bin/schedule && \

3
app/cli.php
View file

@ -1,6 +1,6 @@
<?php
require_once __DIR__.'/workers.php';
require_once __DIR__.'/init.php';
use Utopia\App;
use Utopia\CLI\CLI;
@ -15,6 +15,7 @@ include 'tasks/migrate.php';
include 'tasks/sdks.php';
include 'tasks/ssl.php';
include 'tasks/vars.php';
include 'tasks/usage.php';
$cli
->task('version')

85
app/config/collections2.php
View file

@ -1728,6 +1728,91 @@ $collections = [
],
],
],
'stats' => [
'$collection' => Database::METADATA,
'$id' => 'stats',
'name' => 'Stats',
'attributes' => [
[
'$id' => 'metric',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'value',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'time',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'period',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 4,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'type',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 1,
'signed' => false,
'required' => true,
'default' => 0, // 0 -> count, 1 -> sum
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => '_key_time',
'type' => Database::INDEX_KEY,
'attributes' => ['time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
[
'$id' => '_key_metric',
'type' => Database::INDEX_KEY,
'attributes' => ['metric'],
'lengths' => [],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_metric_period',
'type' => Database::INDEX_KEY,
'attributes' => ['metric', 'period'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
],
]
];
return $collections;

9
app/config/variables.php
View file

@ -149,6 +149,15 @@ return [
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_USAGE_AGGREGATION_INTERVAL',
'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to mariadb from InfluxDB. The default value is 30 seconds.',
'introduction' => '0.10.0',
'default' => '30',
'required' => false,
'question' => '',
'filter' => ''
]
],
],

154
app/controllers/api/account.php
View file

@ -52,12 +52,14 @@ App::post('/v1/account')
->inject('project')
->inject('dbForInternal')
->inject('audits')
->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits) {
->inject('usage')
->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $project */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$email = \strtolower($email);
if ('console' === $project->getId()) {
@ -120,6 +122,9 @@ App::post('/v1/account')
->setParam('resource', 'user/' . $user->getId())
;
$usage
->setParam('users.create', 1)
;
$response->setStatusCode(Response::STATUS_CODE_CREATED);
$response->dynamic($user, Response::MODEL_USER);
});
@ -147,13 +152,15 @@ App::post('/v1/account/sessions')
->inject('locale')
->inject('geodb')
->inject('audits')
->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits) {
->inject('usage')
->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Locale\Locale $locale */
/** @var MaxMind\Db\Reader $geodb */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$email = \strtolower($email);
$protocol = $request->getProtocol();
@ -227,6 +234,11 @@ App::post('/v1/account/sessions')
->setAttribute('countryName', $countryName)
;
$usage
->setParam('users.update', 1)
->setParam('users.sessions.create', 1)
->setParam('provider', 'email')
;
$response->dynamic($session, Response::MODEL_SESSION);
});
@ -357,7 +369,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
->inject('geodb')
->inject('audits')
->inject('events')
->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events) use ($oauthDefaultSuccess) {
->inject('usage')
->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events, $usage) use ($oauthDefaultSuccess) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $project */
@ -365,6 +378,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
/** @var Utopia\Database\Database $dbForInternal */
/** @var MaxMind\Db\Reader $geodb */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$protocol = $request->getProtocol();
$callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId();
@ -545,6 +559,11 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
$events->setParam('eventData', $response->output($session, Response::MODEL_SESSION));
$usage
->setParam('users.sessions.create', 1)
->setParam('projectId', $project->getId())
->setParam('provider', 'oauth2-'.$provider)
;
if (!Config::getParam('domainVerification')) {
$response
->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)]))
@ -595,7 +614,8 @@ App::post('/v1/account/sessions/anonymous')
->inject('dbForInternal')
->inject('geodb')
->inject('audits')
->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits) {
->inject('usage')
->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Locale\Locale $locale */
@ -604,6 +624,7 @@ App::post('/v1/account/sessions/anonymous')
/** @var Utopia\Database\Database $dbForInternal */
/** @var MaxMind\Db\Reader $geodb */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$protocol = $request->getProtocol();
@ -686,6 +707,11 @@ App::post('/v1/account/sessions/anonymous')
->setParam('resource', 'user/' . $user->getId())
;
$usage
->setParam('users.sessions.create', 1)
->setParam('provider', 'anonymous')
;
if (!Config::getParam('domainVerification')) {
$response
->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)]))
@ -771,10 +797,15 @@ App::get('/v1/account')
->label('sdk.response.model', Response::MODEL_USER)
->inject('response')
->inject('user')
->action(function ($response, $user) {
->inject('usage')
->action(function ($response, $user, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Appwrite\Stats\Stats $usage */
$usage
->setParam('users.read', 1)
;
$response->dynamic($user, Response::MODEL_USER);
});
@ -791,12 +822,17 @@ App::get('/v1/account/prefs')
->label('sdk.response.model', Response::MODEL_PREFERENCES)
->inject('response')
->inject('user')
->action(function ($response, $user) {
->inject('usage')
->action(function ($response, $user, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Appwrite\Stats\Stats $usage */
$prefs = $user->getAttribute('prefs', new \stdClass());
$usage
->setParam('users.read', 1)
;
$response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES);
});
@ -814,10 +850,12 @@ App::get('/v1/account/sessions')
->inject('response')
->inject('user')
->inject('locale')
->action(function ($response, $user, $locale) {
->inject('usage')
->action(function ($response, $user, $locale, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Utopia\Locale\Locale $locale */
/** @var Appwrite\Stats\Stats $usage */
$sessions = $user->getAttribute('sessions', []);
$current = Auth::sessionVerify($sessions, Auth::$secret);
@ -831,6 +869,9 @@ App::get('/v1/account/sessions')
$sessions[$key] = $session;
}
$usage
->setParam('users.read', 1)
;
$response->dynamic(new Document([
'sessions' => $sessions,
'sum' => count($sessions),
@ -853,13 +894,15 @@ App::get('/v1/account/logs')
->inject('locale')
->inject('geodb')
->inject('dbForInternal')
->action(function ($response, $user, $locale, $geodb, $dbForInternal) {
->inject('usage')
->action(function ($response, $user, $locale, $geodb, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $project */
/** @var Utopia\Database\Document $user */
/** @var Utopia\Locale\Locale $locale */
/** @var MaxMind\Db\Reader $geodb */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$audit = new Audit($dbForInternal);
@ -906,6 +949,9 @@ App::get('/v1/account/logs')
}
$usage
->setParam('users.read', 1)
;
$response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST);
});
@ -925,11 +971,13 @@ App::get('/v1/account/sessions/:sessionId')
->inject('user')
->inject('locale')
->inject('dbForInternal')
->action(function ($sessionId, $response, $user, $locale, $dbForInternal) {
->inject('usage')
->action(function ($sessionId, $response, $user, $locale, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Utopia\Locale\Locale $locale */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$sessions = $user->getAttribute('sessions', []);
$sessionId = ($sessionId === 'current')
@ -948,6 +996,10 @@ App::get('/v1/account/sessions/:sessionId')
->setAttribute('countryName', $countryName)
;
$usage
->setParam('users.read', 1)
;
return $response->dynamic($session, Response::MODEL_SESSION);
}
}
@ -972,11 +1024,13 @@ App::patch('/v1/account/name')
->inject('user')
->inject('dbForInternal')
->inject('audits')
->action(function ($name, $response, $user, $dbForInternal, $audits) {
->inject('usage')
->action(function ($name, $response, $user, $dbForInternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('name', $name));
@ -986,6 +1040,10 @@ App::patch('/v1/account/name')
->setParam('resource', 'user/' . $user->getId())
;
$usage
->setParam('users.update', 1)
;
$response->dynamic($user, Response::MODEL_USER);
});
@ -1007,11 +1065,13 @@ App::patch('/v1/account/password')
->inject('user')
->inject('dbForInternal')
->inject('audits')
->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits) {
->inject('usage')
->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
// Check old password only if its an existing user.
if ($user->getAttribute('passwordUpdate') !== 0 && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'))) { // Double check user password
@ -1029,6 +1089,9 @@ App::patch('/v1/account/password')
->setParam('resource', 'user/' . $user->getId())
;
$usage
->setParam('users.update', 1)
;
$response->dynamic($user, Response::MODEL_USER);
});
@ -1050,11 +1113,13 @@ App::patch('/v1/account/email')
->inject('user')
->inject('dbForInternal')
->inject('audits')
->action(function ($email, $password, $response, $user, $dbForInternal, $audits) {
->inject('usage')
->action(function ($email, $password, $response, $user, $dbForInternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$isAnonymousUser = is_null($user->getAttribute('email')) && is_null($user->getAttribute('password')); // Check if request is from an anonymous account for converting
@ -1082,6 +1147,9 @@ App::patch('/v1/account/email')
->setParam('resource', 'user/' . $user->getId())
;
$usage
->setParam('users.update', 1)
;
$response->dynamic($user, Response::MODEL_USER);
});
@ -1102,11 +1170,13 @@ App::patch('/v1/account/prefs')
->inject('user')
->inject('dbForInternal')
->inject('audits')
->action(function ($prefs, $response, $user, $dbForInternal, $audits) {
->inject('usage')
->action(function ($prefs, $response, $user, $dbForInternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs));
@ -1115,6 +1185,9 @@ App::patch('/v1/account/prefs')
->setParam('resource', 'user/' . $user->getId())
;
$usage
->setParam('users.update', 1)
;
$response->dynamic($user, Response::MODEL_USER);
});
@ -1135,13 +1208,15 @@ App::delete('/v1/account')
->inject('dbForInternal')
->inject('audits')
->inject('events')
->action(function ($request, $response, $user, $dbForInternal, $audits, $events) {
->inject('usage')
->action(function ($request, $response, $user, $dbForInternal, $audits, $events, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Stats\Stats $usage */
$protocol = $request->getProtocol();
$user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', false));
@ -1171,6 +1246,9 @@ App::delete('/v1/account')
;
}
$usage
->setParam('users.delete', 1)
;
$response
->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null)
->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite'))
@ -1198,7 +1276,8 @@ App::delete('/v1/account/sessions/:sessionId')
->inject('locale')
->inject('audits')
->inject('events')
->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events) {
->inject('usage')
->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
@ -1206,6 +1285,7 @@ App::delete('/v1/account/sessions/:sessionId')
/** @var Utopia\Locale\Locale $locale */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Stats\Stats $usage */
$protocol = $request->getProtocol();
$sessionId = ($sessionId === 'current')
@ -1252,6 +1332,10 @@ App::delete('/v1/account/sessions/:sessionId')
->setParam('eventData', $response->output($session, Response::MODEL_SESSION))
;
$usage
->setParam('users.sessions.delete', 1)
->setParam('users.update', 1)
;
return $response->noContent();
}
}
@ -1278,7 +1362,8 @@ App::delete('/v1/account/sessions')
->inject('locale')
->inject('audits')
->inject('events')
->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events) {
->inject('usage')
->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
@ -1286,6 +1371,7 @@ App::delete('/v1/account/sessions')
/** @var Utopia\Locale\Locale $locale */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Stats\Stats $usage */
$protocol = $request->getProtocol();
$sessions = $user->getAttribute('sessions', []);
@ -1321,13 +1407,19 @@ App::delete('/v1/account/sessions')
$dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('sessions', []));
$numOfSessions = count($sessions);
$events
->setParam('eventData', $response->output(new Document([
'sessions' => $sessions,
'sum' => count($sessions),
'sum' => $numOfSessions,
]), Response::MODEL_SESSION_LIST))
;
$usage
->setParam('users.sessions.delete', $numOfSessions)
->setParam('users.update', 1)
;
$response->noContent();
});
@ -1355,7 +1447,8 @@ App::post('/v1/account/recovery')
->inject('mails')
->inject('audits')
->inject('events')
->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events) {
->inject('usage')
->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
@ -1364,6 +1457,7 @@ App::post('/v1/account/recovery')
/** @var Appwrite\Event\Event $mails */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Stats\Stats $usage */
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles);
$isAppUser = Auth::isAppUser(Authorization::$roles);
@ -1431,6 +1525,9 @@ App::post('/v1/account/recovery')
->setParam('resource', 'user/' . $profile->getId())
;
$usage
->setParam('users.update', 1)
;
$response->setStatusCode(Response::STATUS_CODE_CREATED);
$response->dynamic($recovery, Response::MODEL_TOKEN);
});
@ -1456,10 +1553,12 @@ App::put('/v1/account/recovery')
->inject('response')
->inject('dbForInternal')
->inject('audits')
->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits) {
->inject('usage')
->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
if ($password !== $passwordAgain) {
throw new Exception('Passwords must match', 400);
@ -1506,6 +1605,9 @@ App::put('/v1/account/recovery')
->setParam('resource', 'user/' . $profile->getId())
;
$usage
->setParam('users.update', 1)
;
$response->dynamic($recovery, Response::MODEL_TOKEN);
});
@ -1533,7 +1635,8 @@ App::post('/v1/account/verification')
->inject('audits')
->inject('events')
->inject('mails')
->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails) {
->inject('usage')
->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $project */
@ -1543,6 +1646,7 @@ App::post('/v1/account/verification')
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $mails */
/** @var Appwrite\Stats\Stats $usage */
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles);
$isAppUser = Auth::isAppUser(Authorization::$roles);
@ -1600,6 +1704,9 @@ App::post('/v1/account/verification')
->setParam('resource', 'user/' . $user->getId())
;
$usage
->setParam('users.update', 1)
;
$response->setStatusCode(Response::STATUS_CODE_CREATED);
$response->dynamic($verification, Response::MODEL_TOKEN);
});
@ -1624,11 +1731,13 @@ App::put('/v1/account/verification')
->inject('user')
->inject('dbForInternal')
->inject('audits')
->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits) {
->inject('usage')
->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $user */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$profile = $dbForInternal->getDocument('users', $userId);
@ -1666,5 +1775,8 @@ App::put('/v1/account/verification')
->setParam('resource', 'user/' . $user->getId())
;
$usage
->setParam('users.update', 1)
;
$response->dynamic($verification, Response::MODEL_TOKEN);
});

339
app/controllers/api/database.php
View file

@ -28,13 +28,15 @@ use Utopia\Database\Exception\Structure as StructureException;
use Appwrite\Utopia\Response;
use Appwrite\Database\Validator\CustomId;
use DeviceDetector\DeviceDetector;
use Utopia\Database\Validator\Authorization;
$attributesCallback = function ($collectionId, $attribute, $response, $dbForInternal, $database, $audits) {
$attributesCallback = function ($collectionId, $attribute, $response, $dbForInternal, $database, $audits, $usage) {
/** @var Utopia\Database\Document $attribute*/
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$attributeId = $attribute->getId();
$type = $attribute->getAttribute('type', '');
@ -99,6 +101,8 @@ $attributesCallback = function ($collectionId, $attribute, $response, $dbForInte
->setParam('document', $attribute)
;
$usage->setParam('database.collections.update', 1);
$audits
->setParam('event', 'database.attributes.create')
->setParam('resource', 'collection/'.$collection->getId())
@ -130,10 +134,12 @@ App::post('/v1/database/collections')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('audits')
->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $dbForExternal, $audits) {
->inject('usage')
->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $dbForExternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$collectionId = $collectionId == 'unique()' ? $dbForExternal->getId() : $collectionId;
@ -160,6 +166,8 @@ App::post('/v1/database/collections')
->setParam('data', $collection->getArrayCopy())
;
$usage->setParam('database.collections.create', 1);
$response->setStatusCode(Response::STATUS_CODE_CREATED);
$response->dynamic($collection, Response::MODEL_COLLECTION);
});
@ -182,7 +190,8 @@ App::get('/v1/database/collections')
->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true)
->inject('response')
->inject('dbForInternal')
->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) {
->inject('usage')
->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
@ -200,6 +209,8 @@ App::get('/v1/database/collections')
}
}
$usage->setParam('database.collections.read', 1);
$response->dynamic(new Document([
'collections' => $dbForInternal->find('collections', $queries, $limit, $offset, [], [$orderType], $afterCollection ?? null),
'sum' => $dbForInternal->count('collections', $queries, APP_LIMIT_COUNT),
@ -220,7 +231,8 @@ App::get('/v1/database/collections/:collectionId')
->param('collectionId', '', new UID(), 'Collection unique ID.')
->inject('response')
->inject('dbForInternal')
->action(function ($collectionId, $response, $dbForInternal) {
->inject('usage')
->action(function ($collectionId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
@ -230,9 +242,191 @@ App::get('/v1/database/collections/:collectionId')
throw new Exception('Collection not found', 404);
}
$usage->setParam('database.collections.read', 1);
$response->dynamic($collection, Response::MODEL_COLLECTION);
});
App::get('/v1/database/usage')
->desc('Get usage stats for the database')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'database')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_DATABASE)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForInternal')
->action(function ($range, $response, $dbForInternal) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForConsole */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Registry\Registry $register */
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$period = [
'24h' => [
'period' => '30m',
'limit' => 48,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
'database.documents.count',
'database.collections.count',
'database.collections.create',
'database.collections.read',
'database.collections.update',
'database.collections.delete',
'database.documents.create',
'database.documents.read',
'database.documents.update',
'database.documents.delete'
];
$stats = [];
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$requestDocs = $dbForInternal->find('stats', [
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
new Query('metric', Query::TYPE_EQUAL, [$metric]),
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'documents.count' => $stats["database.documents.count"],
'collections.count' => $stats["database.collections.count"],
'documents.create' => $stats["database.documents.create"],
'documents.read' => $stats["database.documents.read"],
'documents.update' => $stats["database.documents.update"],
'documents.delete' => $stats["database.documents.delete"],
'collections.create' => $stats["database.collections.create"],
'collections.read' => $stats["database.collections.read"],
'collections.update' => $stats["database.collections.update"],
'collections.delete' => $stats["database.collections.delete"],
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_DATABASE);
});
App::get('/v1/database/:collectionId/usage')
->desc('Get usage stats for a collection')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'database')
->label('sdk.method', 'getCollectionUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('collectionId', '', new UID(), 'Collection unique ID.')
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->action(function ($range, $collectionId, $response, $dbForInternal, $dbForExternal) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForConsole */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Registry\Registry $register */
$collection = $dbForExternal->getCollection($collectionId);
if ($collection->isEmpty()) {
throw new Exception('Collection not found', 404);
}
$usage = [];
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$period = [
'24h' => [
'period' => '30m',
'limit' => 48,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
"database.collections.$collectionId.documents.count",
"database.collections.$collectionId.documents.create",
"database.collections.$collectionId.documents.read",
"database.collections.$collectionId.documents.update",
"database.collections.$collectionId.documents.delete",
];
$stats = [];
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$requestDocs = $dbForInternal->find('stats', [
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
new Query('metric', Query::TYPE_EQUAL, [$metric]),
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'documents.count' => $stats["database.collections.$collectionId.documents.count"],
'documents.create' => $stats["database.collections.$collectionId.documents.create"],
'documents.read' => $stats["database.collections.$collectionId.documents.read"],
'documents.update' => $stats["database.collections.$collectionId.documents.update"],
'documents.delete' => $stats["database.collections.$collectionId.documents.delete"]
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_COLLECTION);
});
App::get('/v1/database/collections/:collectionId/logs')
->desc('List Collection Logs')
->groups(['api', 'database'])
@ -349,10 +543,12 @@ App::put('/v1/database/collections/:collectionId')
->inject('response')
->inject('dbForInternal')
->inject('audits')
->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $audits) {
->inject('usage')
->action(function ($collectionId, $name, $permission, $read, $write, $response, $dbForInternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$collection = $dbForInternal->getDocument('collections', $collectionId);
@ -380,6 +576,8 @@ App::put('/v1/database/collections/:collectionId')
throw new Exception('Bad structure. '.$exception->getMessage(), 400);
}
$usage->setParam('database.collections.update', 1);
$audits
->setParam('event', 'database.collections.update')
->setParam('resource', 'collection/'.$collection->getId())
@ -407,12 +605,14 @@ App::delete('/v1/database/collections/:collectionId')
->inject('events')
->inject('audits')
->inject('deletes')
->action(function ($collectionId, $response, $dbForInternal, $dbForExternal, $events, $audits, $deletes) {
->inject('usage')
->action(function ($collectionId, $response, $dbForInternal, $dbForExternal, $events, $audits, $deletes, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Database\Database $dbForExternal */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $audits */
$collection = $dbForInternal->getDocument('collections', $collectionId);
@ -431,6 +631,8 @@ App::delete('/v1/database/collections/:collectionId')
->setParam('document', $collection)
;
$usage->setParam('database.collections.delete', 1);
$events
->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION))
;
@ -466,12 +668,13 @@ App::post('/v1/database/collections/:collectionId/attributes/string')
->inject('dbForInternal')
->inject('database')
->inject('audits')
->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) {
->inject('usage')
->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
return $attributesCallback($collectionId, new Document([
'$id' => $attributeId,
@ -480,7 +683,7 @@ App::post('/v1/database/collections/:collectionId/attributes/string')
'required' => $required,
'default' => $default,
'array' => $array,
]), $response, $dbForInternal, $database, $audits);
]), $response, $dbForInternal, $database, $audits, $usage);
});
App::post('/v1/database/collections/:collectionId/attributes/email')
@ -504,11 +707,13 @@ App::post('/v1/database/collections/:collectionId/attributes/email')
->inject('dbForInternal')
->inject('database')
->inject('audits')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) {
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
return $attributesCallback($collectionId, new Document([
'$id' => $attributeId,
@ -518,7 +723,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email')
'default' => $default,
'array' => $array,
'format' => 'email',
]), $response, $dbForInternal, $database, $audits);
]), $response, $dbForInternal, $database, $audits, $usage);
});
App::post('/v1/database/collections/:collectionId/attributes/ip')
@ -542,11 +747,13 @@ App::post('/v1/database/collections/:collectionId/attributes/ip')
->inject('dbForInternal')
->inject('database')
->inject('audits')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) {
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
return $attributesCallback($collectionId, new Document([
'$id' => $attributeId,
@ -556,7 +763,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip')
'default' => $default,
'array' => $array,
'format' => 'ip',
]), $response, $dbForInternal, $database, $audits);
]), $response, $dbForInternal, $database, $audits, $usage);
});
App::post('/v1/database/collections/:collectionId/attributes/url')
@ -580,11 +787,13 @@ App::post('/v1/database/collections/:collectionId/attributes/url')
->inject('dbForInternal')
->inject('database')
->inject('audits')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) {
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
return $attributesCallback($collectionId, new Document([
'$id' => $attributeId,
@ -594,7 +803,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url')
'default' => $default,
'array' => $array,
'format' => 'url',
]), $response, $dbForInternal, $database, $audits);
]), $response, $dbForInternal, $database, $audits, $usage);
});
App::post('/v1/database/collections/:collectionId/attributes/integer')
@ -620,11 +829,13 @@ App::post('/v1/database/collections/:collectionId/attributes/integer')
->inject('dbForInternal')
->inject('database')
->inject('audits')
->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) {
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
return $attributesCallback($collectionId, new Document([
'$id' => $attributeId,
@ -638,7 +849,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer')
'min' => (is_null($min)) ? PHP_INT_MIN : \intval($min),
'max' => (is_null($max)) ? PHP_INT_MAX : \intval($max),
],
]), $response, $dbForInternal, $database, $audits);
]), $response, $dbForInternal, $database, $audits, $usage);
});
App::post('/v1/database/collections/:collectionId/attributes/float')
@ -664,11 +875,13 @@ App::post('/v1/database/collections/:collectionId/attributes/float')
->inject('dbForInternal')
->inject('database')
->inject('audits')
->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) {
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
return $attributesCallback($collectionId, new Document([
'$id' => $attributeId,
@ -682,7 +895,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float')
'min' => (is_null($min)) ? PHP_FLOAT_MIN : \floatval($min),
'max' => (is_null($max)) ? PHP_FLOAT_MAX : \floatval($max),
],
]), $response, $dbForInternal, $database, $audits);
]), $response, $dbForInternal, $database, $audits, $usage);
});
App::post('/v1/database/collections/:collectionId/attributes/boolean')
@ -706,11 +919,13 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean')
->inject('dbForInternal')
->inject('database')
->inject('audits')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits) use ($attributesCallback) {
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) use ($attributesCallback) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
return $attributesCallback($collectionId, new Document([
'$id' => $attributeId,
@ -719,7 +934,7 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean')
'required' => $required,
'default' => $default,
'array' => $array,
]), $response, $dbForInternal, $database, $audits);
]), $response, $dbForInternal, $database, $audits, $usage);
});
App::get('/v1/database/collections/:collectionId/attributes')
@ -736,7 +951,8 @@ App::get('/v1/database/collections/:collectionId/attributes')
->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).')
->inject('response')
->inject('dbForInternal')
->action(function ($collectionId, $response, $dbForInternal) {
->inject('usage')
->action(function ($collectionId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
@ -754,6 +970,8 @@ App::get('/v1/database/collections/:collectionId/attributes')
])]);
}, $attributes);
$usage->setParam('database.collections.read', 1);
$response->dynamic(new Document([
'sum' => \count($attributes),
'attributes' => $attributes
@ -775,7 +993,8 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId')
->param('attributeId', '', new Key(), 'Attribute ID.')
->inject('response')
->inject('dbForInternal')
->action(function ($collectionId, $attributeId, $response, $dbForInternal) {
->inject('usage')
->action(function ($collectionId, $attributeId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
@ -797,6 +1016,8 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId')
$attribute = new Document([\array_merge($attributes[$attributeIndex], [
'collectionId' => $collectionId,
])]);
$usage->setParam('database.collections.read', 1);
$response->dynamic($attribute, Response::MODEL_ATTRIBUTE);
});
@ -819,12 +1040,14 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId')
->inject('database')
->inject('events')
->inject('audits')
->action(function ($collectionId, $attributeId, $response, $dbForInternal, $database, $events, $audits) {
->inject('usage')
->action(function ($collectionId, $attributeId, $response, $dbForInternal, $database, $events, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$collection = $dbForInternal->getDocument('collections', $collectionId);
@ -847,6 +1070,8 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId')
->setParam('document', $attribute)
;
$usage->setParam('database.collections.update', 1);
$events
->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE))
;
@ -881,11 +1106,13 @@ App::post('/v1/database/collections/:collectionId/indexes')
->inject('dbForInternal')
->inject('database')
->inject('audits')
->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForInternal, $database, $audits) {
->inject('usage')
->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForInternal, $database, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $audits */
$collection = $dbForInternal->getDocument('collections', $collectionId);
@ -950,6 +1177,8 @@ App::post('/v1/database/collections/:collectionId/indexes')
->setParam('document', $index)
;
$usage->setParam('database.collections.update', 1);
$audits
->setParam('event', 'database.indexes.create')
->setParam('resource', 'collection/'.$collection->getId())
@ -974,7 +1203,8 @@ App::get('/v1/database/collections/:collectionId/indexes')
->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).')
->inject('response')
->inject('dbForInternal')
->action(function ($collectionId, $response, $dbForInternal) {
->inject('usage')
->action(function ($collectionId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
@ -992,6 +1222,8 @@ App::get('/v1/database/collections/:collectionId/indexes')
])]);
}, $indexes);
$usage->setParam('database.collections.read', 1);
$response->dynamic(new Document([
'sum' => \count($indexes),
'attributes' => $indexes,
@ -1013,7 +1245,8 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId')
->param('indexId', null, new Key(), 'Index ID.')
->inject('response')
->inject('dbForInternal')
->action(function ($collectionId, $indexId, $response, $dbForInternal) {
->inject('usage')
->action(function ($collectionId, $indexId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
@ -1035,6 +1268,8 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId')
$index = new Document([\array_merge($indexes[$indexIndex], [
'collectionId' => $collectionId,
])]);
$usage->setParam('database.collections.read', 1);
$response->dynamic($index, Response::MODEL_INDEX);
});
@ -1057,12 +1292,14 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId')
->inject('database')
->inject('events')
->inject('audits')
->action(function ($collectionId, $indexId, $response, $dbForInternal, $database, $events, $audits) {
->inject('usage')
->action(function ($collectionId, $indexId, $response, $dbForInternal, $database, $events, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$collection = $dbForInternal->getDocument('collections', $collectionId);
@ -1085,6 +1322,8 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId')
->setParam('document', $index)
;
$usage->setParam('database.collections.update', 1);
$events
->setParam('payload', $response->output($index, Response::MODEL_INDEX))
;
@ -1120,12 +1359,14 @@ App::post('/v1/database/collections/:collectionId/documents')
->inject('dbForExternal')
->inject('user')
->inject('audits')
->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $user, $audits) {
->inject('usage')
->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $user, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Database\Database $dbForExternal */
/** @var Utopia\Database\Document $user */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array
@ -1158,6 +1399,11 @@ App::post('/v1/database/collections/:collectionId/documents')
throw new Exception('Document already exists', 409);
}
$usage
->setParam('database.documents.create', 1)
->setParam('collectionId', $collectionId)
;
$audits
->setParam('event', 'database.documents.create')
->setParam('resource', 'document/'.$document->getId())
@ -1189,10 +1435,12 @@ App::get('/v1/database/collections/:collectionId/documents')
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForInternal, $dbForExternal) {
->inject('usage')
->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForInternal, $dbForExternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Database\Database $dbForExternal */
/** @var Appwrite\Stats\Stats $usage */
$collection = $dbForInternal->getDocument('collections', $collectionId);
@ -1219,6 +1467,11 @@ App::get('/v1/database/collections/:collectionId/documents')
}
}
$usage
->setParam('database.documents.read', 1)
->setParam('collectionId', $collectionId)
;
$response->dynamic(new Document([
'sum' => $dbForExternal->count($collectionId, $queries, APP_LIMIT_COUNT),
'documents' => $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null),
@ -1241,7 +1494,8 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId')
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal) {
->inject('usage')
->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $$dbForInternal */
/** @var Utopia\Database\Database $dbForExternal */
@ -1258,6 +1512,11 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId')
throw new Exception('No document found', 404);
}
$usage
->setParam('database.documents.read', 1)
->setParam('collectionId', $collectionId)
;
$response->dynamic($document, Response::MODEL_DOCUMENT);
});
@ -1282,11 +1541,13 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('audits')
->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $audits) {
->inject('usage')
->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForInternal, $dbForExternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Database\Database $dbForExternal */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$collection = $dbForInternal->getDocument('collections', $collectionId);
@ -1329,6 +1590,11 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId')
catch (StructureException $exception) {
throw new Exception($exception->getMessage(), 400);
}
$usage
->setParam('database.documents.update', 1)
->setParam('collectionId', $collectionId)
;
$audits
->setParam('event', 'database.documents.update')
@ -1357,11 +1623,13 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId')
->inject('dbForExternal')
->inject('events')
->inject('audits')
->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal, $events, $audits) {
->inject('usage')
->action(function ($collectionId, $documentId, $response, $dbForInternal, $dbForExternal, $events, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForExternal */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
$collection = $dbForInternal->getDocument('collections', $collectionId);
@ -1377,6 +1645,11 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId')
$dbForExternal->deleteDocument($collectionId, $documentId);
$usage
->setParam('database.documents.delete', 1)
->setParam('collectionId', $collectionId)
;
$events
->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT))
;

122
app/controllers/api/functions.php
View file

@ -146,6 +146,9 @@ App::get('/v1/functions/:functionId/usage')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'functions')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
->param('functionId', '', new UID(), 'Function unique ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
->inject('response')
@ -164,99 +167,62 @@ App::get('/v1/functions/:functionId/usage')
throw new Exception('Function not found', 404);
}
$usage = [];
if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$period = [
'24h' => [
'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')),
'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')),
'group' => '30m',
'period' => '30m',
'limit' => 48,
],
'7d' => [
'start' => DateTime::createFromFormat('U', \strtotime('-7 days')),
'end' => DateTime::createFromFormat('U', \strtotime('now')),
'group' => '1d',
'period' => '1d',
'limit' => 7,
],
'30d' => [
'start' => DateTime::createFromFormat('U', \strtotime('-30 days')),
'end' => DateTime::createFromFormat('U', \strtotime('now')),
'group' => '1d',
'period' => '1d',
'limit' => 30,
],
'90d' => [
'start' => DateTime::createFromFormat('U', \strtotime('-90 days')),
'end' => DateTime::createFromFormat('U', \strtotime('now')),
'group' => '1d',
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
"functions.$functionId.executions",
"functions.$functionId.failures",
"functions.$functionId.compute"
];
$stats = [];
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$requestDocs = $dbForInternal->find('stats', [
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
new Query('metric', Query::TYPE_EQUAL, [$metric]),
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
$client = $register->get('influxdb');
$executions = [];
$failures = [];
$compute = [];
if ($client) {
$start = $period[$range]['start']->format(DateTime::RFC3339);
$end = $period[$range]['end']->format(DateTime::RFC3339);
$database = $client->selectDB('telegraf');
// Executions
$result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)');
$points = $result->getPoints();
foreach ($points as $point) {
$executions[] = [
'value' => (!empty($point['value'])) ? $point['value'] : 0,
'date' => \strtotime($point['time']),
];
}
// Failures
$result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' AND "functionStatus"=\'failed\' GROUP BY time('.$period[$range]['group'].') FILL(null)');
$points = $result->getPoints();
foreach ($points as $point) {
$failures[] = [
'value' => (!empty($point['value'])) ? $point['value'] : 0,
'date' => \strtotime($point['time']),
];
}
// Compute
$result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_time" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)');
$points = $result->getPoints();
foreach ($points as $point) {
$compute[] = [
'value' => round((!empty($point['value'])) ? $point['value'] / 1000 : 0, 2), // minutes
'date' => \strtotime($point['time']),
];
}
}
$response->json([
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'executions' => [
'data' => $executions,
'total' => \array_sum(\array_map(function ($item) {
return $item['value'];
}, $executions)),
],
'failures' => [
'data' => $failures,
'total' => \array_sum(\array_map(function ($item) {
return $item['value'];
}, $failures)),
],
'compute' => [
'data' => $compute,
'total' => \array_sum(\array_map(function ($item) {
return $item['value'];
}, $compute)),
],
'functions.executions' => $stats["functions.$functionId.executions"],
'functions.failures' => $stats["functions.$functionId.failures"],
'functions.compute' => $stats["functions.$functionId.compute"]
]);
} else {
$response->json([]);
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS);
});
App::put('/v1/functions/:functionId')

217
app/controllers/api/projects.php
View file

@ -6,10 +6,14 @@ use Appwrite\Network\Validator\CNAME;
use Appwrite\Network\Validator\Domain as DomainValidator;
use Appwrite\Network\Validator\URL;
use Appwrite\Utopia\Response;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\App;
use Utopia\Audit\Audit;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Domains\Domain;
use Utopia\Exception;
@ -19,13 +23,11 @@ use Utopia\Validator\Integer;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Utopia\Audit\Audit;
use Utopia\Abuse\Adapters\TimeLimit;
App::init(function ($project) {
/** @var Utopia\Database\Document $project */
if($project->getId() !== 'console') {
if ($project->getId() !== 'console') {
throw new Exception('Access to this API is forbidden.', 401);
}
}, ['project'], 'projects');
@ -69,7 +71,7 @@ App::post('/v1/projects')
if ($team->isEmpty()) {
throw new Exception('Team not found', 404);
}
$auth = Config::getParam('auth', []);
$auths = ['limit' => 0];
foreach ($auth as $index => $method) {
@ -215,22 +217,25 @@ App::get('/v1/projects/:projectId')
});
App::get('/v1/projects/:projectId/usage')
->desc('Get Project')
->desc('Get usage stats for a project')
->groups(['api', 'projects'])
->label('scope', 'projects.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'projects')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
->param('projectId', '', new UID(), 'Project unique ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForConsole')
->inject('projectDB')
->inject('dbForInternal')
->inject('register')
->action(function ($projectId, $range, $response, $dbForConsole, $projectDB, $register) {
->action(function ($projectId, $range, $response, $dbForConsole, $dbForInternal, $register) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForConsole */
/** @var Appwrite\Database\Database $projectDB */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Registry\Registry $register */
$project = $dbForConsole->getDocument('projects', $projectId);
@ -239,172 +244,72 @@ App::get('/v1/projects/:projectId/usage')
throw new Exception('Project not found', 404);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$period = [
'24h' => [
'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')),
'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')),
'group' => '30m',
'period' => '30m',
'limit' => 48,
],
'7d' => [
'start' => DateTime::createFromFormat('U', \strtotime('-7 days')),
'end' => DateTime::createFromFormat('U', \strtotime('now')),
'group' => '1d',
'period' => '1d',
'limit' => 7,
],
'30d' => [
'start' => DateTime::createFromFormat('U', \strtotime('-30 days')),
'end' => DateTime::createFromFormat('U', \strtotime('now')),
'group' => '1d',
'period' => '1d',
'limit' => 30,
],
'90d' => [
'start' => DateTime::createFromFormat('U', \strtotime('-90 days')),
'end' => DateTime::createFromFormat('U', \strtotime('now')),
'group' => '1d',
'period' => '1d',
'limit' => 90,
],
];
$client = $register->get('influxdb');
$dbForInternal->setNamespace('project_' . $projectId . '_internal');
$requests = [];
$network = [];
$functions = [];
$metrics = [
'requests',
'network',
'executions',
'users.count',
'database.documents.count',
'database.collections.count',
'storage.total'
];
if ($client) {
$start = $period[$range]['start']->format(DateTime::RFC3339);
$end = $period[$range]['end']->format(DateTime::RFC3339);
$database = $client->selectDB('telegraf');
$stats = [];
// Requests
$result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)');
$points = $result->getPoints();
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$requestDocs = $dbForInternal->find('stats', [
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
new Query('metric', Query::TYPE_EQUAL, [$metric]),
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
foreach ($points as $point) {
$requests[] = [
'value' => (!empty($point['value'])) ? $point['value'] : 0,
'date' => \strtotime($point['time']),
];
}
// Network
$result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)');
$points = $result->getPoints();
foreach ($points as $point) {
$network[] = [
'value' => (!empty($point['value'])) ? $point['value'] : 0,
'date' => \strtotime($point['time']),
];
}
// Functions
$result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)');
$points = $result->getPoints();
foreach ($points as $point) {
$functions[] = [
'value' => (!empty($point['value'])) ? $point['value'] : 0,
'date' => \strtotime($point['time']),
];
}
}
} else {
$requests = [];
$network = [];
$functions = [];
}
// Users
$projectDB->getCollection([
'limit' => 0,
'offset' => 0,
'filters' => [
'$collection=users',
],
]);
$usersTotal = $projectDB->getSum();
// Documents
$collections = $projectDB->getCollection([
'limit' => 100,
'offset' => 0,
'filters' => [
'$collection=collections',
],
]);
$collectionsTotal = $projectDB->getSum();
$documents = [];
foreach ($collections as $collection) {
$result = $projectDB->getCollection([
'limit' => 0,
'offset' => 0,
'filters' => [
'$collection=' . $collection['$id'],
],
$usage = new Document([
'range' => $range,
'requests' => $stats['requests'],
'network' => $stats['network'],
'functions' => $stats['executions'],
'documents' => $stats['database.documents.count'],
'collections' => $stats['database.collections.count'],
'users' => $stats['users.count'],
'storage' => $stats['storage.total']
]);
$documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()];
}
$response->json([
'range' => $range,
'requests' => [
'data' => $requests,
'total' => \array_sum(\array_map(function ($item) {
return $item['value'];
}, $requests)),
],
'network' => [
'data' => \array_map(function ($value) {return ['value' => \round($value['value'] / 1000000, 2), 'date' => $value['date']];}, $network), // convert bytes to mb
'total' => \array_sum(\array_map(function ($item) {
return $item['value'];
}, $network)),
],
'functions' => [
'data' => $functions,
'total' => \array_sum(\array_map(function ($item) {
return $item['value'];
}, $functions)),
],
'collections' => [
'data' => $collections,
'total' => $collectionsTotal,
],
'documents' => [
'data' => $documents,
'total' => \array_sum(\array_map(function ($item) {
return $item['total'];
}, $documents)),
],
'users' => [
'data' => [],
'total' => $usersTotal,
],
'storage' => [
'total' => $projectDB->getCount(
[
'attribute' => 'sizeOriginal',
'filters' => [
'$collection=files',
],
]
) +
$projectDB->getCount(
[
'attribute' => 'size',
'filters' => [
'$collection=tags',
],
]
),
],
]);
$response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
});
App::patch('/v1/projects/:projectId')
@ -467,7 +372,7 @@ App::patch('/v1/projects/:projectId/service')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_PROJECT)
->param('projectId', '', new UID(), 'Project unique ID.')
->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function($element) {return $element['optional'];})), true), 'Service name.')
->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function ($element) {return $element['optional'];})), true), 'Service name.')
->param('status', null, new Boolean(), 'Service status.')
->inject('response')
->inject('dbForConsole')

218
app/controllers/api/storage.php
View file

@ -10,6 +10,7 @@ use Utopia\Validator\HexColor;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Filesystem;
use Appwrite\ClamAV\Network;
use Utopia\Database\Validator\Authorization;
use Appwrite\Database\Validator\CustomId;
use Utopia\Database\Document;
use Utopia\Database\Validator\UID;
@ -22,6 +23,7 @@ use Utopia\Image\Image;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\Utopia\Response;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Query;
App::post('/v1/storage/files')
@ -54,7 +56,7 @@ App::post('/v1/storage/files')
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Database\Document $user */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $usage */
/** @var Appwrite\Stats\Stats $usage */
$file = $request->getFiles('file');
@ -150,6 +152,8 @@ App::post('/v1/storage/files')
$usage
->setParam('storage', $sizeActual)
->setParam('storage.files.create', 1)
->setParam('bucketId', 'default')
;
$response->setStatusCode(Response::STATUS_CODE_CREATED);
@ -175,9 +179,11 @@ App::get('/v1/storage/files')
->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true)
->inject('response')
->inject('dbForInternal')
->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) {
->inject('usage')
->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, $search)] : [];
@ -189,6 +195,11 @@ App::get('/v1/storage/files')
}
}
$usage
->setParam('storage.files.read', 1)
->setParam('bucketId', 'default')
;
$response->dynamic(new Document([
'files' => $dbForInternal->find('files', $queries, $limit, $offset, [], [$orderType], $afterFile ?? null),
'sum' => $dbForInternal->count('files', $queries, APP_LIMIT_COUNT),
@ -209,16 +220,21 @@ App::get('/v1/storage/files/:fileId')
->param('fileId', '', new UID(), 'File unique ID.')
->inject('response')
->inject('dbForInternal')
->action(function ($fileId, $response, $dbForInternal) {
->inject('usage')
->action(function ($fileId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$file = $dbForInternal->getDocument('files', $fileId);
if (empty($file->getId())) {
throw new Exception('File not found', 404);
}
$usage
->setParam('storage.files.read', 1)
->setParam('bucketId', 'default')
;
$response->dynamic($file, Response::MODEL_FILE);
});
@ -249,11 +265,13 @@ App::get('/v1/storage/files/:fileId/preview')
->inject('response')
->inject('project')
->inject('dbForInternal')
->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal) {
->inject('usage')
->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $project */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $stats */
$storage = 'files';
@ -366,6 +384,11 @@ App::get('/v1/storage/files/:fileId/preview')
$cache->save($key, $data);
$usage
->setParam('storage.files.read', 1)
->setParam('bucketId', 'default')
;
$response
->setContentType($outputs[$output])
->addHeader('Expires', $date)
@ -390,9 +413,11 @@ App::get('/v1/storage/files/:fileId/download')
->param('fileId', '', new UID(), 'File unique ID.')
->inject('response')
->inject('dbForInternal')
->action(function ($fileId, $response, $dbForInternal) {
->inject('usage')
->action(function ($fileId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$file = $dbForInternal->getDocument('files', $fileId);
@ -424,6 +449,11 @@ App::get('/v1/storage/files/:fileId/download')
$source = $compressor->decompress($source);
$usage
->setParam('storage.files.read', 1)
->setParam('bucketId', 'default')
;
// Response
$response
->setContentType($file->getAttribute('mimeType'))
@ -448,9 +478,11 @@ App::get('/v1/storage/files/:fileId/view')
->param('fileId', '', new UID(), 'File unique ID.')
->inject('response')
->inject('dbForInternal')
->action(function ($fileId, $response, $dbForInternal) {
->inject('usage')
->action(function ($fileId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$file = $dbForInternal->getDocument('files', $fileId);
$mimes = Config::getParam('storage-mimes');
@ -490,6 +522,11 @@ App::get('/v1/storage/files/:fileId/view')
$output = $compressor->decompress($source);
$fileName = $file->getAttribute('name', '');
$usage
->setParam('storage.files.read', 1)
->setParam('bucketId', 'default')
;
// Response
$response
->setContentType($contentType)
@ -520,7 +557,8 @@ App::put('/v1/storage/files/:fileId')
->inject('response')
->inject('dbForInternal')
->inject('audits')
->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits) {
->inject('usage')
->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $audits */
@ -542,6 +580,11 @@ App::put('/v1/storage/files/:fileId')
->setParam('resource', 'file/'.$file->getId())
;
$usage
->setParam('storage.files.update', 1)
->setParam('bucketId', 'default')
;
$response->dynamic($file, Response::MODEL_FILE);
});
@ -567,7 +610,7 @@ App::delete('/v1/storage/files/:fileId')
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Event\Event $usage */
/** @var Appwrite\Stats\Stats $usage */
$file = $dbForInternal->getDocument('files', $fileId);
@ -590,6 +633,8 @@ App::delete('/v1/storage/files/:fileId')
$usage
->setParam('storage', $file->getAttribute('size', 0) * -1)
->setParam('storage.files.delete', 1)
->setParam('bucketId', 'default')
;
$events
@ -597,4 +642,159 @@ App::delete('/v1/storage/files/:fileId')
;
$response->noContent();
});
App::get('/v1/storage/usage')
->desc('Get usage stats for storage')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_STORAGE)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForInternal')
->action(function ($range, $response, $dbForInternal) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$period = [
'24h' => [
'period' => '30m',
'limit' => 48,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
"storage.total",
"storage.files.count"
];
$stats = [];
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$requestDocs = $dbForInternal->find('stats', [
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
new Query('metric', Query::TYPE_EQUAL, [$metric]),
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'storage' => $stats['storage.total'],
'files' => $stats['storage.files.count']
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
});
App::get('/v1/storage/:bucketId/usage')
->desc('Get usage stats for a storage bucket')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucketUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS)
->param('bucketId', '', new UID(), 'Bucket unique ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForInternal')
->action(function ($bucketId, $range, $response, $dbForInternal) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
// TODO: Check if the storage bucket exists else throw 404
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$period = [
'24h' => [
'period' => '30m',
'limit' => 48,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
"storage.buckets.$bucketId.files.count",
"storage.buckets.$bucketId.files.create",
"storage.buckets.$bucketId.files.read",
"storage.buckets.$bucketId.files.update",
"storage.buckets.$bucketId.files.delete"
];
$stats = [];
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$requestDocs = $dbForInternal->find('stats', [
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
new Query('metric', Query::TYPE_EQUAL, [$metric]),
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'files.count' => $stats["storage.buckets.$bucketId.files.count"],
'files.create' => $stats["storage.buckets.$bucketId.files.create"],
'files.read' => $stats["storage.buckets.$bucketId.files.read"],
'files.update' => $stats["storage.buckets.$bucketId.files.update"],
'files.delete' => $stats["storage.buckets.$bucketId.files.delete"]
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
});

185
app/controllers/api/users.php
View file

@ -17,7 +17,10 @@ use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Validator\UID;
use DeviceDetector\DeviceDetector;
use Appwrite\Database\Validator\CustomId;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
App::post('/v1/users')
->desc('Create User')
@ -37,9 +40,11 @@ App::post('/v1/users')
->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true)
->inject('response')
->inject('dbForInternal')
->action(function ($userId, $email, $password, $name, $response, $dbForInternal) {
->inject('usage')
->action(function ($userId, $email, $password, $name, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$email = \strtolower($email);
@ -66,6 +71,10 @@ App::post('/v1/users')
throw new Exception('Account already exists', 409);
}
$usage
->setParam('users.create', 1)
;
$response->setStatusCode(Response::STATUS_CODE_CREATED);
$response->dynamic($user, Response::MODEL_USER);
});
@ -88,9 +97,11 @@ App::get('/v1/users')
->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true)
->inject('response')
->inject('dbForInternal')
->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) {
->inject('usage')
->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
if (!empty($after)) {
$afterUser = $dbForInternal->getDocument('users', $after);
@ -102,6 +113,10 @@ App::get('/v1/users')
$results = $dbForInternal->find('users', [], $limit, $offset, [], [$orderType], $afterUser ?? null);
$sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT);
$usage
->setParam('users.read', 1)
;
$response->dynamic(new Document([
'users' => $results,
@ -123,9 +138,11 @@ App::get('/v1/users/:userId')
->param('userId', '', new UID(), 'User unique ID.')
->inject('response')
->inject('dbForInternal')
->action(function ($userId, $response, $dbForInternal) {
->inject('usage')
->action(function ($userId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -133,6 +150,9 @@ App::get('/v1/users/:userId')
throw new Exception('User not found', 404);
}
$usage
->setParam('users.read', 1)
;
$response->dynamic($user, Response::MODEL_USER);
});
@ -150,9 +170,11 @@ App::get('/v1/users/:userId/prefs')
->param('userId', '', new UID(), 'User unique ID.')
->inject('response')
->inject('dbForInternal')
->action(function ($userId, $response, $dbForInternal) {
->inject('usage')
->action(function ($userId, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -162,6 +184,9 @@ App::get('/v1/users/:userId/prefs')
$prefs = $user->getAttribute('prefs', new \stdClass());
$usage
->setParam('users.read', 1)
;
$response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES);
});
@ -180,10 +205,12 @@ App::get('/v1/users/:userId/sessions')
->inject('response')
->inject('dbForInternal')
->inject('locale')
->action(function ($userId, $response, $dbForInternal, $locale) {
->inject('usage')
->action(function ($userId, $response, $dbForInternal, $locale, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Locale\Locale $locale */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -203,6 +230,9 @@ App::get('/v1/users/:userId/sessions')
$sessions[$key] = $session;
}
$usage
->setParam('users.read', 1)
;
$response->dynamic(new Document([
'sessions' => $sessions,
'sum' => count($sessions),
@ -225,12 +255,14 @@ App::get('/v1/users/:userId/logs')
->inject('dbForInternal')
->inject('locale')
->inject('geodb')
->action(function ($userId, $response, $dbForInternal, $locale, $geodb) {
->inject('usage')
->action(function ($userId, $response, $dbForInternal, $locale, $geodb, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Document $project */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Locale\Locale $locale */
/** @var MaxMind\Db\Reader $geodb */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -312,6 +344,9 @@ App::get('/v1/users/:userId/logs')
}
}
$usage
->setParam('users.read', 1)
;
$response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST);
});
@ -331,9 +366,11 @@ App::patch('/v1/users/:userId/status')
->param('status', null, new Boolean(true), 'User Status. To activate the user pass `true` and to block the user pass `false`')
->inject('response')
->inject('dbForInternal')
->action(function ($userId, $status, $response, $dbForInternal) {
->inject('usage')
->action(function ($userId, $status, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -343,6 +380,9 @@ App::patch('/v1/users/:userId/status')
$user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', (bool) $status));
$usage
->setParam('users.update', 1)
;
$response->dynamic($user, Response::MODEL_USER);
});
@ -362,9 +402,11 @@ App::patch('/v1/users/:userId/verification')
->param('emailVerification', false, new Boolean(), 'User Email Verification Status.')
->inject('response')
->inject('dbForInternal')
->action(function ($userId, $emailVerification, $response, $dbForInternal) {
->inject('usage')
->action(function ($userId, $emailVerification, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -374,6 +416,9 @@ App::patch('/v1/users/:userId/verification')
$user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('emailVerification', $emailVerification));
$usage
->setParam('users.update', 1)
;
$response->dynamic($user, Response::MODEL_USER);
});
@ -516,9 +561,11 @@ App::patch('/v1/users/:userId/prefs')
->param('prefs', '', new Assoc(), 'Prefs key-value JSON object.')
->inject('response')
->inject('dbForInternal')
->action(function ($userId, $prefs, $response, $dbForInternal) {
->inject('usage')
->action(function ($userId, $prefs, $response, $dbForInternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -528,6 +575,9 @@ App::patch('/v1/users/:userId/prefs')
$user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs));
$usage
->setParam('users.update', 1)
;
$response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES);
});
@ -547,10 +597,12 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
->inject('response')
->inject('dbForInternal')
->inject('events')
->action(function ($userId, $sessionId, $response, $dbForInternal, $events) {
->inject('usage')
->action(function ($userId, $sessionId, $response, $dbForInternal, $events, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -577,6 +629,11 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
}
}
$usage
->setParam('users.update', 1)
->setParam('users.sessions.delete', 1)
;
$response->noContent();
});
@ -595,10 +652,12 @@ App::delete('/v1/users/:userId/sessions')
->inject('response')
->inject('dbForInternal')
->inject('events')
->action(function ($userId, $response, $dbForInternal, $events) {
->inject('usage')
->action(function ($userId, $response, $dbForInternal, $events, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -618,6 +677,10 @@ App::delete('/v1/users/:userId/sessions')
->setParam('eventData', $response->output($user, Response::MODEL_USER))
;
$usage
->setParam('users.update', 1)
->setParam('users.sessions.delete', 1)
;
$response->noContent();
});
@ -637,11 +700,13 @@ App::delete('/v1/users/:userId')
->inject('dbForInternal')
->inject('events')
->inject('deletes')
->action(function ($userId, $response, $dbForInternal, $events, $deletes) {
->inject('usage')
->action(function ($userId, $response, $dbForInternal, $events, $deletes, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $deletes */
/** @var Appwrite\Stats\Stats $usage */
$user = $dbForInternal->getDocument('users', $userId);
@ -652,11 +717,6 @@ App::delete('/v1/users/:userId')
if (!$dbForInternal->deleteDocument('users', $userId)) {
throw new Exception('Failed to remove user from DB', 500);
}
// $dbForInternal->createDocument('users', new Document([
// '$id' => $userId,
// '$read' => ['role:all'],
// ]));
$deletes
->setParam('type', DELETE_TYPE_DOCUMENT)
@ -667,5 +727,96 @@ App::delete('/v1/users/:userId')
->setParam('eventData', $response->output($user, Response::MODEL_USER))
;
$usage
->setParam('users.delete', 1)
;
$response->noContent();
});
App::get('/v1/users/usage')
->desc('Get usage stats for the users API')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'users')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_USERS)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(function($value) { return "oauth-".$value; }, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true)
->inject('response')
->inject('dbForInternal')
->inject('register')
->action(function ($range, $provider, $response, $dbForInternal) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$period = [
'24h' => [
'period' => '30m',
'limit' => 48,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
"users.count",
"users.create",
"users.read",
"users.update",
"users.delete",
"users.sessions.create",
"users.sessions.$provider.create",
"users.sessions.delete"
];
$stats = [];
Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$requestDocs = $dbForInternal->find('stats', [
new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]),
new Query('metric', Query::TYPE_EQUAL, [$metric]),
], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'users.count' => $stats["users.count"],
'users.create' => $stats["users.create"],
'users.read' => $stats["users.read"],
'users.update' => $stats["users.update"],
'users.delete' => $stats["users.delete"],
'sessions.create' => $stats["users.sessions.create"],
'sessions.provider.create' => $stats["users.sessions.$provider.create"],
'sessions.delete' => $stats["users.sessions.delete"]
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_USERS);
});

1
app/controllers/shared/api.php
View file

@ -104,6 +104,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $events, $aud
->setParam('httpRequest', 1)
->setParam('httpUrl', $request->getHostname().$request->getURI())
->setParam('httpMethod', $request->getMethod())
->setParam('httpPath', $route->getPath())
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->setParam('storage', 0)

25
app/init.php
View file

@ -17,6 +17,7 @@ ini_set('display_startup_errors', 1);
ini_set('default_socket_timeout', -1);
error_reporting(E_ALL);
use Appwrite\Extend\PDO;
use Ahc\Jwt\JWT;
use Ahc\Jwt\JWTException;
use Appwrite\Auth\Auth;
@ -88,6 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions';
const DELETE_TYPE_AUDIT = 'audit';
const DELETE_TYPE_ABUSE = 'abuse';
const DELETE_TYPE_CERTIFICATES = 'certificates';
const DELETE_TYPE_USAGE = 'usage';
// Mail Worker Types
const MAIL_TYPE_VERIFICATION = 'verification';
const MAIL_TYPE_RECOVERY = 'recovery';
@ -345,6 +347,29 @@ $register->set('smtp', function () {
$register->set('geodb', function () {
return new Reader(__DIR__.'/db/DBIP/dbip-country-lite-2021-06.mmdb');
});
$register->set('db', function () { // This is usually for our workers or CLI commands scope
$dbHost = App::getEnv('_APP_DB_HOST', '');
$dbUser = App::getEnv('_APP_DB_USER', '');
$dbPass = App::getEnv('_APP_DB_PASS', '');
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
$pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4',
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
));
return $pdo;
});
$register->set('cache', function () { // This is usually for our workers or CLI commands scope
$redis = new Redis();
$redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
return $redis;
});
/*
* Localization

14
app/tasks/maintenance.php
View file

@ -39,17 +39,29 @@ $cli
]);
}
function notifyDeleteUsageStats(int $interval30m, int $interval1d)
{
Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [
'type' => DELETE_TYPE_USAGE_STATS,
'timestamp1d' => time() - $interval1d,
'timestamp30m' => time() - $interval30m,
]);
}
// # of days in seconds (1 day = 86400s)
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
$executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
$auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600');
$abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
$usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours
$usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days
Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention){
Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) {
$time = date('d-m-Y H:i:s', time());
Console::info("[{$time}] Notifying deletes workers every {$interval} seconds");
notifyDeleteExecutionLogs($executionLogsRetention);
notifyDeleteAbuseLogs($abuseLogsRetention);
notifyDeleteAuditLogs($auditLogRetention);
notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d);
}, $interval);
});

586
app/tasks/usage.php Normal file
View file

@ -0,0 +1,586 @@
<?php
global $cli, $register;
require_once __DIR__ . '/../init.php';
use Utopia\App;
use Utopia\Cache\Adapter\Redis;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
/**
* Metrics We collect
*
* General
*
* requests
* network
* executions
*
* Database
*
* database.collections.create
* database.collections.read
* database.collections.update
* database.collections.delete
* database.documents.create
* database.documents.read
* database.documents.update
* database.documents.delete
* database.collections.{collectionId}.documents.create
* database.collections.{collectionId}.documents.read
* database.collections.{collectionId}.documents.update
* database.collections.{collectionId}.documents.delete
*
* Storage
*
* storage.buckets.{bucketId}.files.create
* storage.buckets.{bucketId}.files.read
* storage.buckets.{bucketId}.files.update
* storage.buckets.{bucketId}.files.delete
*
* Users
*
* users.create
* users.read
* users.update
* users.delete
* users.sessions.create
* users.sessions.{provider}.create
* users.sessions.delete
*
* Functions
*
* functions.{functionId}.executions
* functions.{functionId}.failures
* functions.{functionId}.compute
*
* Counters
*
* users.count
* storage.files.count
* database.collections.count
* database.documents.count
* database.collections.{collectionId}.documents.count
*
* Totals
*
* storage.total
*
*/
$cli
->task('usage')
->desc('Schedules syncing data from influxdb to Appwrite console db')
->action(function () use ($register) {
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
$periods = [
[
'key' => '30m',
'startTime' => '-24 hours',
],
[
'key' => '1d',
'startTime' => '-90 days',
],
];
// all the metrics that we are collecting at the moment
$globalMetrics = [
'requests' => [
'table' => 'appwrite_usage_requests_all',
],
'network' => [
'table' => 'appwrite_usage_network_all',
],
'executions' => [
'table' => 'appwrite_usage_executions_all',
],
'database.collections.create' => [
'table' => 'appwrite_usage_database_collections_create',
],
'database.collections.read' => [
'table' => 'appwrite_usage_database_collections_read',
],
'database.collections.update' => [
'table' => 'appwrite_usage_database_collections_update',
],
'database.collections.delete' => [
'table' => 'appwrite_usage_database_collections_delete',
],
'database.documents.create' => [
'table' => 'appwrite_usage_database_documents_create',
],
'database.documents.read' => [
'table' => 'appwrite_usage_database_documents_read',
],
'database.documents.update' => [
'table' => 'appwrite_usage_database_documents_update',
],
'database.documents.delete' => [
'table' => 'appwrite_usage_database_documents_delete',
],
'database.collections.collectionId.documents.create' => [
'table' => 'appwrite_usage_database_documents_create',
'groupBy' => 'collectionId',
],
'database.collections.collectionId.documents.read' => [
'table' => 'appwrite_usage_database_documents_read',
'groupBy' => 'collectionId',
],
'database.collections.collectionId.documents.update' => [
'table' => 'appwrite_usage_database_documents_update',
'groupBy' => 'collectionId',
],
'database.collections.collectionId.documents.delete' => [
'table' => 'appwrite_usage_database_documents_delete',
'groupBy' => 'collectionId',
],
'storage.buckets.bucketId.files.create' => [
'table' => 'appwrite_usage_storage_files_create',
'groupBy' => 'bucketId',
],
'storage.buckets.bucketId.files.read' => [
'table' => 'appwrite_usage_storage_files_read',
'groupBy' => 'bucketId',
],
'storage.buckets.bucketId.files.update' => [
'table' => 'appwrite_usage_storage_files_update',
'groupBy' => 'bucketId',
],
'storage.buckets.bucketId.files.delete' => [
'table' => 'appwrite_usage_storage_files_delete',
'groupBy' => 'bucketId',
],
'users.create' => [
'table' => 'appwrite_usage_users_create',
],
'users.read' => [
'table' => 'appwrite_usage_users_read',
],
'users.update' => [
'table' => 'appwrite_usage_users_update',
],
'users.delete' => [
'table' => 'appwrite_usage_users_delete',
],
'users.sessions.create' => [
'table' => 'appwrite_usage_users_sessions_create',
],
'users.sessions.provider.create' => [
'table' => 'appwrite_usage_users_sessions_create',
'groupBy' => 'provider',
],
'users.sessions.delete' => [
'table' => 'appwrite_usage_users_sessions_delete',
],
'functions.functionId.executions' => [
'table' => 'appwrite_usage_executions_all',
'groupBy' => 'functionId',
],
'functions.functionId.compute' => [
'table' => 'appwrite_usage_executions_time',
'groupBy' => 'functionId',
],
'functions.functionId.failures' => [
'table' => 'appwrite_usage_executions_all',
'groupBy' => 'functionId',
'filters' => [
'functionStatus' => 'failed',
],
],
];
// TODO Maybe move this to the setResource method, and reuse in the http.php file
$attempts = 0;
$max = 10;
$sleep = 1;
do { // connect to db
try {
$attempts++;
$db = $register->get('db');
$redis = $register->get('cache');
break; // leave the do-while if successful
} catch (\Exception $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
}
sleep($sleep);
}
} while ($attempts < $max);
// TODO use inject
$cacheAdapter = new Cache(new Redis($redis));
$dbForProject = new Database(new MariaDB($db), $cacheAdapter);
$dbForConsole = new Database(new MariaDB($db), $cacheAdapter);
$dbForConsole->setNamespace('project_console_internal');
$latestTime = [];
Authorization::disable();
$iterations = 0;
Console::loop(function () use ($interval, $register, $dbForProject, $dbForConsole, $globalMetrics, $periods, &$latestTime, &$iterations) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating usage data every {$interval} seconds");
$loopStart = microtime(true);
/**
* Aggregate InfluxDB every 30 seconds
* @var InfluxDB\Client $client
*/
$client = $register->get('influxdb');
if ($client) {
$attempts = 0;
$max = 10;
$sleep = 1;
$database = $client->selectDB('telegraf');
do { // check if telegraf database is ready
$attempts++;
if(!in_array('telegraf', $client->listDatabases())) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
} else {
break; // leave the do-while if successful
}
} while ($attempts < $max);
// sync data
foreach ($globalMetrics as $metric => $options) { //for each metrics
foreach ($periods as $period) { // aggregate data for each period
$start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
if (!empty($latestTime[$metric][$period['key']])) {
$start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339);
}
$end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339);
$table = $options['table']; //Which influxdb table to query for this metric
$groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
$filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
if (!empty($filters)) {
$filters = ' AND ' . implode(' AND ', array_map(function ($filter, $value) {
return '"' . $filter . '"=\'' . $value . '\'';
}, array_keys($filters), array_values($filters)));
}
$result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\'' . (empty($filters) ? '' : $filters) . ' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)');
$points = $result->getPoints();
foreach ($points as $point) {
$projectId = $point['projectId'];
if (!empty($projectId) && $projectId != 'console') {
$dbForProject->setNamespace('project_' . $projectId . '_internal');
$metricUpdated = $metric;
if (!empty($groupBy)) {
$groupedBy = $point[$options['groupBy']] ?? '';
if (empty($groupedBy)) {
continue;
}
$metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric);
}
$time = \strtotime($point['time']);
$id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric
$value = (!empty($point['value'])) ? $point['value'] : 0;
try {
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period['key'],
'time' => $time,
'metric' => $metricUpdated,
'value' => $value,
'type' => 0,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $value));
}
$latestTime[$metric][$period['key']] = $time;
} catch (\Exception $e) { // if projects are deleted this might fail
Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}");
}
}
}
}
}
}
/**
* Aggregate MariaDB every 15 minutes
* Some of the queries here might contain full-table scans.
*/
if ($iterations % 30 == 0) { // Every 15 minutes aggregate number of objects in database
$latestProject = null;
do { // Loop over all the projects
$attempts = 0;
$max = 10;
$sleep = 1;
do { // list projects
try {
$attempts++;
$projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject);
break; // leave the do-while if successful
} catch (\Exception $e) {
Console::warning("Console DB not ready yet. Retrying ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('Failed access console db: ' . $e->getMessage());
}
sleep($sleep);
}
} while ($attempts < $max);
if (empty($projects)) {
continue;
}
$latestProject = $projects[array_key_last($projects)];
foreach ($projects as $project) {
$projectId = $project->getId();
// Get total storage
$dbForProject->setNamespace('project_' . $projectId . '_internal');
$storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size');
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_storage.total'); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => '30m',
'time' => $time,
'metric' => 'storage.total',
'value' => $storageTotal,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $storageTotal));
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_storage.total'); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => '1d',
'time' => $time,
'metric' => 'storage.total',
'value' => $storageTotal,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $storageTotal));
}
$collections = [
'users' => [
'namespace' => 'internal',
],
'collections' => [
'metricPrefix' => 'database',
'namespace' => 'internal',
'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting
'documents' => [
'namespace' => 'external',
],
],
],
'files' => [
'metricPrefix' => 'storage',
'namespace' => 'internal',
],
];
foreach ($collections as $collection => $options) {
try {
$dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}");
$count = $dbForProject->count($collection);
$dbForProject->setNamespace("project_{$projectId}_internal");
$metricPrefix = $options['metricPrefix'] ?? '';
$metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
$subCollections = $options['subCollections'] ?? [];
if (empty($subCollections)) {
continue;
}
$latestParent = null;
$subCollectionCounts = []; //total project level count of sub collections
do { // Loop over all the parent collection document for each sub collection
$dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}");
$parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections
if (empty($parents)) {
continue;
}
$latestParent = $parents[array_key_last($parents)];
foreach ($parents as $parent) {
foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count
$dbForProject->setNamespace("project_{$projectId}_{$subOptions['namespace']}");
$count = $dbForProject->count($parent->getId());
$subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count
$dbForProject->setNamespace("project_{$projectId}_internal");
$metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
}
}
} while (!empty($parents));
/**
* Inserting project level counts for sub collections like database.documents.count
*/
foreach ($subCollectionCounts as $subCollection => $count) {
$dbForProject->setNamespace("project_{$projectId}_internal");
$metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '30m',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
$time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day
$id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric
$document = $dbForProject->getDocument('stats', $id);
if ($document->isEmpty()) {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'time' => $time,
'period' => '1d',
'metric' => $metric,
'value' => $count,
'type' => 1,
]));
} else {
$dbForProject->updateDocument('stats', $document->getId(),
$document->setAttribute('value', $count));
}
}
} catch (\Exception$e) {
Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}");
}
}
}
} while (!empty($projects));
}
$iterations++;
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
});

20
app/views/install/compose.phtml
View file

@ -294,6 +294,26 @@ services:
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
appwrite-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: usage
container_name: appwrite-usage
restart: unless-stopped
networks:
- appwrite
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
appwrite-schedule:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>

34
app/workers.php
View file

@ -1,34 +0,0 @@
<?php
use Appwrite\Extend\PDO;
use Utopia\App;
/** @var Utopia\Registry\Registry $register */
require_once __DIR__.'/init.php';
$register->set('db', function () {
$dbHost = App::getEnv('_APP_DB_HOST', '');
$dbUser = App::getEnv('_APP_DB_USER', '');
$dbPass = App::getEnv('_APP_DB_PASS', '');
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
$pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4',
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
));
return $pdo;
});
$register->set('cache', function () { // Register cache connection
$redis = new Redis();
$redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
return $redis;
});

2
app/workers/audits.php
View file

@ -4,7 +4,7 @@ use Appwrite\Resque\Worker;
use Utopia\Audit\Audit;
use Utopia\CLI\Console;
require_once __DIR__.'/../workers.php';
require_once __DIR__.'/../init.php';
Console::title('Audits V1 Worker');
Console::success(APP_NAME.' audits worker v1 has started');

2
app/workers/certificates.php
View file

@ -9,7 +9,7 @@ use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Domains\Domain;
require_once __DIR__.'/../workers.php';
require_once __DIR__.'/../init.php';
Console::title('Certificates V1 Worker');
Console::success(APP_NAME.' certificates worker v1 has started');

2
app/workers/database.php
View file

@ -5,7 +5,7 @@ use Utopia\CLI\Console;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
require_once __DIR__.'/../workers.php';
require_once __DIR__.'/../init.php';
Console::title('Database V1 Worker');
Console::success(APP_NAME.' database worker v1 has started'."\n");

30
app/workers/deletes.php
View file

@ -11,7 +11,7 @@ use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\CLI\Console;
use Utopia\Audit\Audit;
require_once __DIR__.'/../workers.php';
require_once __DIR__.'/../init.php';
Console::title('Deletes V1 Worker');
Console::success(APP_NAME.' deletes worker v1 has started'."\n");
@ -81,7 +81,10 @@ class DeletesV1 extends Worker
$document = new Document($this->args['document']);
$this->deleteCertificates($document);
break;
case DELETE_TYPE_USAGE_STATS:
$this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']);
break;
default:
Console::error('No delete operation for type: '.$type);
break;
@ -110,6 +113,29 @@ class DeletesV1 extends Worker
new Query('collectionId', Query::TYPE_EQUAL, [$collectionId])
], $dbForInternal);
}
/**
* @param int $timestamp1d
* @param int $timestamp30m
*/
protected function deleteUsageStats(int $timestamp1d, int $timestamp30m) {
$this->deleteForProjectIds(function($projectId) use ($timestamp1d, $timestamp30m) {
if (!($dbForInternal = $this->getInternalDB($projectId))) {
throw new Exception('Failed to get projectDB for project '.$projectId);
}
// Delete Usage stats
$this->deleteByGroup('stats', [
new Query('time', Query::TYPE_LESSER, [$timestamp1d]),
new Query('period', Query::TYPE_EQUAL, ['1d']),
], $dbForInternal);
$this->deleteByGroup('stats', [
new Query('time', Query::TYPE_LESSER, [$timestamp30m]),
new Query('period', Query::TYPE_EQUAL, ['30m']),
], $dbForInternal);
});
}
/**
* @param Document $document teams document

2
app/workers/functions.php
View file

@ -13,7 +13,7 @@ use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
require_once __DIR__.'/../workers.php';
require_once __DIR__.'/../init.php';
Runtime::enableCoroutine(0);

2
app/workers/mails.php
View file

@ -6,7 +6,7 @@ use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Locale\Locale;
require_once __DIR__ . '/../workers.php';
require_once __DIR__ . '/../init.php';
Console::title('Mails V1 Worker');
Console::success(APP_NAME . ' mails worker v1 has started' . "\n");

2
app/workers/webhooks.php
View file

@ -4,7 +4,7 @@ use Appwrite\Resque\Worker;
use Utopia\App;
use Utopia\CLI\Console;
require_once __DIR__.'/../workers.php';
require_once __DIR__.'/../init.php';
Console::title('Webhooks V1 Worker');
Console::success(APP_NAME.' webhooks worker v1 has started');

3
bin/usage Executable file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php usage $@

2
composer.json
View file

@ -83,4 +83,4 @@
"php": "8.0"
}
}
}
}

12
composer.lock generated
View file

@ -355,16 +355,16 @@
},
{
"name": "composer/package-versions-deprecated",
"version": "1.11.99.2",
"version": "1.11.99.3",
"source": {
"type": "git",
"url": "https://github.com/composer/package-versions-deprecated.git",
"reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c"
"reference": "fff576ac850c045158a250e7e27666e146e78d18"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/c6522afe5540d5fc46675043d3ed5a45a740b27c",
"reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c",
"url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18",
"reference": "fff576ac850c045158a250e7e27666e146e78d18",
"shasum": ""
},
"require": {
@ -408,7 +408,7 @@
"description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)",
"support": {
"issues": "https://github.com/composer/package-versions-deprecated/issues",
"source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.2"
"source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3"
},
"funding": [
{
@ -424,7 +424,7 @@
"type": "tidelift"
}
],
"time": "2021-05-24T07:46:03+00:00"
"time": "2021-08-17T13:49:14+00:00"
},
{
"name": "dragonmantank/cron-expression",

34
docker-compose.yml
View file

@ -73,7 +73,6 @@ services:
- mariadb
- redis
# - clamav
- influxdb
entrypoint:
- php
- -e
@ -112,8 +111,6 @@ services:
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS
@ -349,6 +346,37 @@ services:
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
appwrite-usage:
entrypoint:
- php
- -e
- /usr/src/code/app/cli.php
- usage
container_name: appwrite-usage
build:
context: .
args:
- DEBUG=false
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./dev:/usr/local/dev
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_SYNC_INTERVAL
appwrite-schedule:
entrypoint: schedule
container_name: appwrite-schedule

28
src/Appwrite/Specification/Format/OpenAPI3.php
View file

@ -21,6 +21,28 @@ class OpenAPI3 extends Format
return 'Open API 3';
}
/**
* Get Used Models
*
* Recursively get all used models
*
* @param object $model
* @param array $models
*
* @return void
*/
protected function getUsedModels($model, array &$usedModels)
{
if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) {
$usedModels[] = $model;
return;
}
if (!is_object($model)) return;
foreach ($model->getRules() as $rule) {
$this->getUsedModels($rule['type'], $usedModels);
}
}
/**
* Parse
*
@ -352,11 +374,7 @@ class OpenAPI3 extends Format
$output['paths'][$url][\strtolower($route->getMethod())] = $temp;
}
foreach ($this->models as $model) {
foreach ($model->getRules() as $rule) {
if (!in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float'])) {
$usedModels[] = $rule['type'];
}
}
$this->getUsedModels($model, $usedModels);
}
foreach ($this->models as $model) {
if (!in_array($model->getType(), $usedModels) && $model->getType() !== 'error') {

32
src/Appwrite/Specification/Format/Swagger2.php
View file

@ -21,6 +21,28 @@ class Swagger2 extends Format
return 'Swagger 2';
}
/**
* Get Used Models
*
* Recursively get all used models
*
* @param object $model
* @param array $models
*
* @return void
*/
protected function getUsedModels($model, array &$usedModels)
{
if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) {
$usedModels[] = $model;
return;
}
if (!is_object($model)) return;
foreach ($model->getRules() as $rule) {
$this->getUsedModels($rule['type'], $usedModels);
}
}
/**
* Parse
*
@ -354,15 +376,9 @@ class Swagger2 extends Format
$output['paths'][$url][\strtolower($route->getMethod())] = $temp;
}
foreach ($this->models as $model) {
foreach ($model->getRules() as $rule) {
if (
in_array($model->getType(), $usedModels)
&& !in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float'])
) {
$usedModels[] = $rule['type'];
}
}
$this->getUsedModels($model, $usedModels);
}
foreach ($this->models as $model) {
if (!in_array($model->getType(), $usedModels)) {
continue;

65
src/Appwrite/Stats/Stats.php
View file

@ -94,7 +94,7 @@ class Stats
$functionExecutionTime = $this->params['functionExecutionTime'] ?? 0;
$functionStatus = $this->params['functionStatus'] ?? '';
$tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
$tags = ",projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
// the global namespace is prepended to every key (optional)
$this->statsd->setNamespace($this->namespace);
@ -112,7 +112,70 @@ class Stats
$this->statsd->count('network.outbound' . $tags, $networkResponseSize);
$this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize);
$dbMetrics = [
'database.collections.create',
'database.collections.read',
'database.collections.update',
'database.collections.delete',
'database.documents.create',
'database.documents.read',
'database.documents.update',
'database.documents.delete',
];
foreach ($dbMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$tags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? '');
$this->statsd->increment($metric . $tags);
}
}
$storageMertics = [
'storage.files.create',
'storage.files.read',
'storage.files.update',
'storage.files.delete',
];
foreach ($storageMertics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? '');
$this->statsd->increment($metric . $tags);
}
}
$usersMetrics = [
'users.create',
'users.read',
'users.update',
'users.delete',
];
foreach ($usersMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$tags = ",projectId={$projectId}";
$this->statsd->increment($metric . $tags);
}
}
$sessionsMetrics = [
'users.sessions.create',
'users.sessions.delete',
];
foreach ($sessionsMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$tags = ",projectId={$projectId},provider=". ($this->params['provider'] ?? '');
$this->statsd->count($metric . $tags, $value);
}
}
if ($storage >= 1) {
$tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? '');
$this->statsd->count('storage.all' . $tags, $storage);
}

27
src/Appwrite/Utopia/Response.php
View file

@ -33,6 +33,7 @@ use Appwrite\Utopia\Response\Model\Team;
use Appwrite\Utopia\Response\Model\Locale;
use Appwrite\Utopia\Response\Model\Log;
use Appwrite\Utopia\Response\Model\Membership;
use Appwrite\Utopia\Response\Model\Metric;
use Appwrite\Utopia\Response\Model\Permissions;
use Appwrite\Utopia\Response\Model\Phone;
use Appwrite\Utopia\Response\Model\Platform;
@ -43,6 +44,13 @@ use Appwrite\Utopia\Response\Model\Token;
use Appwrite\Utopia\Response\Model\Webhook;
use Appwrite\Utopia\Response\Model\Preferences;
use Appwrite\Utopia\Response\Model\Mock; // Keep last
use Appwrite\Utopia\Response\Model\UsageBuckets;
use Appwrite\Utopia\Response\Model\UsageCollection;
use Appwrite\Utopia\Response\Model\UsageDatabase;
use Appwrite\Utopia\Response\Model\UsageFunctions;
use Appwrite\Utopia\Response\Model\UsageProject;
use Appwrite\Utopia\Response\Model\UsageStorage;
use Appwrite\Utopia\Response\Model\UsageUsers;
use stdClass;
/**
@ -56,8 +64,17 @@ class Response extends SwooleResponse
const MODEL_LOG = 'log';
const MODEL_LOG_LIST = 'logList';
const MODEL_ERROR = 'error';
const MODEL_METRIC = 'metric';
const MODEL_METRIC_LIST = 'metricList';
const MODEL_ERROR_DEV = 'errorDev';
const MODEL_BASE_LIST = 'baseList';
const MODEL_USAGE_DATABASE = 'usageDatabase';
const MODEL_USAGE_COLLECTION = 'usageCollection';
const MODEL_USAGE_USERS = 'usageUsers';
const MODEL_USAGE_BUCKETS = 'usageBuckets';
const MODEL_USAGE_STORAGE = 'usageStorage';
const MODEL_USAGE_FUNCTIONS = 'usageFunctions';
const MODEL_USAGE_PROJECT = 'usageProject';
// Database
const MODEL_COLLECTION = 'collection';
@ -125,6 +142,7 @@ class Response extends SwooleResponse
// Deprecated
const MODEL_PERMISSIONS = 'permissions';
const MODEL_RULE = 'rule';
const MODEL_TASK = 'task';
// Tests (keep last)
const MODEL_MOCK = 'mock';
@ -176,6 +194,7 @@ class Response extends SwooleResponse
->setModel(new BaseList('Languages List', self::MODEL_LANGUAGE_LIST, 'languages', self::MODEL_LANGUAGE))
->setModel(new BaseList('Currencies List', self::MODEL_CURRENCY_LIST, 'currencies', self::MODEL_CURRENCY))
->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE))
->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false))
// Entities
->setModel(new Collection())
->setModel(new Attribute())
@ -204,6 +223,14 @@ class Response extends SwooleResponse
->setModel(new Language())
->setModel(new Currency())
->setModel(new Phone())
->setModel(new Metric())
->setModel(new UsageDatabase())
->setModel(new UsageCollection())
->setModel(new UsageUsers())
->setModel(new UsageStorage())
->setModel(new UsageBuckets())
->setModel(new UsageFunctions())
->setModel(new UsageProject())
// Verification
// Recovery
// Tests (keep last)

47
src/Appwrite/Utopia/Response/Model/Metric.php Normal file
View file

@ -0,0 +1,47 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class Metric extends Model
{
public function __construct()
{
$this
->addRule('value', [
'type' => self::TYPE_INTEGER,
'description' => 'The value of this metric at the timestamp.',
'default' => -1,
'example' => 1,
])
->addRule('timestamp', [
'type' => self::TYPE_INTEGER,
'description' => 'The UNIX timestamp at which this metric was aggregated.',
'default' => 0,
'example' => 1592981250
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName():string
{
return 'Metric';
}
/**
* Get Collection
*
* @return string
*/
public function getType():string
{
return Response::MODEL_METRIC;
}
}

77
src/Appwrite/Utopia/Response/Model/UsageBuckets.php Normal file
View file

@ -0,0 +1,77 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use stdClass;
class UsageBuckets extends Model
{
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('files.count', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for total number of files in this bucket.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('files.create', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('files.read', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('files.update', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('files.delete', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for files deleted.',
'default' => [],
'example' => new stdClass,
'array' => true
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName():string
{
return 'UsageBuckets';
}
/**
* Get Type
*
* @return string
*/
public function getType():string
{
return Response::MODEL_USAGE_BUCKETS;
}
}

77
src/Appwrite/Utopia/Response/Model/UsageCollection.php Normal file
View file

@ -0,0 +1,77 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use stdClass;
class UsageCollection extends Model
{
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('documents.count', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('documents.create', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('documents.read', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('documents.update', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('documents.delete', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => new stdClass,
'array' => true
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName():string
{
return 'UsageCollection';
}
/**
* Get Type
*
* @return string
*/
public function getType():string
{
return Response::MODEL_USAGE_COLLECTION;
}
}

112
src/Appwrite/Utopia/Response/Model/UsageDatabase.php Normal file
View file

@ -0,0 +1,112 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use stdClass;
class UsageDatabase extends Model
{
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('documents.count', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('collections.count', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('documents.create', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('documents.read', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('documents.update', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('documents.delete', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('collections.create', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('collections.read', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('collections.update', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('collections.delete', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for collections delete.',
'default' => [],
'example' => new stdClass,
'array' => true
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName():string
{
return 'UsageDatabase';
}
/**
* Get Type
*
* @return string
*/
public function getType():string
{
return Response::MODEL_USAGE_DATABASE;
}
}

63
src/Appwrite/Utopia/Response/Model/UsageFunctions.php Normal file
View file

@ -0,0 +1,63 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use stdClass;
class UsageFunctions extends Model
{
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('functions.executions', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for function executions.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('functions.failures', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for function execution failures.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('functions.compute', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for function execution duration.',
'default' => [],
'example' => new stdClass,
'array' => true
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName():string
{
return 'UsageFunctions';
}
/**
* Get Type
*
* @return string
*/
public function getType():string
{
return Response::MODEL_USAGE_FUNCTIONS;
}
}

91
src/Appwrite/Utopia/Response/Model/UsageProject.php Normal file
View file

@ -0,0 +1,91 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use stdClass;
class UsageProject extends Model
{
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('requests', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for number of requests.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('network', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for consumed bandwidth.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('functions', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for function executions.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('documents', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for number of documents.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('collections', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for number of collections.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('users', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for number of users.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('storage', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => new stdClass,
'array' => true
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName():string
{
return 'UsageProject';
}
/**
* Get Type
*
* @return string
*/
public function getType():string
{
return Response::MODEL_USAGE_PROJECT;
}
}

56
src/Appwrite/Utopia/Response/Model/UsageStorage.php Normal file
View file

@ -0,0 +1,56 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use stdClass;
class UsageStorage extends Model
{
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('storage', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('files', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for total number of files.',
'default' => [],
'example' => new stdClass,
'array' => true
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName():string
{
return 'StorageUsage';
}
/**
* Get Type
*
* @return string
*/
public function getType():string
{
return Response::MODEL_USAGE_STORAGE;
}
}

98
src/Appwrite/Utopia/Response/Model/UsageUsers.php Normal file
View file

@ -0,0 +1,98 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use stdClass;
class UsageUsers extends Model
{
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('users.count', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for total number of users.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('users.create', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for users created.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('users.read', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for users read.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('users.update', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for users updated.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('users.delete', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for users deleted.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('sessions.create', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for sessions created.',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('sessions.provider.create', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).',
'default' => [],
'example' => new stdClass,
'array' => true
])
->addRule('sessions.delete', [
'type' => Response::MODEL_METRIC_LIST,
'description' => 'Aggregated stats for sessions deleted.',
'default' => [],
'example' => new stdClass,
'array' => true
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName():string
{
return 'UsageUsers';
}
/**
* Get Type
*
* @return string
*/
public function getType():string
{
return Response::MODEL_USAGE_USERS;
}
}

23
tests/e2e/Scopes/SideConsole.php Normal file
View file

@ -0,0 +1,23 @@
<?php
namespace Tests\E2E\Scopes;
trait SideConsole
{
public function getHeaders():array
{
return [
'origin' => 'http://localhost',
'cookie' => 'a_session_console='. $this->getRoot()['session'],
'x-appwrite-mode' => 'admin'
];
}
/**
* @return string
*/
public function getSide()
{
return 'console';
}
}

125
tests/e2e/Services/Database/DatabaseConsoleClientTest.php Normal file
View file

@ -0,0 +1,125 @@
<?php
namespace Tests\E2E\Services\Database;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Client;
use Tests\E2E\Scopes\SideConsole;
class DatabaseConsoleClientTest extends Scope
{
use ProjectCustom;
use SideConsole;
public function testCreateCollection():array
{
/**
* Test for SUCCESS
*/
$movies = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'collectionId' => 'unique()',
'name' => 'Movies',
'read' => ['role:all'],
'write' => ['role:all'],
'permission' => 'document',
]);
$this->assertEquals($movies['headers']['status-code'], 201);
$this->assertEquals($movies['body']['name'], 'Movies');
return ['moviesId' => $movies['body']['$id']];
}
public function testGetDatabaseUsage()
{
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h'
]);
$this->assertEquals($response['headers']['status-code'], 400);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 11);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['documents.count']);
$this->assertIsArray($response['body']['collections.count']);
$this->assertIsArray($response['body']['documents.create']);
$this->assertIsArray($response['body']['documents.read']);
$this->assertIsArray($response['body']['documents.update']);
$this->assertIsArray($response['body']['documents.delete']);
$this->assertIsArray($response['body']['collections.create']);
$this->assertIsArray($response['body']['collections.read']);
$this->assertIsArray($response['body']['collections.update']);
$this->assertIsArray($response['body']['collections.delete']);
}
/**
* @depends testCreateCollection
*/
public function testGetCollectionUsage(array $data)
{
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h'
]);
$this->assertEquals($response['headers']['status-code'], 400);
$response = $this->client->call(Client::METHOD_GET, '/database/randomCollectionId/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 404);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 6);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['documents.count']);
$this->assertIsArray($response['body']['documents.create']);
$this->assertIsArray($response['body']['documents.read']);
$this->assertIsArray($response['body']['documents.update']);
$this->assertIsArray($response['body']['documents.delete']);
}
}

91
tests/e2e/Services/Functions/FunctionsConsoleClientTest.php Normal file
View file

@ -0,0 +1,91 @@
<?php
namespace Tests\E2E\Services\Functions;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Client;
use Tests\E2E\Scopes\SideConsole;
class FunctionsConsoleClientTest extends Scope
{
use ProjectCustom;
use SideConsole;
public function testCreateFunction():array
{
$function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'functionId' => 'unique()',
'name' => 'Test',
'execute' => ['user:'.$this->getUser()['$id']],
'runtime' => 'php-8.0',
'vars' => [
'funcKey1' => 'funcValue1',
'funcKey2' => 'funcValue2',
'funcKey3' => 'funcValue3',
],
'events' => [
'account.create',
'account.delete',
],
'schedule' => '0 0 1 1 *',
'timeout' => 10,
]);
$this->assertEquals(201, $function['headers']['status-code']);
return [
'functionId' => $function['body']['$id']
];
}
/**
* @depends testCreateFunction
*/
public function testGetCollectionUsage(array $data)
{
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '232h'
]);
$this->assertEquals(400, $response['headers']['status-code']);
$response = $this->client->call(Client::METHOD_GET, '/functions/randomFunctionId/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals(404, $response['headers']['status-code']);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 4);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['functions.executions']);
$this->assertIsArray($response['body']['functions.failures']);
$this->assertIsArray($response['body']['functions.compute']);
}
}

26
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
View file

@ -215,24 +215,16 @@ class ProjectsConsoleClientTest extends Scope
], $this->getHeaders()));
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 8);
$this->assertNotEmpty($response['body']);
$this->assertArrayHasKey('collections', $response['body']);
$this->assertArrayHasKey('documents', $response['body']);
$this->assertArrayHasKey('network', $response['body']);
$this->assertArrayHasKey('requests', $response['body']);
$this->assertArrayHasKey('storage', $response['body']);
$this->assertArrayHasKey('users', $response['body']);
$this->assertIsArray($response['body']['collections']['data']);
$this->assertIsInt($response['body']['collections']['total']);
$this->assertIsArray($response['body']['documents']['data']);
$this->assertIsInt($response['body']['documents']['total']);
$this->assertIsArray($response['body']['network']['data']);
$this->assertIsInt($response['body']['network']['total']);
$this->assertIsArray($response['body']['requests']['data']);
$this->assertIsInt($response['body']['requests']['total']);
$this->assertIsInt($response['body']['storage']['total']);
$this->assertIsArray($response['body']['users']['data']);
$this->assertIsInt($response['body']['users']['total']);
$this->assertEquals('30d', $response['body']['range']);
$this->assertIsArray($response['body']['requests']);
$this->assertIsArray($response['body']['network']);
$this->assertIsArray($response['body']['functions']);
$this->assertIsArray($response['body']['documents']);
$this->assertIsArray($response['body']['collections']);
$this->assertIsArray($response['body']['users']);
$this->assertIsArray($response['body']['storage']);
/**
* Test for FAILURE

86
tests/e2e/Services/Storage/StorageConsoleClientTest.php
View file

@ -2,13 +2,91 @@
namespace Tests\E2E\Services\Storage;
use Tests\E2E\Client;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectConsole;
use Tests\E2E\Scopes\SideClient;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideConsole;
class StorageConsoleClientTest extends Scope
{
use SideConsole;
use StorageBase;
use ProjectConsole;
use SideClient;
use ProjectCustom;
public function testGetStorageUsage()
{
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h'
]);
$this->assertEquals($response['headers']['status-code'], 400);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 3);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['storage']);
$this->assertIsArray($response['body']['files']);
}
public function testGetStorageBucketUsage()
{
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h'
]);
$this->assertEquals($response['headers']['status-code'], 400);
// TODO: Uncomment once we implement check for missing bucketId in the usage endpoint.
// $response = $this->client->call(Client::METHOD_GET, '/storage/randomBucketId/usage', array_merge([
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id']
// ], $this->getHeaders()), [
// 'range' => '24h'
// ]);
// $this->assertEquals($response['headers']['status-code'], 404);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 6);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['files.count']);
$this->assertIsArray($response['body']['files.create']);
$this->assertIsArray($response['body']['files.read']);
$this->assertIsArray($response['body']['files.update']);
$this->assertIsArray($response['body']['files.delete']);
}
}

83
tests/e2e/Services/Users/UsersConsoleClientTest.php Normal file
View file

@ -0,0 +1,83 @@
<?php
namespace Tests\E2E\Services\Users;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Client;
use Tests\E2E\Scopes\SideConsole;
class UsersConsoleClientTest extends Scope
{
use ProjectCustom;
use SideConsole;
public function testGetUsersUsage()
{
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h',
'provider' => 'email'
]);
$this->assertEquals($response['headers']['status-code'], 400);
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h',
'provider' => 'some-random-provider'
]);
$this->assertEquals($response['headers']['status-code'], 400);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h',
'provider' => 'email'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['users.count']);
$this->assertIsArray($response['body']['users.create']);
$this->assertIsArray($response['body']['users.read']);
$this->assertIsArray($response['body']['users.update']);
$this->assertIsArray($response['body']['users.delete']);
$this->assertIsArray($response['body']['sessions.create']);
$this->assertIsArray($response['body']['sessions.provider.create']);
$this->assertIsArray($response['body']['sessions.delete']);
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['users.count']);
$this->assertIsArray($response['body']['users.create']);
$this->assertIsArray($response['body']['users.read']);
$this->assertIsArray($response['body']['users.update']);
$this->assertIsArray($response['body']['users.delete']);
$this->assertIsArray($response['body']['sessions.create']);
$this->assertIsArray($response['body']['sessions.provider.create']);
$this->assertIsArray($response['body']['sessions.delete']);
}
}