CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-27 02:31:04 +12:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/1.1.x' into feat-graphql-support

Browse source 
# Conflicts:
#	CHANGES.md
#	app/config/specs/open-api3-latest-client.json
#	app/config/specs/open-api3-latest-console.json
#	app/config/specs/open-api3-latest-server.json
#	app/config/specs/swagger2-latest-client.json
#	app/config/specs/swagger2-latest-console.json
#	app/config/specs/swagger2-latest-server.json
#	app/init.php
#	app/tasks/sdks.php
#	composer.lock
#	docker-compose.yml
This commit is contained in:
Jake Barnby 2022-11-16 19:37:35 +13:00
parent dd6c5c056f 27ddf8630b
commit f9d2976c1d
No known key found for this signature in database
GPG key ID: C437A8CC85B96E9C
48 changed files with 1071 additions and 1079 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.env
View file

@ -76,8 +76,8 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
_APP_USAGE_TIMESERIES_INTERVAL=2
_APP_USAGE_DATABASE_INTERVAL=15
_APP_USAGE_AGGREGATION_INTERVAL=5
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
_APP_USAGE_STATS=enabled
_APP_LOGGING_PROVIDER=
_APP_LOGGING_CONFIG=

9
CHANGES.md
View file

@ -1,13 +1,12 @@
# Version 1.1.0
## Features
- Added new property to projects configuration: `authDuration` which allows you to alter the duration of signed in sessions for your project. [#4618](https://github.com/appwrite/appwrite/pull/4618)
## Bugs
- Fix license detection for Flutter and Dart SDKs [#4435](https://github.com/appwrite/appwrite/pull/4435)
- Fix missing realtime event for create function deployment [#4574](https://github.com/appwrite/appwrite/pull/4574)
- Fix missing status, buildStderr and buildStderr from get deployment response [#4611](https://github.com/appwrite/appwrite/pull/4611)
# Version 1.0.4
## Bugs
- Fix project pagination in DB usage collector [#4517](https://github.com/appwrite/appwrite/pull/4517)
- Fix missing `status`, `buildStderr` and `buildStderr` from get deployment response [#4611](https://github.com/appwrite/appwrite/pull/4611)
- Fix project pagination in DB usage aggregation [#4517](https://github.com/appwrite/appwrite/pull/4517)
# Version 1.0.3
## Bugs

4
Dockerfile
View file

@ -246,13 +246,13 @@ ENV _APP_SERVER=swoole \
_APP_SETUP=self-hosted \
_APP_VERSION=$VERSION \
_APP_USAGE_STATS=enabled \
_APP_USAGE_TIMESERIES_INTERVAL=30 \
_APP_USAGE_DATABASE_INTERVAL=900 \
_APP_USAGE_AGGREGATION_INTERVAL=30 \
# 14 Days = 1209600 s
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600 \
_APP_MAINTENANCE_RETENTION_AUDIT=1209600 \
# 1 Day = 86400 s
_APP_MAINTENANCE_RETENTION_ABUSE=86400 \
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 \
_APP_MAINTENANCE_INTERVAL=86400 \
_APP_LOGGING_PROVIDER= \
_APP_LOGGING_CONFIG=

11
app/config/collections.php
View file

@ -1632,17 +1632,6 @@ $collections = [
'array' => false,
'filters' => ['encrypt'],
],
[
'$id' => ID::custom('expire'),
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('userAgent'),
'type' => Database::VAR_STRING,

2
app/config/specs/open-api3-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-server.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-server.json
View file

File diff suppressed because one or more lines are too long

19
app/config/variables.php
View file

@ -170,8 +170,8 @@ return [
],
[
'name' => '_APP_USAGE_AGGREGATION_INTERVAL',
'description' => 'Deprecated since 1.0.0, use `_APP_USAGE_TIMESERIES_INTERVAL` and `_APP_USAGE_DATABASE_INTERVAL` instead.',
'introduction' => '0.10.0',
'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to Database from TimeSeries data. The default value is 30 seconds. Reintroduced in 1.1.0.',
'introduction' => '1.1.0',
'default' => '30',
'required' => false,
'question' => '',
@ -179,7 +179,7 @@ return [
],
[
'name' => '_APP_USAGE_TIMESERIES_INTERVAL',
'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to Appwrite Database from Timeseries Database. The default value is 30 seconds.',
'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.',
'introduction' => '1.0.0',
'default' => '30',
'required' => false,
@ -188,7 +188,7 @@ return [
],
[
'name' => '_APP_USAGE_DATABASE_INTERVAL',
'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats from data in Appwrite Database. The default value is 15 minutes.',
'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.',
'introduction' => '1.0.0',
'default' => '900',
'required' => false,
@ -857,7 +857,16 @@ return [
'required' => false,
'question' => '',
'filter' => ''
]
],
[
'name' => '_APP_MAINTENANCE_RETENTION_USAGE_HOURLY',
'description' => 'The maximum duration (in seconds) upto which to retain hourly usage metrics. The default value is 8640000 seconds (100 days).',
'introduction' => '',
'default' => '8640000',
'required' => false,
'question' => '',
'filter' => ''
],
],
[
'category' => 'GraphQL',

1
app/console Submodule

@ -0,0 +1 @@
Subproject commit 0ed6e0c497931f16fcb0750fe351d1d3577a7d97

68
app/controllers/api/account.php
View file

@ -163,10 +163,11 @@ App::post('/v1/account/sessions/email')
->inject('request')
->inject('response')
->inject('dbForProject')
->inject('project')
->inject('locale')
->inject('geodb')
->inject('events')
->action(function (string $email, string $password, Request $request, Response $response, Database $dbForProject, Locale $locale, Reader $geodb, Event $events) {
->action(function (string $email, string $password, Request $request, Response $response, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $events) {
$email = \strtolower($email);
$protocol = $request->getProtocol();
@ -183,9 +184,11 @@ App::post('/v1/account/sessions/email')
throw new Exception(Exception::USER_BLOCKED); // User is in status blocked
}
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$detector = new Detector($request->getUserAgent('UNKNOWN'));
$record = $geodb->get($request->getIP());
$expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_LOGIN_LONG);
$expire = DateTime::addSeconds(new \DateTime(), $duration);
$secret = Auth::tokenGenerator();
$session = new Document(array_merge(
[
@ -195,7 +198,6 @@ App::post('/v1/account/sessions/email')
'provider' => Auth::SESSION_PROVIDER_EMAIL,
'providerUid' => $email,
'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak
'expire' => $expire,
'userAgent' => $request->getUserAgent('UNKNOWN'),
'ip' => $request->getIP(),
'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--',
@ -242,6 +244,7 @@ App::post('/v1/account/sessions/email')
$session
->setAttribute('current', true)
->setAttribute('countryName', $countryName)
->setAttribute('expire', $expire)
;
$events
@ -448,7 +451,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
}
$sessions = $user->getAttribute('sessions', []);
$current = Auth::sessionVerify($sessions, Auth::$secret);
$authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$current = Auth::sessionVerify($sessions, Auth::$secret, $authDuration);
if ($current) { // Delete current session of new one.
$currentDocument = $dbForProject->getDocument('sessions', $current);
@ -523,10 +527,11 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
}
// Create session token, verify user account and update OAuth2 ID and Access Token
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$detector = new Detector($request->getUserAgent('UNKNOWN'));
$record = $geodb->get($request->getIP());
$secret = Auth::tokenGenerator();
$expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_LOGIN_LONG);
$expire = DateTime::addSeconds(new \DateTime(), $duration);
$session = new Document(array_merge([
'$id' => ID::unique(),
@ -538,7 +543,6 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
'providerRefreshToken' => $refreshToken,
'providerAccessTokenExpiry' => DateTime::addSeconds(new \DateTime(), (int)$accessTokenExpiry),
'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak
'expire' => $expire,
'userAgent' => $request->getUserAgent('UNKNOWN'),
'ip' => $request->getIP(),
'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--',
@ -569,6 +573,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
$dbForProject->deleteCachedDocument('users', $user->getId());
$session->setAttribute('expire', $expire);
$events
->setParam('userId', $user->getId())
->setParam('sessionId', $session->getId())
@ -757,10 +763,11 @@ App::put('/v1/account/sessions/magic-url')
->inject('request')
->inject('response')
->inject('dbForProject')
->inject('project')
->inject('locale')
->inject('geodb')
->inject('events')
->action(function (string $userId, string $secret, Request $request, Response $response, Database $dbForProject, Locale $locale, Reader $geodb, Event $events) {
->action(function (string $userId, string $secret, Request $request, Response $response, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $events) {
/** @var Utopia\Database\Document $user */
@ -776,10 +783,11 @@ App::put('/v1/account/sessions/magic-url')
throw new Exception(Exception::USER_INVALID_TOKEN);
}
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$detector = new Detector($request->getUserAgent('UNKNOWN'));
$record = $geodb->get($request->getIP());
$secret = Auth::tokenGenerator();
$expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_LOGIN_LONG);
$expire = DateTime::addSeconds(new \DateTime(), $duration);
$session = new Document(array_merge(
[
@ -788,7 +796,6 @@ App::put('/v1/account/sessions/magic-url')
'userInternalId' => $user->getInternalId(),
'provider' => Auth::SESSION_PROVIDER_MAGIC_URL,
'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak
'expire' => $expire,
'userAgent' => $request->getUserAgent('UNKNOWN'),
'ip' => $request->getIP(),
'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--',
@ -848,6 +855,7 @@ App::put('/v1/account/sessions/magic-url')
$session
->setAttribute('current', true)
->setAttribute('countryName', $countryName)
->setAttribute('expire', $expire)
;
$response->dynamic($session, Response::MODEL_SESSION);
@ -994,10 +1002,11 @@ App::put('/v1/account/sessions/phone')
->inject('request')
->inject('response')
->inject('dbForProject')
->inject('project')
->inject('locale')
->inject('geodb')
->inject('events')
->action(function (string $userId, string $secret, Request $request, Response $response, Database $dbForProject, Locale $locale, Reader $geodb, Event $events) {
->action(function (string $userId, string $secret, Request $request, Response $response, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $events) {
$user = Authorization::skip(fn() => $dbForProject->getDocument('users', $userId));
@ -1011,10 +1020,11 @@ App::put('/v1/account/sessions/phone')
throw new Exception(Exception::USER_INVALID_TOKEN);
}
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$detector = new Detector($request->getUserAgent('UNKNOWN'));
$record = $geodb->get($request->getIP());
$secret = Auth::tokenGenerator();
$expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_LOGIN_LONG);
$expire = DateTime::addSeconds(new \DateTime(), $duration);
$session = new Document(array_merge(
[
@ -1023,7 +1033,6 @@ App::put('/v1/account/sessions/phone')
'userInternalId' => $user->getInternalId(),
'provider' => Auth::SESSION_PROVIDER_PHONE,
'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak
'expire' => $expire,
'userAgent' => $request->getUserAgent('UNKNOWN'),
'ip' => $request->getIP(),
'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--',
@ -1081,6 +1090,7 @@ App::put('/v1/account/sessions/phone')
$session
->setAttribute('current', true)
->setAttribute('countryName', $countryName)
->setAttribute('expire', $expire)
;
$response->dynamic($session, Response::MODEL_SESSION);
@ -1162,11 +1172,11 @@ App::post('/v1/account/sessions/anonymous')
])));
// Create session token
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$detector = new Detector($request->getUserAgent('UNKNOWN'));
$record = $geodb->get($request->getIP());
$secret = Auth::tokenGenerator();
$expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_LOGIN_LONG);
$expire = DateTime::addSeconds(new \DateTime(), $duration);
$session = new Document(array_merge(
[
@ -1175,7 +1185,6 @@ App::post('/v1/account/sessions/anonymous')
'userInternalId' => $user->getInternalId(),
'provider' => Auth::SESSION_PROVIDER_ANONYMOUS,
'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak
'expire' => $expire,
'userAgent' => $request->getUserAgent('UNKNOWN'),
'ip' => $request->getIP(),
'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--',
@ -1215,6 +1224,7 @@ App::post('/v1/account/sessions/anonymous')
$session
->setAttribute('current', true)
->setAttribute('countryName', $countryName)
->setAttribute('expire', $expire)
;
$response->dynamic($session, Response::MODEL_SESSION);
@ -1322,10 +1332,12 @@ App::get('/v1/account/sessions')
->inject('response')
->inject('user')
->inject('locale')
->action(function (Response $response, Document $user, Locale $locale) {
->inject('project')
->action(function (Response $response, Document $user, Locale $locale, Document $project) {
$sessions = $user->getAttribute('sessions', []);
$current = Auth::sessionVerify($sessions, Auth::$secret);
$authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$current = Auth::sessionVerify($sessions, Auth::$secret, $authDuration);
foreach ($sessions as $key => $session) {/** @var Document $session */
$countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'));
@ -1420,11 +1432,13 @@ App::get('/v1/account/sessions/:sessionId')
->inject('user')
->inject('locale')
->inject('dbForProject')
->action(function (?string $sessionId, Response $response, Document $user, Locale $locale, Database $dbForProject) {
->inject('project')
->action(function (?string $sessionId, Response $response, Document $user, Locale $locale, Database $dbForProject, Document $project) {
$sessions = $user->getAttribute('sessions', []);
$authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$sessionId = ($sessionId === 'current')
? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret)
? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret, $authDuration)
: $sessionId;
foreach ($sessions as $session) {/** @var Document $session */
@ -1434,6 +1448,7 @@ App::get('/v1/account/sessions/:sessionId')
$session
->setAttribute('current', ($session->getAttribute('secret') == Auth::hash(Auth::$secret)))
->setAttribute('countryName', $countryName)
->setAttribute('expire', DateTime::addSeconds(new \DateTime($session->getCreatedAt()), $authDuration))
;
return $response->dynamic($session, Response::MODEL_SESSION);
@ -1700,11 +1715,13 @@ App::delete('/v1/account/sessions/:sessionId')
->inject('dbForProject')
->inject('locale')
->inject('events')
->action(function (?string $sessionId, Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $events) {
->inject('project')
->action(function (?string $sessionId, Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $events, Document $project) {
$protocol = $request->getProtocol();
$authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$sessionId = ($sessionId === 'current')
? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret)
? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret, $authDuration)
: $sessionId;
$sessions = $user->getAttribute('sessions', []);
@ -1775,9 +1792,9 @@ App::patch('/v1/account/sessions/:sessionId')
->inject('locale')
->inject('events')
->action(function (?string $sessionId, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Event $events) {
$authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$sessionId = ($sessionId === 'current')
? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret)
? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret, $authDuration)
: $sessionId;
$sessions = $user->getAttribute('sessions', []);
@ -1818,6 +1835,10 @@ App::patch('/v1/account/sessions/:sessionId')
$dbForProject->deleteCachedDocument('users', $user->getId());
$authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$session->setAttribute('expire', DateTime::addSeconds(new \DateTime($session->getCreatedAt()), $authDuration));
$events
->setParam('userId', $user->getId())
->setParam('sessionId', $session->getId())
@ -1871,6 +1892,7 @@ App::delete('/v1/account/sessions')
if ($session->getAttribute('secret') == Auth::hash(Auth::$secret)) {
$session->setAttribute('current', true);
$session->setAttribute('expire', DateTime::addSeconds(new \DateTime($session->getCreatedAt()), Auth::TOKEN_EXPIRATION_LOGIN_LONG));
// If current session delete the cookies too
$response

1
app/controllers/api/avatars.php
View file

@ -342,7 +342,6 @@ App::get('/v1/avatars/initials')
->desc('Get User Initials')
->groups(['api', 'avatars'])
->label('scope', 'avatars.read')
->label('cache', true)
->label('cache.resource', 'avatar/initials')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'avatars')

18
app/controllers/api/databases.php
View file

@ -2465,8 +2465,8 @@ App::get('/v1/databases/usage')
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
@ -2527,7 +2527,7 @@ App::get('/v1/databases/usage')
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
@ -2584,8 +2584,8 @@ App::get('/v1/databases/:databaseId/usage')
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
@ -2641,7 +2641,7 @@ App::get('/v1/databases/:databaseId/usage')
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
@ -2704,8 +2704,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
@ -2756,7 +2756,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [

12
app/controllers/api/functions.php
View file

@ -237,8 +237,8 @@ App::get('/v1/functions/:functionId/usage')
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
@ -292,7 +292,7 @@ App::get('/v1/functions/:functionId/usage')
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
@ -340,8 +340,8 @@ App::get('/v1/functions/usage')
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
@ -395,7 +395,7 @@ App::get('/v1/functions/usage')
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [

46
app/controllers/api/projects.php
View file

@ -32,6 +32,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Projects;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Boolean;
use Utopia\Validator\Hostname;
use Utopia\Validator\Integer;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
@ -80,7 +81,7 @@ App::post('/v1/projects')
}
$auth = Config::getParam('auth', []);
$auths = ['limit' => 0];
$auths = ['limit' => 0, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG];
foreach ($auth as $index => $method) {
$auths[$method['key'] ?? ''] = true;
}
@ -271,8 +272,8 @@ App::get('/v1/projects/:projectId/usage')
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
@ -295,9 +296,10 @@ App::get('/v1/projects/:projectId/usage')
'project.$all.network.bandwidth',
'project.$all.storage.size',
'users.$all.count.total',
'collections.$all.count.total',
'databases.$all.count.total',
'documents.$all.count.total',
'executions.$all.compute.total',
'buckets.$all.count.total'
];
$stats = [];
@ -327,7 +329,7 @@ App::get('/v1/projects/:projectId/usage')
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
@ -346,9 +348,10 @@ App::get('/v1/projects/:projectId/usage')
'network' => $stats[$metrics[1]] ?? [],
'storage' => $stats[$metrics[2]] ?? [],
'users' => $stats[$metrics[3]] ?? [],
'collections' => $stats[$metrics[4]] ?? [],
'databases' => $stats[$metrics[4]] ?? [],
'documents' => $stats[$metrics[5]] ?? [],
'executions' => $stats[$metrics[6]] ?? [],
'buckets' => $stats[$metrics[7]] ?? [],
]);
}
@ -508,6 +511,37 @@ App::patch('/v1/projects/:projectId/auth/limit')
$response->dynamic($project, Response::MODEL_PROJECT);
});
App::patch('/v1/projects/:projectId/auth/duration')
->desc('Update Project Authentication Duration')
->groups(['api', 'projects'])
->label('scope', 'projects.write')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'projects')
->label('sdk.method', 'updateAuthDuration')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_PROJECT)
->param('projectId', '', new UID(), 'Project unique ID.')
->param('duration', 31536000, new Range(0, 31536000), 'Project session length in seconds. Max length: 31536000 seconds.')
->inject('response')
->inject('dbForConsole')
->action(function (string $projectId, int $duration, Response $response, Database $dbForConsole) {
$project = $dbForConsole->getDocument('projects', $projectId);
if ($project->isEmpty()) {
throw new Exception(Exception::PROJECT_NOT_FOUND);
}
$auths = $project->getAttribute('auths', []);
$auths['duration'] = $duration;
$dbForConsole->updateDocument('projects', $project->getId(), $project
->setAttribute('auths', $auths));
$response->dynamic($project, Response::MODEL_PROJECT);
});
App::patch('/v1/projects/:projectId/auth/:method')
->desc('Update Project auth method status. Use this endpoint to enable or disable a given auth method for this project.')
->groups(['api', 'projects'])

16
app/controllers/api/storage.php
View file

@ -783,6 +783,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('cache', true)
->label('cache.resourceType', 'bucket/{request.bucketId}')
->label('cache.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
@ -840,9 +841,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$outputs = Config::getParam('storage-outputs');
$fileLogos = Config::getParam('storage-logos');
$date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT'; // 45 days cache
$key = \md5($fileId . $width . $height . $gravity . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $output);
if ($fileSecurity && !$valid) {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
} else {
@ -1454,8 +1452,8 @@ App::get('/v1/storage/usage')
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
@ -1513,7 +1511,7 @@ App::get('/v1/storage/usage')
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
@ -1571,8 +1569,8 @@ App::get('/v1/storage/:bucketId/usage')
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
@ -1624,7 +1622,7 @@ App::get('/v1/storage/:bucketId/usage')
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [

7
app/controllers/api/teams.php
View file

@ -677,9 +677,10 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status')
->inject('response')
->inject('user')
->inject('dbForProject')
->inject('project')
->inject('geodb')
->inject('events')
->action(function (string $teamId, string $membershipId, string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Reader $geodb, Event $events) {
->action(function (string $teamId, string $membershipId, string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Reader $geodb, Event $events) {
$protocol = $request->getProtocol();
$membership = $dbForProject->getDocument('memberships', $membershipId);
@ -731,7 +732,8 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status')
$detector = new Detector($request->getUserAgent('UNKNOWN'));
$record = $geodb->get($request->getIP());
$expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_LOGIN_LONG);
$authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expire = DateTime::addSeconds(new \DateTime(), $authDuration);
$secret = Auth::tokenGenerator();
$session = new Document(array_merge([
'$id' => ID::unique(),
@ -740,7 +742,6 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status')
'provider' => Auth::SESSION_PROVIDER_EMAIL,
'providerUid' => $user->getAttribute('email'),
'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak
'expire' => $expire,
'userAgent' => $request->getUserAgent('UNKNOWN'),
'ip' => $request->getIP(),
'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--',

6
app/controllers/api/users.php
View file

@ -1116,8 +1116,8 @@ App::get('/v1/users/usage')
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
@ -1171,7 +1171,7 @@ App::get('/v1/users/usage')
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [

100
app/controllers/shared/api.php
View file

@ -16,6 +16,7 @@ use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
@ -47,6 +48,47 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
return $label;
};
$databaseListener = function (string $event, Document $document, Stats $usage) {
$multiplier = 1;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$multiplier = -1;
}
$collection = $document->getCollection();
switch ($collection) {
case 'users':
$usage->setParam('users.{scope}.count.total', 1 * $multiplier);
break;
case 'databases':
$usage->setParam('databases.{scope}.count.total', 1 * $multiplier);
break;
case 'buckets':
$usage->setParam('buckets.{scope}.count.total', 1 * $multiplier);
break;
case 'deployments':
$usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier);
break;
default:
if (strpos($collection, 'bucket_') === 0) {
$usage
->setParam('bucketId', $document->getAttribute('bucketId'))
->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier)
->setParam('files.{scope}.count.total', 1 * $multiplier);
} elseif (strpos($collection, 'database_') === 0) {
$usage
->setParam('databaseId', $document->getAttribute('databaseId'));
if (strpos($collection, '_collection_') !== false) {
$usage
->setParam('collectionId', $document->getAttribute('$collectionId'))
->setParam('documents.{scope}.count.total', 1 * $multiplier);
} else {
$usage->setParam('collections.{scope}.count.total', 1 * $multiplier);
}
}
break;
}
};
App::init()
->groups(['api'])
->inject('utopia')
@ -62,7 +104,7 @@ App::init()
->inject('database')
->inject('dbForProject')
->inject('mode')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Mail $mails, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Mail $mails, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode) use ($databaseListener) {
$route = $utopia->match($request);
@ -149,6 +191,7 @@ App::init()
->setUser($user);
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('project.{scope}.network.requests', 1)
->setParam('httpMethod', $request->getMethod())
@ -158,22 +201,59 @@ App::init()
$deletes->setProject($project);
$database->setProject($project);
$dbForProject->on(Database::EVENT_DOCUMENT_CREATE, fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject->on(Database::EVENT_DOCUMENT_DELETE, fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$useCache = $route->getLabel('cache', false);
if ($useCache) {
$key = md5($request->getURI() . implode('*', $request->getParams()));
$key = md5($request->getURI() . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER;
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())
);
$timestamp = 60 * 60 * 24 * 30;
$data = $cache->load($key, $timestamp);
if (!empty($data)) {
$data = json_decode($data, true);
$parts = explode('/', $data['resourceType']);
$type = $parts[0] ?? null;
if ($type === 'bucket') {
$bucketId = $parts[1] ?? null;
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$fileSecurity = $bucket->getAttribute('fileSecurity', false);
$validator = new Authorization(Database::PERMISSION_READ);
$valid = $validator->isValid($bucket->getRead());
if (!$fileSecurity && !$valid) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
$parts = explode('/', $data['resource']);
$fileId = $parts[1] ?? null;
if ($fileSecurity && !$valid) {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
} else {
$file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
}
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
}
}
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $timestamp) . ' GMT')
->addHeader('X-Appwrite-Cache', 'hit')
->setContentType($data['content-type'])
->setContentType($data['contentType'])
->send(base64_decode($data['payload']))
;
@ -361,7 +441,7 @@ App::shutdown()
*/
$useCache = $route->getLabel('cache', false);
if ($useCache) {
$resource = null;
$resource = $resourceType = null;
$data = $response->getPayload();
if (!empty($data['payload'])) {
@ -370,9 +450,16 @@ App::shutdown()
$resource = $parseLabel($pattern, $responsePayload, $requestParams, $user);
}
$key = md5($request->getURI() . implode('*', $request->getParams()));
$pattern = $route->getLabel('cache.resourceType', null);
if (!empty($pattern)) {
$resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user);
}
$key = md5($request->getURI() . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER;
$data = json_encode([
'content-type' => $response->getContentType(),
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
]) ;
@ -404,7 +491,6 @@ App::shutdown()
if (
App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled'
&& $project->getId()
&& $mode !== APP_MODE_ADMIN // TODO: add check to make sure user is admin
&& !empty($route->getLabel('sdk.namespace', null))
) { // Don't calculate console usage on admin mode
$metric = $route->getLabel('usage.metric', '');

5
app/init.php
View file

@ -842,9 +842,11 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
$user = $dbForConsole->getDocument('users', Auth::$unique);
}
$authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
if (
$user->isEmpty() // Check a document has been found in the DB
|| !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret)
|| !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret, $authDuration)
) { // Validate user has valid login token
$user = new Document(['$id' => ID::custom(''), '$collection' => 'users']);
}
@ -926,6 +928,7 @@ App::setResource('console', function () {
'legalTaxId' => '',
'auths' => [
'limit' => (App::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user
'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds
],
'authWhitelistEmails' => (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [],
'authWhitelistIPs' => (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [],

14
app/realtime.php
View file

@ -306,7 +306,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) {
$connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId]));
[$consoleDatabase, $returnConsoleDatabase] = getDatabase($register, '_console');
$project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId));
$project = Authorization::skip(fn () => $consoleDatabase->getDocument('projects', $projectId));
[$database, $returnDatabase] = getDatabase($register, "_{$project->getInternalId()}");
$user = $database->getDocument('users', $userId);
@ -484,6 +484,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
$server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) {
try {
$app = new App('UTC');
$response = new Response(new SwooleResponse());
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
@ -493,12 +494,8 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace("_console");
$projectId = $realtime->connections[$connection]['projectId'];
if ($projectId !== 'console') {
$project = Authorization::skip(fn() => $database->getDocument('projects', $projectId));
$database->setNamespace("_{$project->getInternalId()}");
}
$project = $projectId === 'console' ? $app->getResource('console') : Authorization::skip(fn () => $database->getDocument('projects', $projectId));
$database->setNamespace("_{$project->getInternalId()}");
/*
* Abuse Check
*
@ -536,10 +533,11 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
Auth::$secret = $session['secret'] ?? '';
$user = $database->getDocument('users', Auth::$unique);
$authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
if (
empty($user->getId()) // Check a document has been found in the DB
|| !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret) // Validate user has valid login token
|| !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret, $authDuration) // Validate user has valid login token
) {
// cookie not valid
throw new Exception('Session is not valid.', 1003);

15
app/tasks/maintenance.php
View file

@ -78,12 +78,11 @@ $cli
->trigger();
}
function notifyDeleteUsageStats(int $interval30m, int $interval1d)
function notifyDeleteUsageStats(int $usageStatsRetentionHourly)
{
(new Delete())
->setType(DELETE_TYPE_USAGE)
->setDateTime1d(DateTime::addSeconds(new \DateTime(), -1 * $interval1d))
->setDateTime30m(DateTime::addSeconds(new \DateTime(), -1 * $interval30m))
->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly))
->trigger();
}
@ -99,7 +98,7 @@ $cli
{
(new Delete())
->setType(DELETE_TYPE_SESSIONS)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * Auth::TOKEN_EXPIRATION_LOGIN_LONG))
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * Auth::TOKEN_EXPIRATION_LOGIN_LONG)) //TODO: Update to use project session expiration instead of default.
->trigger();
}
@ -144,11 +143,11 @@ $cli
$executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
$auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600');
$abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
$usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours
$usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days
$usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) {
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetentionHourly, $cacheRetention) {
$database = getConsoleDB();
$time = DateTime::now();
@ -157,7 +156,7 @@ $cli
notifyDeleteExecutionLogs($executionLogsRetention);
notifyDeleteAbuseLogs($abuseLogsRetention);
notifyDeleteAuditLogs($auditLogRetention);
notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d);
notifyDeleteUsageStats($usageStatsRetentionHourly);
notifyDeleteConnections();
notifyDeleteExpiredSessions();
renewCertificates($database);

72
app/tasks/usage.php
View file

@ -2,10 +2,6 @@
global $cli, $register;
use Appwrite\Stats\Usage;
use Appwrite\Stats\UsageDB;
use Appwrite\Usage\Calculators\Aggregator;
use Appwrite\Usage\Calculators\Database;
use Appwrite\Usage\Calculators\TimeSeries;
use InfluxDB\Database as InfluxDatabase;
use Utopia\App;
@ -114,65 +110,29 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use (
Console::warning($error->getTraceAsString());
};
function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
{
$interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
$region = App::getEnv('region', 'default');
$usage = new TimeSeries($region, $database, $influxDB, $logError);
Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
$loopStart = microtime(true);
$usage->collect();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
}
function aggregateDatabase(UtopiaDatabase $database, callable $logError): void
{
$interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default)
$region = App::getEnv('region', 'default');
$usage = new Database($region, $database, $logError);
$aggregrator = new Aggregator($region, $database, $logError);
Console::loop(function () use ($interval, $usage, $aggregrator) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating database usage every {$interval} seconds.");
$loopStart = microtime(true);
$usage->collect();
$aggregrator->collect();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
}
$cli
->task('usage')
->param('type', 'timeseries', new WhiteList(['timeseries', 'database']))
->desc('Schedules syncing data from influxdb to Appwrite console db')
->action(function (string $type) use ($register, $logError) {
->action(function () use ($register, $logError) {
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$database = getDatabase($register, '_console');
$influxDB = getInfluxDB($register);
switch ($type) {
case 'timeseries':
aggregateTimeseries($database, $influxDB, $logError);
break;
case 'database':
aggregateDatabase($database, $logError);
break;
default:
Console::error("Unsupported usage aggregation type");
}
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
$region = App::getEnv('region', 'default');
$usage = new TimeSeries($region, $database, $influxDB, $logError);
Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
$loopStart = microtime(true);
$usage->collect();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
});

45
app/views/install/compose.phtml
View file

@ -150,6 +150,7 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_SMS_PROVIDER
- _APP_SMS_FROM
@ -549,13 +550,12 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
appwrite-usage-timeseries:
appwrite-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint:
- usage
- --type=timeseries
container_name: appwrite-usage-timeseries
entrypoint: usage
container_name: appwrite-usage
<<: *x-logging
restart: unless-stopped
networks:
@ -573,40 +573,7 @@ services:
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-usage-database:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint:
- usage
- --type=database
container_name: appwrite-usage-database
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER

18
app/workers/deletes.php
View file

@ -105,7 +105,7 @@ class DeletesV1 extends Worker
break;
case DELETE_TYPE_USAGE:
$this->deleteUsageStats($this->args['dateTime1d'], $this->args['dateTime30m']);
$this->deleteUsageStats($this->args['dateTime1d'], $this->args['hourlyUsageRetentionDatetime']);
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
@ -215,21 +215,15 @@ class DeletesV1 extends Worker
/**
* @param string $datetime1d
* @param string $datetime30m
* @param string $hourlyUsageRetentionDatetime
*/
protected function deleteUsageStats(string $datetime1d, string $datetime30m)
protected function deleteUsageStats(string $hourlyUsageRetentionDatetime)
{
$this->deleteForProjectIds(function (string $projectId) use ($datetime1d, $datetime30m) {
$this->deleteForProjectIds(function (string $projectId) use ($hourlyUsageRetentionDatetime) {
$dbForProject = $this->getProjectDB($projectId);
// Delete Usage stats
$this->deleteByGroup('stats', [
Query::lessThan('time', $datetime1d),
Query::equal('period', ['1d']),
], $dbForProject);
$this->deleteByGroup('stats', [
Query::lessThan('time', $datetime30m),
Query::equal('period', ['30m']),
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),
], $dbForProject);
});
}

1
app/workers/functions.php
View file

@ -368,6 +368,7 @@ class FunctionsV1 extends Worker
$usage = new Stats($statsd);
$usage
->setParam('projectId', $project->getId())
->setParam('projectInternalId', $project->getInternalId())
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))

148
composer.lock generated
View file

@ -115,15 +115,15 @@
},
{
"name": "appwrite/php-runtimes",
"version": "0.11.0",
"version": "0.11.1",
"source": {
"type": "git",
"url": "https://github.com/appwrite/runtimes.git",
"reference": "547fc026e11c0946846a8ac690898f5bf53be101"
"reference": "9d74a477ba3333cbcfac565c46fcf19606b7b603"
},
"require": {
"php": ">=8.0",
"utopia-php/system": "0.4.*"
"utopia-php/system": "0.6.*"
},
"require-dev": {
"phpunit/phpunit": "^9.3",
@ -154,7 +154,7 @@
"php",
"runtimes"
],
"time": "2022-08-15T14:03:36+00:00"
"time": "2022-11-07T16:45:52+00:00"
},
{
"name": "chillerlan/php-qrcode",
@ -300,16 +300,16 @@
},
{
"name": "colinmollenhour/credis",
"version": "v1.13.1",
"version": "v1.14.0",
"source": {
"type": "git",
"url": "https://github.com/colinmollenhour/credis.git",
"reference": "85df015088e00daf8ce395189de22c8eb45c8d49"
"reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/colinmollenhour/credis/zipball/85df015088e00daf8ce395189de22c8eb45c8d49",
"reference": "85df015088e00daf8ce395189de22c8eb45c8d49",
"url": "https://api.github.com/repos/colinmollenhour/credis/zipball/dccc8a46586475075fbb012d8bd523b8a938c2dc",
"reference": "dccc8a46586475075fbb012d8bd523b8a938c2dc",
"shasum": ""
},
"require": {
@ -341,9 +341,9 @@
"homepage": "https://github.com/colinmollenhour/credis",
"support": {
"issues": "https://github.com/colinmollenhour/credis/issues",
"source": "https://github.com/colinmollenhour/credis/tree/v1.13.1"
"source": "https://github.com/colinmollenhour/credis/tree/v1.14.0"
},
"time": "2022-06-20T22:56:59+00:00"
"time": "2022-11-09T01:18:39+00:00"
},
{
"name": "dragonmantank/cron-expression",
@ -803,6 +803,72 @@
},
"time": "2020-12-26T17:45:17+00:00"
},
{
"name": "laravel/pint",
"version": "v1.2.0",
"source": {
"type": "git",
"url": "https://github.com/laravel/pint.git",
"reference": "1d276e4c803397a26cc337df908f55c2a4e90d86"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86",
"reference": "1d276e4c803397a26cc337df908f55c2a4e90d86",
"shasum": ""
},
"require": {
"ext-json": "*",
"ext-mbstring": "*",
"ext-tokenizer": "*",
"ext-xml": "*",
"php": "^8.0"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "^3.11.0",
"illuminate/view": "^9.27",
"laravel-zero/framework": "^9.1.3",
"mockery/mockery": "^1.5.0",
"nunomaduro/larastan": "^2.2",
"nunomaduro/termwind": "^1.14.0",
"pestphp/pest": "^1.22.1"
},
"bin": [
"builds/pint"
],
"type": "project",
"autoload": {
"psr-4": {
"App\\": "app/",
"Database\\Seeders\\": "database/seeders/",
"Database\\Factories\\": "database/factories/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nuno Maduro",
"email": "enunomaduro@gmail.com"
}
],
"description": "An opinionated code formatter for PHP.",
"homepage": "https://laravel.com",
"keywords": [
"format",
"formatter",
"lint",
"linter",
"php"
],
"support": {
"issues": "https://github.com/laravel/pint/issues",
"source": "https://github.com/laravel/pint"
},
"time": "2022-09-13T15:07:15+00:00"
},
{
"name": "matomo/device-detector",
"version": "6.0.0",
@ -2411,23 +2477,25 @@
},
{
"name": "utopia-php/system",
"version": "0.4.0",
"version": "0.6.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/system.git",
"reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0"
"reference": "289c4327713deadc9c748b5317d248133a02f245"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/system/zipball/67c92c66ce8f0cc925a00bca89f7a188bf9183c0",
"reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0",
"url": "https://api.github.com/repos/utopia-php/system/zipball/289c4327713deadc9c748b5317d248133a02f245",
"reference": "289c4327713deadc9c748b5317d248133a02f245",
"shasum": ""
},
"require": {
"laravel/pint": "1.2.*",
"php": ">=7.4"
},
"require-dev": {
"phpunit/phpunit": "^9.3",
"squizlabs/php_codesniffer": "^3.6",
"vimeo/psalm": "4.0.1"
},
"type": "library",
@ -2460,9 +2528,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/system/issues",
"source": "https://github.com/utopia-php/system/tree/0.4.0"
"source": "https://github.com/utopia-php/system/tree/0.6.0"
},
"time": "2021-02-04T14:14:49+00:00"
"time": "2022-11-07T13:51:59+00:00"
},
{
"name": "utopia-php/websocket",
@ -2654,12 +2722,12 @@
"source": {
"type": "git",
"url": "https://github.com/appwrite/sdk-generator.git",
"reference": "4bbff1538724274a92b74e39cf4cda4580bafb68"
"reference": "c240f93972eeea57443c97974d33811db585d537"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/4bbff1538724274a92b74e39cf4cda4580bafb68",
"reference": "4bbff1538724274a92b74e39cf4cda4580bafb68",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/c240f93972eeea57443c97974d33811db585d537",
"reference": "c240f93972eeea57443c97974d33811db585d537",
"shasum": ""
},
"require": {
@ -2697,7 +2765,7 @@
"issues": "https://github.com/appwrite/sdk-generator/issues",
"source": "https://github.com/appwrite/sdk-generator/tree/feat-graphql"
},
"time": "2022-10-26T06:49:27+00:00"
"time": "2022-11-16T02:54:18+00:00"
},
{
"name": "doctrine/instantiator",
@ -2953,16 +3021,16 @@
},
{
"name": "nikic/php-parser",
"version": "v4.15.1",
"version": "v4.15.2",
"source": {
"type": "git",
"url": "https://github.com/nikic/PHP-Parser.git",
"reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900"
"reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/0ef6c55a3f47f89d7a374e6f835197a0b5fcf900",
"reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900",
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc",
"reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc",
"shasum": ""
},
"require": {
@ -3003,9 +3071,9 @@
],
"support": {
"issues": "https://github.com/nikic/PHP-Parser/issues",
"source": "https://github.com/nikic/PHP-Parser/tree/v4.15.1"
"source": "https://github.com/nikic/PHP-Parser/tree/v4.15.2"
},
"time": "2022-09-04T07:30:47+00:00"
"time": "2022-11-12T15:38:23+00:00"
},
{
"name": "phar-io/manifest",
@ -4835,16 +4903,16 @@
},
{
"name": "symfony/polyfill-ctype",
"version": "v1.26.0",
"version": "v1.27.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-ctype.git",
"reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4"
"reference": "5bbc823adecdae860bb64756d639ecfec17b050a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4",
"reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4",
"url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/5bbc823adecdae860bb64756d639ecfec17b050a",
"reference": "5bbc823adecdae860bb64756d639ecfec17b050a",
"shasum": ""
},
"require": {
@ -4859,7 +4927,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.26-dev"
"dev-main": "1.27-dev"
},
"thanks": {
"name": "symfony/polyfill",
@ -4897,7 +4965,7 @@
"portable"
],
"support": {
"source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0"
"source": "https://github.com/symfony/polyfill-ctype/tree/v1.27.0"
},
"funding": [
{
@ -4913,20 +4981,20 @@
"type": "tidelift"
}
],
"time": "2022-05-24T11:49:31+00:00"
"time": "2022-11-03T14:55:06+00:00"
},
{
"name": "symfony/polyfill-mbstring",
"version": "v1.26.0",
"version": "v1.27.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-mbstring.git",
"reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e"
"reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e",
"reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/8ad114f6b39e2c98a8b0e3bd907732c207c2b534",
"reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534",
"shasum": ""
},
"require": {
@ -4941,7 +5009,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.26-dev"
"dev-main": "1.27-dev"
},
"thanks": {
"name": "symfony/polyfill",
@ -4980,7 +5048,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-mbstring/tree/v1.26.0"
"source": "https://github.com/symfony/polyfill-mbstring/tree/v1.27.0"
},
"funding": [
{
@ -4996,7 +5064,7 @@
"type": "tidelift"
}
],
"time": "2022-05-24T11:49:31+00:00"
"time": "2022-11-03T14:55:06+00:00"
},
{
"name": "textalk/websocket",

47
docker-compose.yml
View file

@ -170,6 +170,7 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_SMS_PROVIDER
- _APP_SMS_FROM
- _APP_GRAPHQL_MAX_COMPLEXITY
@ -603,13 +604,12 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
appwrite-usage-timeseries:
entrypoint:
- usage
- --type=timeseries
appwrite-usage:
entrypoint: usage
<<: *x-logging
container_name: appwrite-usage-timeseries
container_name: appwrite-usage
image: appwrite-dev
networks:
- appwrite
@ -629,42 +629,7 @@ services:
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-usage-database:
entrypoint:
- usage
- --type=database
<<: *x-logging
container_name: appwrite-usage-database
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER

121
package-lock.json generated
View file

@ -433,12 +433,15 @@
}
},
"node_modules/buffer-equal": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz",
"integrity": "sha512-tcBWO2Dl4e7Asr9hTGcpVrCe+F7DubpmqWCTbj4FHLmjqO2hIaC383acQubWtRJhdceqs5uBHs6Es+Sk//RKiQ==",
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.1.tgz",
"integrity": "sha512-QoV3ptgEaQpvVwbXdSO39iqPQTCxSF7A5U99AxbHYqUdCizL/lH2Z0A2y6nbZucxMEOtNyZfG2s6gsVugGpKkg==",
"dev": true,
"engines": {
"node": ">=0.4.0"
"node": ">=0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/buffer-from": {
@ -805,13 +808,10 @@
}
},
"node_modules/convert-source-map": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz",
"integrity": "sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA==",
"dev": true,
"dependencies": {
"safe-buffer": "~5.1.1"
}
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz",
"integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==",
"dev": true
},
"node_modules/copy-anything": {
"version": "2.0.6",
@ -1540,9 +1540,9 @@
"dev": true
},
"node_modules/get-intrinsic": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz",
"integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==",
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz",
"integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==",
"dev": true,
"dependencies": {
"function-bind": "^1.1.1",
@ -2196,9 +2196,9 @@
"dev": true
},
"node_modules/is-core-module": {
"version": "2.10.0",
"resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.10.0.tgz",
"integrity": "sha512-Erxj2n/LDAZ7H8WNJXd9tw38GYM3dv8rk8Zcs+jJuxYTW7sozH+SS8NtrSjVL1/vpLvWi1hxy96IzjJ3EHTJJg==",
"version": "2.11.0",
"resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz",
"integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==",
"dev": true,
"dependencies": {
"has": "^1.0.3"
@ -2912,10 +2912,13 @@
}
},
"node_modules/meow/node_modules/minimist": {
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
"integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
"dev": true
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz",
"integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==",
"dev": true,
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/micromatch": {
"version": "3.1.10",
@ -3013,10 +3016,13 @@
}
},
"node_modules/minimist": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.1.tgz",
"integrity": "sha512-GY8fANSrTMfBVfInqJAY41QkOM+upUTytK1jZ0c8+3HdHrJxBJ3rF5i9moClXTE8uUSnUo8cAsCoxDXvSY4DHg==",
"dev": true
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.2.tgz",
"integrity": "sha512-g92kDfAOAszDRtHNagjZPPI/9lfOFaRBL/Ud6Z0RKZua/x+49awTydZLh5Gkhb80Xy5hmcvZNLGzscW5n5yd0g==",
"dev": true,
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/mixin-deep": {
"version": "1.3.2",
@ -3080,9 +3086,9 @@
}
},
"node_modules/nan": {
"version": "2.16.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.16.0.tgz",
"integrity": "sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA==",
"version": "2.17.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz",
"integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==",
"dev": true,
"optional": true
},
@ -4854,9 +4860,9 @@
}
},
"node_modules/tslib": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz",
"integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==",
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz",
"integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==",
"dev": true
},
"node_modules/turndown": {
@ -5636,9 +5642,9 @@
}
},
"buffer-equal": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz",
"integrity": "sha512-tcBWO2Dl4e7Asr9hTGcpVrCe+F7DubpmqWCTbj4FHLmjqO2hIaC383acQubWtRJhdceqs5uBHs6Es+Sk//RKiQ==",
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.1.tgz",
"integrity": "sha512-QoV3ptgEaQpvVwbXdSO39iqPQTCxSF7A5U99AxbHYqUdCizL/lH2Z0A2y6nbZucxMEOtNyZfG2s6gsVugGpKkg==",
"dev": true
},
"buffer-from": {
@ -5941,13 +5947,10 @@
}
},
"convert-source-map": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz",
"integrity": "sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA==",
"dev": true,
"requires": {
"safe-buffer": "~5.1.1"
}
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz",
"integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==",
"dev": true
},
"copy-anything": {
"version": "2.0.6",
@ -6559,9 +6562,9 @@
"dev": true
},
"get-intrinsic": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz",
"integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==",
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz",
"integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==",
"dev": true,
"requires": {
"function-bind": "^1.1.1",
@ -7092,9 +7095,9 @@
"dev": true
},
"is-core-module": {
"version": "2.10.0",
"resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.10.0.tgz",
"integrity": "sha512-Erxj2n/LDAZ7H8WNJXd9tw38GYM3dv8rk8Zcs+jJuxYTW7sozH+SS8NtrSjVL1/vpLvWi1hxy96IzjJ3EHTJJg==",
"version": "2.11.0",
"resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz",
"integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==",
"dev": true,
"requires": {
"has": "^1.0.3"
@ -7693,9 +7696,9 @@
},
"dependencies": {
"minimist": {
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
"integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz",
"integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==",
"dev": true
}
}
@ -7774,9 +7777,9 @@
}
},
"minimist": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.1.tgz",
"integrity": "sha512-GY8fANSrTMfBVfInqJAY41QkOM+upUTytK1jZ0c8+3HdHrJxBJ3rF5i9moClXTE8uUSnUo8cAsCoxDXvSY4DHg==",
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.2.tgz",
"integrity": "sha512-g92kDfAOAszDRtHNagjZPPI/9lfOFaRBL/Ud6Z0RKZua/x+49awTydZLh5Gkhb80Xy5hmcvZNLGzscW5n5yd0g==",
"dev": true
},
"mixin-deep": {
@ -7831,9 +7834,9 @@
"dev": true
},
"nan": {
"version": "2.16.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.16.0.tgz",
"integrity": "sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA==",
"version": "2.17.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz",
"integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==",
"dev": true,
"optional": true
},
@ -9271,9 +9274,9 @@
"dev": true
},
"tslib": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz",
"integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==",
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz",
"integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==",
"dev": true
},
"turndown": {

2
phpcs.xml
View file

@ -7,6 +7,8 @@
<ini name="memory_limit" value="4096M"/>
<!-- Exclude SDK's for performance reasons -->
<exclude-pattern>./app/sdks</exclude-pattern>
<!-- Exclude console -->
<exclude-pattern>./app/console</exclude-pattern>
<!-- Ignore max line width -->
<rule ref="Generic.Files.LineLength">
<exclude-pattern>*</exclude-pattern>

6
src/Appwrite/Auth/Auth.php
View file

@ -352,19 +352,19 @@ class Auth
*
* @param array $sessions
* @param string $secret
* @param string $expires
*
* @return bool|string
*/
public static function sessionVerify(array $sessions, string $secret)
public static function sessionVerify(array $sessions, string $secret, int $expires)
{
foreach ($sessions as $session) {
/** @var Document $session */
if (
$session->isSet('secret') &&
$session->isSet('expire') &&
$session->isSet('provider') &&
$session->getAttribute('secret') === self::hash($secret) &&
DateTime::formatTz($session->getAttribute('expire')) >= DateTime::formatTz(DateTime::now())
DateTime::formatTz(DateTime::addSeconds(new \DateTime($session->getCreatedAt()), $expires)) >= DateTime::formatTz(DateTime::now())
) {
return $session->getId();
}

24
src/Appwrite/Event/Delete.php
View file

@ -11,8 +11,7 @@ class Delete extends Event
protected ?Document $document = null;
protected ?string $resource = null;
protected ?string $datetime = null;
protected ?string $dateTime30m = null;
protected ?string $dateTime1d = null;
protected ?string $hourlyUsageRetentionDatetime = null;
public function __construct()
@ -56,26 +55,14 @@ class Delete extends Event
}
/**
* Set datetime for 1 day interval.
* Sets datetime for 1h interval.
*
* @param string $datetime
* @return self
*/
public function setDateTime1d(string $datetime): self
public function setUsageRetentionHourlyDateTime(string $datetime): self
{
$this->dateTime1d = $datetime;
return $this;
}
/**
* Sets datetime for 30m interval.
*
* @param string $datetime
* @return self
*/
public function setDateTime30m(string $datetime): self
{
$this->dateTime30m = $datetime;
$this->hourlyUsageRetentionDatetime = $datetime;
return $this;
}
@ -140,8 +127,7 @@ class Delete extends Event
'document' => $this->document,
'resource' => $this->resource,
'datetime' => $this->datetime,
'dateTime1d' => $this->dateTime1d,
'dateTime30m' => $this->dateTime30m,
'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime,
]);
}
}

3
src/Appwrite/Migration/Migration.php
View file

@ -44,7 +44,8 @@ abstract class Migration
'1.0.0-RC1' => 'V15',
'1.0.0' => 'V15',
'1.0.1' => 'V15',
'1.0.3' => 'V15'
'1.0.3' => 'V15',
'1.1.0' => 'V16',
];
/**

116
src/Appwrite/Migration/Version/V16.php Normal file
View file

@ -0,0 +1,116 @@
<?php
namespace Appwrite\Migration\Version;
use Appwrite\Auth\Auth;
use Appwrite\Migration\Migration;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
class V16 extends Migration
{
public function execute(): void
{
/**
* Disable SubQueries for Performance.
*/
foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subqueryVariables'] as $name) {
Database::addFilter(
$name,
fn () => null,
fn () => []
);
}
Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
Console::info('Migrating Collections');
$this->migrateCollections();
Console::info('Migrating Documents');
$this->forEachDocument([$this, 'fixDocument']);
}
/**
* Migrate all Collections.
*
* @return void
*/
protected function migrateCollections(): void
{
foreach ($this->collections as $collection) {
$id = $collection['$id'];
Console::log("Migrating Collection \"{$id}\"");
$this->projectDB->setNamespace("_{$this->project->getInternalId()}");
switch ($id) {
case 'sessions':
try {
/**
* Create 'compression' attribute
*/
$this->projectDB->deleteAttribute($id, 'expire');
} catch (\Throwable $th) {
Console::warning("'expire' from {$id}: {$th->getMessage()}");
}
break;
case 'projects':
try {
/**
* Create 'region' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'region');
} catch (\Throwable $th) {
Console::warning("'region' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create '_key_team' index
*/
$this->createIndexFromCollection($this->projectDB, $id, '_key_team');
} catch (\Throwable $th) {
Console::warning("'_key_team' from {$id}: {$th->getMessage()}");
}
break;
default:
break;
}
usleep(50000);
}
}
/**
* Fix run on each document
*
* @param \Utopia\Database\Document $document
* @return \Utopia\Database\Document
*/
protected function fixDocument(Document $document)
{
switch ($document->getCollection()) {
case 'projects':
/**
* Bump version number.
*/
$document->setAttribute('version', '1.1.0');
/**
* Set default authDuration
*/
$document->setAttribute('auths', array_merge($document->getAttribute('auths', []), [
'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG
]));
break;
}
return $document;
}
}

231
src/Appwrite/Usage/Calculators/Aggregator.php
View file

@ -1,231 +0,0 @@
<?php
namespace Appwrite\Usage\Calculators;
use DateTime;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Database\Document;
use Utopia\Database\Query;
class Aggregator extends Database
{
protected function aggregateDatabaseMetrics(string $projectId): void
{
$this->database->setNamespace('_' . $projectId);
$databasesGeneralMetrics = [
'databases.$all.requests.create',
'databases.$all.requests.read',
'databases.$all.requests.update',
'databases.$all.requests.delete',
'collections.$all.requests.create',
'collections.$all.requests.read',
'collections.$all.requests.update',
'collections.$all.requests.delete',
'documents.$all.requests.create',
'documents.$all.requests.read',
'documents.$all.requests.update',
'documents.$all.requests.delete'
];
foreach ($databasesGeneralMetrics as $metric) {
$this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($projectId, $metric);
}
$databasesDatabaseMetrics = [
'collections.databaseId.requests.create',
'collections.databaseId.requests.read',
'collections.databaseId.requests.update',
'collections.databaseId.requests.delete',
'documents.databaseId.requests.create',
'documents.databaseId.requests.read',
'documents.databaseId.requests.update',
'documents.databaseId.requests.delete',
];
$this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) {
$databaseId = $database->getId();
foreach ($databasesDatabaseMetrics as $metric) {
$metric = str_replace('databaseId', $databaseId, $metric);
$this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($projectId, $metric);
}
$databasesCollectionMetrics = [
'documents.' . $databaseId . '/collectionId.requests.create',
'documents.' . $databaseId . '/collectionId.requests.read',
'documents.' . $databaseId . '/collectionId.requests.update',
'documents.' . $databaseId . '/collectionId.requests.delete',
];
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) {
$collectionId = $collection->getId();
foreach ($databasesCollectionMetrics as $metric) {
$metric = str_replace('collectionId', $collectionId, $metric);
$this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($projectId, $metric);
}
});
});
}
protected function aggregateStorageMetrics(string $projectId): void
{
$this->database->setNamespace('_' . $projectId);
$storageGeneralMetrics = [
'buckets.$all.requests.create',
'buckets.$all.requests.read',
'buckets.$all.requests.update',
'buckets.$all.requests.delete',
'files.$all.requests.create',
'files.$all.requests.read',
'files.$all.requests.update',
'files.$all.requests.delete',
];
foreach ($storageGeneralMetrics as $metric) {
$this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($projectId, $metric);
}
$storageBucketMetrics = [
'files.bucketId.requests.create',
'files.bucketId.requests.read',
'files.bucketId.requests.update',
'files.bucketId.requests.delete',
];
$this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) {
$bucketId = $bucket->getId();
foreach ($storageBucketMetrics as $metric) {
$metric = str_replace('bucketId', $bucketId, $metric);
$this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($projectId, $metric);
}
});
}
protected function aggregateFunctionMetrics(string $projectId): void
{
$this->database->setNamespace('_' . $projectId);
$functionsGeneralMetrics = [
'project.$all.compute.total',
'project.$all.compute.time',
'executions.$all.compute.total',
'executions.$all.compute.success',
'executions.$all.compute.failure',
'executions.$all.compute.time',
'builds.$all.compute.total',
'builds.$all.compute.success',
'builds.$all.compute.failure',
'builds.$all.compute.time',
];
foreach ($functionsGeneralMetrics as $metric) {
$this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($projectId, $metric);
}
$functionMetrics = [
'executions.functionId.compute.total',
'executions.functionId.compute.success',
'executions.functionId.compute.failure',
'executions.functionId.compute.time',
'builds.functionId.compute.total',
'builds.functionId.compute.success',
'builds.functionId.compute.failure',
'builds.functionId.compute.time',
];
$this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) {
$functionId = $function->getId();
foreach ($functionMetrics as $metric) {
$metric = str_replace('functionId', $functionId, $metric);
$this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($projectId, $metric);
}
});
}
protected function aggregateUsersMetrics(string $projectId): void
{
$metrics = [
'users.$all.requests.create',
'users.$all.requests.read',
'users.$all.requests.update',
'users.$all.requests.delete',
'sessions.$all.requests.create',
'sessions.$all.requests.delete'
];
foreach ($metrics as $metric) {
$this->aggregateDailyMetric($projectId, $metric);
$this->aggregateMonthlyMetric($projectId, $metric);
}
}
protected function aggregateGeneralMetrics(string $projectId): void
{
$this->aggregateDailyMetric($projectId, 'project.$all.network.requests');
$this->aggregateDailyMetric($projectId, 'project.$all.network.bandwidth');
$this->aggregateDailyMetric($projectId, 'project.$all.network.inbound');
$this->aggregateDailyMetric($projectId, 'project.$all.network.outbound');
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.requests');
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.bandwidth');
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.inbound');
$this->aggregateMonthlyMetric($projectId, 'project.$all.network.outbound');
}
protected function aggregateDailyMetric(string $projectId, string $metric): void
{
$beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339);
$endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339);
$this->database->setNamespace('_' . $projectId);
$value = (int) $this->database->sum('stats', 'value', [
Query::equal('metric', [$metric]),
Query::equal('period', ['30m']),
Query::greaterThanEqual('time', $beginOfDay),
Query::lessThanEqual('time', $endOfDay),
]);
$this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value);
}
protected function aggregateMonthlyMetric(string $projectId, string $metric): void
{
$beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
$endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339);
$this->database->setNamespace('_' . $projectId);
$value = (int) $this->database->sum('stats', 'value', [
Query::equal('metric', [$metric]),
Query::equal('period', ['1d']),
Query::greaterThanEqual('time', $beginOfMonth),
Query::lessThanEqual('time', $endOfMonth),
]);
$this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value);
}
/**
* Collect Stats
* Collect all database related stats
*
* @return void
*/
public function collect(): void
{
$this->foreachDocument('console', 'projects', [], function (Document $project) {
$projectId = $project->getInternalId();
// Aggregate new metrics from already collected usage metrics
// for lower time period (1day and 1 month metric from 30 minute metrics)
$this->aggregateGeneralMetrics($projectId);
$this->aggregateFunctionMetrics($projectId);
$this->aggregateDatabaseMetrics($projectId);
$this->aggregateStorageMetrics($projectId);
$this->aggregateUsersMetrics($projectId);
});
}
}

364
src/Appwrite/Usage/Calculators/Database.php
View file

@ -1,364 +0,0 @@
<?php
namespace Appwrite\Usage\Calculators;
use Exception;
use Utopia\App;
use Appwrite\Usage\Calculator;
use DateTime;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Database\Document;
use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Query;
class Database extends Calculator
{
protected array $periods = [
[
'key' => '30m',
'multiplier' => 1800,
],
[
'key' => '1d',
'multiplier' => 86400,
],
];
public function __construct(string $region, UtopiaDatabase $database, callable $errorHandler = null)
{
parent::__construct($region);
$this->database = $database;
$this->errorHandler = $errorHandler;
}
/**
* Create Per Period Metric
*
* Create given metric for each defined period
*
* @param string $projectId
* @param string $metric
* @param int $value
* @param bool $monthly
* @return void
* @throws Authorization
* @throws Structure
*/
protected function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void
{
foreach ($this->periods as $options) {
$period = $options['key'];
$date = new \DateTime();
if ($period === '30m') {
$minutes = $date->format('i') >= '30' ? "30" : "00";
$time = $date->format('Y-m-d H:' . $minutes . ':00');
} elseif ($period === '1d') {
$time = $date->format('Y-m-d 00:00:00');
} else {
throw new Exception("Period type not found", 500);
}
$this->createOrUpdateMetric($projectId, $metric, $period, $time, $value);
}
// Required for billing
if ($monthly) {
$time = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339);
$this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value);
}
}
/**
* Create or Update Metric
*
* Create or update each metric in the stats collection for the given project
*
* @param string $projectId
* @param string $metric
* @param string $period
* @param string $time
* @param int $value
*
* @return void
* @throws Authorization
* @throws Structure
*/
protected function createOrUpdateMetric(string $projectId, string $metric, string $period, string $time, int $value): void
{
$id = \md5("{$time}_{$period}_{$metric}");
$this->database->setNamespace('_' . $projectId);
try {
$document = $this->database->getDocument('stats', $id);
if ($document->isEmpty()) {
$this->database->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $metric,
'value' => $value,
'region' => $this->region,
'type' => 2, // these are cumulative metrics
]));
} else {
$this->database->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $value)
);
}
} catch (\Exception$e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
} else {
throw $e;
}
}
}
/**
* Foreach Document
*
* Call provided callback for each document in the collection
*
* @param string $projectId
* @param string $collection
* @param array $queries
* @param callable $callback
*
* @return void
* @throws Exception
*/
protected function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void
{
$limit = 50;
$results = [];
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
try {
$paginationQueries = [Query::limit($limit)];
if ($latestDocument !== null) {
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$this->database->setNamespace('_' . $projectId);
$results = $this->database->find($collection, \array_merge($paginationQueries, $queries));
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}");
return;
} else {
throw $e;
}
}
if (empty($results)) {
return;
}
$sum = count($results);
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
}
$latestDocument = $results[array_key_last($results)];
}
}
/**
* Sum
*
* Calculate sum of an attribute of documents in collection
*
* @param string $projectId
* @param string $collection
* @param string $attribute
* @param string|null $metric
* @param int $multiplier
* @return int
* @throws Exception
*/
private function sum(string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int
{
$this->database->setNamespace('_' . $projectId);
try {
$sum = $this->database->sum($collection, $attribute);
$sum = (int) ($sum * $multiplier);
if (!is_null($metric)) {
$this->createPerPeriodMetric($projectId, $metric, $sum);
}
return $sum;
} catch (Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}");
} else {
throw $e;
}
}
return 0;
}
/**
* Count
*
* Count number of documents in collection
*
* @param string $projectId
* @param string $collection
* @param ?string $metric
*
* @return int
* @throws Exception
*/
private function count(string $projectId, string $collection, ?string $metric = null): int
{
$this->database->setNamespace('_' . $projectId);
try {
$count = $this->database->count($collection);
if (!is_null($metric)) {
$this->createPerPeriodMetric($projectId, (string) $metric, $count);
}
return $count;
} catch (Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}");
} else {
throw $e;
}
}
return 0;
}
/**
* Deployments Total
*
* Total sum of storage used by deployments
*
* @param string $projectId
*
* @return int
* @throws Exception
*/
private function deploymentsTotal(string $projectId): int
{
return $this->sum($projectId, 'deployments', 'size', 'deployments.$all.storage.size');
}
/**
* Users Stats
*
* Metric: users.count
*
* @param string $projectId
*
* @return void
* @throws Exception
*/
private function usersStats(string $projectId): void
{
$this->count($projectId, 'users', 'users.$all.count.total');
}
/**
* Storage Stats
*
* Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total,
* files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size
*
* @param string $projectId
*
* @return void
* @throws Authorization
* @throws Structure
*/
private function storageStats(string $projectId): void
{
$projectFilesTotal = 0;
$projectFilesCount = 0;
$metric = 'buckets.$all.count.total';
$this->count($projectId, 'buckets', $metric);
$this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) {
$metric = "files.{$bucket->getId()}.count.total";
$count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric);
$projectFilesCount += $count;
$metric = "files.{$bucket->getId()}.storage.size";
$sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric);
$projectFilesTotal += $sum;
});
$this->createPerPeriodMetric($projectId, 'files.$all.count.total', $projectFilesCount);
$this->createPerPeriodMetric($projectId, 'files.$all.storage.size', $projectFilesTotal);
$deploymentsTotal = $this->deploymentsTotal($projectId);
$this->createPerPeriodMetric($projectId, 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal);
}
/**
* Database Stats
*
* Collect all database stats
* Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total,
* documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total
*
* @param string $projectId
*
* @return void
* @throws Authorization
* @throws Structure
*/
private function databaseStats(string $projectId): void
{
$projectDocumentsCount = 0;
$projectCollectionsCount = 0;
$this->count($projectId, 'databases', 'databases.$all.count.total');
$this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) {
$metric = "collections.{$database->getId()}.count.total";
$count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric);
$projectCollectionsCount += $count;
$databaseDocumentsCount = 0;
$this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) {
$metric = "documents.{$database->getId()}/{$collection->getId()}.count.total";
$count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric);
$projectDocumentsCount += $count;
$databaseDocumentsCount += $count;
});
$this->createPerPeriodMetric($projectId, "documents.{$database->getId()}.count.total", $databaseDocumentsCount);
});
$this->createPerPeriodMetric($projectId, 'collections.$all.count.total', $projectCollectionsCount);
$this->createPerPeriodMetric($projectId, 'documents.$all.count.total', $projectDocumentsCount);
}
/**
* Collect Stats
*
* Collect all database related stats
*
* @return void
* @throws Exception
*/
public function collect(): void
{
$this->foreachDocument('console', 'projects', [], function (Document $project) {
$projectId = $project->getInternalId();
$this->usersStats($projectId);
$this->databaseStats($projectId);
$this->storageStats($projectId);
});
}
}

171
src/Appwrite/Usage/Calculators/TimeSeries.php
View file

@ -11,12 +11,54 @@ use DateTime;
class TimeSeries extends Calculator
{
/**
* InfluxDB
*
* @var InfluxDatabase
*/
protected InfluxDatabase $influxDB;
/**
* Utopia Database
*
* @var Database
*/
protected Database $database;
/**
* Error Handler Callback
*
* @var callable
*/
protected $errorHandler;
/**
* Latest times for metric that was synced to the database
*
* @var array
*/
private array $latestTime = [];
// all the mertics that we are collecting
/**
* Periods the metrics are collected for
* @var array
*/
protected array $periods = [
[
'key' => '1h',
'startTime' => '-24 hours'
],
[
'key' => '1d',
'startTime' => '-30 days'
]
];
/**
* All the metrics that we are collecting
*
* @var array
*/
protected array $metrics = [
'project.$all.network.requests' => [
'table' => 'appwrite_usage_project_{scope}_network_requests',
@ -190,12 +232,6 @@ class TimeSeries extends Calculator
'executions.$all.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
],
'builds.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'builds.$all.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
],
@ -231,14 +267,7 @@ class TimeSeries extends Calculator
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
],
'executions.functionId.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.time' => [
'table' => 'appwrite_usage_builds_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'executions.functionId.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
@ -268,15 +297,89 @@ class TimeSeries extends Calculator
],
],
// counters
'users.$all.count.total' => [
'table' => 'appwrite_usage_users_{scope}_count_total',
],
'buckets.$all.count.total' => [
'table' => 'appwrite_usage_buckets_{scope}_count_total',
],
'files.$all.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
],
'files.bucketId.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
'groupBy' => ['bucketId']
],
'databases.$all.count.total' => [
'table' => 'appwrite_usage_databases_{scope}_count_total',
],
'collections.$all.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
],
'documents.$all.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
],
'collections.databaseId.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId/collectionId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId', 'collectionId']
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size',
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size',
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
],
'files.$bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
],
'builds.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.functionId.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.time' => [
'table' => 'appwrite_usage_builds_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'project.$all.compute.time' => [ // Built time + execution time
'table' => 'appwrite_usage_project_{scope}_compute_time',
'groupBy' => ['functionId'],
],
];
protected array $period = [
'key' => '30m',
'startTime' => '-24 hours',
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size'
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size'
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size'
],
'files.bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
]
];
public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $errorHandler = null)
@ -303,9 +406,7 @@ class TimeSeries extends Calculator
private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
{
$id = \md5("{$time}_{$period}_{$metric}");
$this->database->setNamespace('_console');
$project = $this->database->getDocument('projects', $projectId);
$this->database->setNamespace('_' . $project->getInternalId());
$this->database->setNamespace('_' . $projectId);
try {
$document = $this->database->getDocument('stats', $id);
@ -368,7 +469,7 @@ class TimeSeries extends Calculator
$query .= "WHERE \"time\" > '{$start}' ";
$query .= "AND \"time\" < '{$end}' ";
$query .= "AND \"metric_type\"='counter' {$filters} ";
$query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} ";
$query .= "GROUP BY time({$period['key']}), \"projectId\", \"projectInternalId\" {$groupBy} ";
$query .= "FILL(null)";
try {
@ -390,9 +491,11 @@ class TimeSeries extends Calculator
}
$value = (!empty($point['value'])) ? $point['value'] : 0;
if (empty($point['projectInternalId'] ?? null)) {
continue;
}
$this->createOrUpdateMetric(
$projectId,
$point['projectInternalId'],
$point['time'],
$period['key'],
$metricUpdated,
@ -419,14 +522,16 @@ class TimeSeries extends Calculator
*/
public function collect(): void
{
foreach ($this->metrics as $metric => $options) { //for each metrics
try {
$this->syncFromInfluxDB($metric, $options, $this->period);
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e);
} else {
throw $e;
foreach ($this->periods as $period) {
foreach ($this->metrics as $metric => $options) { //for each metrics
try {
$this->syncFromInfluxDB($metric, $options, $period);
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e);
} else {
throw $e;
}
}
}
}

49
src/Appwrite/Usage/Stats.php
View file

@ -76,11 +76,14 @@ class Stats
/**
* Submit data to StatsD.
* Send various metrics to StatsD based on the parameters that are set
* @return void
*/
public function submit(): void
{
$projectId = $this->params['projectId'] ?? '';
$tags = ",projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
$projectInternalId = $this->params['projectInternalId'];
$tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
// the global namespace is prepended to every key (optional)
$this->statsd->setNamespace($this->namespace);
@ -91,8 +94,8 @@ class Stats
$this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod));
}
$inbound = $this->params['networkRequestSize'] ?? 0;
$outbound = $this->params['networkResponseSize'] ?? 0;
$inbound = $this->params['project.{scope}.network.inbound'] ?? 0;
$outbound = $this->params['project.{scope}.network.outbound'] ?? 0;
$this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound);
$this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound);
$this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound);
@ -102,12 +105,13 @@ class Stats
'users.{scope}.requests.read',
'users.{scope}.requests.update',
'users.{scope}.requests.delete',
'users.{scope}.count.total',
];
foreach ($usersMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$this->statsd->increment($metric . $tags);
if ($value === 1 || $value === -1) {
$this->statsd->count($metric . $tags, $value);
}
}
@ -124,13 +128,16 @@ class Stats
'documents.{scope}.requests.read',
'documents.{scope}.requests.update',
'documents.{scope}.requests.delete',
'databases.{scope}.count.total',
'collections.{scope}.count.total',
'documents.{scope}.count.total'
];
foreach ($dbMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
if ($value === 1 || $value === -1) {
$dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? '');
$this->statsd->increment($metric . $dbTags);
$this->statsd->count($metric . $dbTags, $value);
}
}
@ -143,13 +150,16 @@ class Stats
'files.{scope}.requests.read',
'files.{scope}.requests.update',
'files.{scope}.requests.delete',
'buckets.{scope}.count.total',
'files.{scope}.count.total',
'files.{scope}.storage.size'
];
foreach ($storageMertics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
if ($value !== 0) {
$storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? '');
$this->statsd->increment($metric . $storageTags);
$this->statsd->count($metric . $storageTags, $value);
}
}
@ -176,19 +186,30 @@ class Stats
$functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms
$functionBuildStatus = $this->params['buildStatus'] ?? '';
$functionCompute = $functionExecutionTime + $functionBuildTime;
$functionTags = $tags . ',functionId=' . $functionId;
$deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0;
$storageSize = $this->params['files.{scope}.storage.size'] ?? 0;
if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) {
$this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize);
}
if ($deploymentSize !== 0) {
$this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize);
}
if ($functionExecution >= 1) {
$this->statsd->increment('executions.{scope}.compute' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionExecutionStatus);
$this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus);
if ($functionExecutionTime > 0) {
$this->statsd->count('executions.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime);
$this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime);
}
}
if ($functionBuild >= 1) {
$this->statsd->increment('builds.{scope}.compute' . $tags . ',functionId=' . $functionId . ',functionBuildStatus=' . $functionBuildStatus);
$this->statsd->count('builds.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionBuildTime);
$this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus);
$this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime);
}
if ($functionBuild + $functionExecution >= 1) {
$this->statsd->count('project.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionCompute);
$this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute);
}
$this->reset();

8
src/Appwrite/Utopia/Response/Model/Project.php
View file

@ -2,6 +2,7 @@
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Auth\Auth;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use Utopia\Config\Config;
@ -101,6 +102,12 @@ class Project extends Model
'default' => '',
'example' => '131102020',
])
->addRule('authDuration', [
'type' => self::TYPE_INTEGER,
'description' => 'Session duration in seconds.',
'default' => Auth::TOKEN_EXPIRATION_LOGIN_LONG,
'example' => 60,
])
->addRule('authLimit', [
'type' => self::TYPE_INTEGER,
'description' => 'Max users allowed. 0 is unlimited.',
@ -225,6 +232,7 @@ class Project extends Model
$auth = Config::getParam('auth', []);
$document->setAttribute('authLimit', $authValues['limit'] ?? 0);
$document->setAttribute('authDuration', $authValues['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG);
foreach ($auth as $index => $method) {
$key = $method['key'];

11
src/Appwrite/Utopia/Response/Model/UsageProject.php
View file

@ -44,9 +44,9 @@ class UsageProject extends Model
'example' => [],
'array' => true
])
->addRule('collections', [
->addRule('databases', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of collections.',
'description' => 'Aggregated stats for number of databases.',
'default' => [],
'example' => [],
'array' => true
@ -65,6 +65,13 @@ class UsageProject extends Model
'example' => [],
'array' => true
])
->addRule('buckets', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of buckets.',
'default' => [],
'example' => [],
'array' => true
])
;
}

3
tests/e2e/General/HTTPTest.php
View file

@ -163,7 +163,8 @@ class HTTPTest extends Scope
$response['body'] = json_decode($response['body'], true);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEmpty($response['body']['schemaValidationMessages']);
// looks like recent change in the validator
$this->assertTrue(empty($response['body']['schemaValidationMessages']));
}
}

42
tests/e2e/General/UsageTest.php
View file

@ -85,7 +85,7 @@ class UsageTest extends Scope
#[Retry(count: 1)]
public function testUsersStats(array $data): array
{
sleep(35);
sleep(20);
$projectId = $data['projectId'];
$headers = $data['headers'];
@ -102,7 +102,7 @@ class UsageTest extends Scope
$res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $cheaders);
$res = $res['body'];
$this->assertEquals(8, count($res));
$this->assertEquals(9, count($res));
$this->assertEquals(30, count($res['requests']));
$this->assertEquals(30, count($res['users']));
$this->assertEquals($usersCount, $res['users'][array_key_last($res['users'])]['value']);
@ -114,6 +114,7 @@ class UsageTest extends Scope
'x-appwrite-project' => $projectId,
'x-appwrite-mode' => 'admin'
]));
$requestsCount++;
$res = $res['body'];
$this->assertEquals(10, $res['usersCreate'][array_key_last($res['usersCreate'])]['value']);
$this->validateDates($res['usersCreate']);
@ -255,7 +256,7 @@ class UsageTest extends Scope
$filesCreate = $data['filesCreate'];
$filesDelete = $data['filesDelete'];
sleep(35);
sleep(20);
// console request
$headers = [
@ -267,7 +268,7 @@ class UsageTest extends Scope
$res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $headers);
$res = $res['body'];
$this->assertEquals(8, count($res));
$this->assertEquals(9, count($res));
$this->assertEquals(30, count($res['requests']));
$this->assertEquals(30, count($res['storage']));
$this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']);
@ -279,6 +280,7 @@ class UsageTest extends Scope
'x-appwrite-project' => $projectId,
'x-appwrite-mode' => 'admin'
]));
$requestsCount++;
$res = $res['body'];
$this->assertEquals($storageTotal, $res['storage'][array_key_last($res['storage'])]['value']);
$this->validateDates($res['storage']);
@ -303,6 +305,7 @@ class UsageTest extends Scope
'x-appwrite-project' => $projectId,
'x-appwrite-mode' => 'admin'
]));
$requestsCount++;
$res = $res['body'];
$this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']);
$this->assertEquals($filesCount, $res['filesCount'][array_key_last($res['filesCount'])]['value']);
@ -411,7 +414,7 @@ class UsageTest extends Scope
$this->assertEquals('name', $res['body']['key']);
$collectionsUpdate++;
$requestsCount++;
sleep(10);
sleep(20);
for ($i = 0; $i < 10; $i++) {
$name = uniqid() . ' collection';
@ -493,7 +496,7 @@ class UsageTest extends Scope
$documentsRead = $data['documentsRead'];
$documentsDelete = $data['documentsDelete'];
sleep(35);
sleep(20);
// check datbase stats
$headers = [
@ -504,13 +507,13 @@ class UsageTest extends Scope
$res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $headers);
$res = $res['body'];
$this->assertEquals(8, count($res));
$this->assertEquals(9, count($res));
$this->assertEquals(30, count($res['requests']));
$this->assertEquals(30, count($res['storage']));
$this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']);
$this->validateDates($res['requests']);
$this->assertEquals($collectionsCount, $res['collections'][array_key_last($res['collections'])]['value']);
$this->validateDates($res['collections']);
$this->assertEquals($databasesCount, $res['databases'][array_key_last($res['databases'])]['value']);
$this->validateDates($res['databases']);
$this->assertEquals($documentsCount, $res['documents'][array_key_last($res['documents'])]['value']);
$this->validateDates($res['documents']);
@ -681,6 +684,25 @@ class UsageTest extends Scope
}
$executionTime += (int) ($execution['body']['duration'] * 1000);
$execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', $headers, [
'async' => true,
]);
$this->assertEquals(202, $execution['headers']['status-code']);
$this->assertNotEmpty($execution['body']['$id']);
$this->assertEquals($functionId, $execution['body']['functionId']);
sleep(10);
$execution = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/executions/' . $execution['body']['$id'], $headers);
if ($execution['body']['status'] == 'failed') {
$failures++;
} elseif ($execution['body']['status'] == 'completed') {
$executions++;
}
$executionTime += (int) ($execution['body']['duration'] * 1000);
$data = array_merge($data, [
'functionId' => $functionId,
'executionTime' => $executionTime,
@ -701,7 +723,7 @@ class UsageTest extends Scope
$executions = $data['executions'];
$failures = $data['failures'];
sleep(25);
sleep(20);
$response = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/usage', $headers, [
'range' => '30d'

125
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
View file

@ -335,14 +335,15 @@ class ProjectsConsoleClientTest extends Scope
], $this->getHeaders()));
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 8);
$this->assertEquals(count($response['body']), 9);
$this->assertNotEmpty($response['body']);
$this->assertEquals('30d', $response['body']['range']);
$this->assertIsArray($response['body']['requests']);
$this->assertIsArray($response['body']['network']);
$this->assertIsArray($response['body']['executions']);
$this->assertIsArray($response['body']['documents']);
$this->assertIsArray($response['body']['collections']);
$this->assertIsArray($response['body']['databases']);
$this->assertIsArray($response['body']['buckets']);
$this->assertIsArray($response['body']['users']);
$this->assertIsArray($response['body']['storage']);
@ -411,6 +412,126 @@ class ProjectsConsoleClientTest extends Scope
return ['projectId' => $projectId];
}
/** @depends testGetProjectUsage */
public function testUpdateProjectAuthDuration($data): array
{
$id = $data['projectId'];
// Check defaults
$response = $this->client->call(Client::METHOD_GET, '/projects/' . $id, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => 'console',
], $this->getHeaders()));
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(Auth::TOKEN_EXPIRATION_LOGIN_LONG, $response['body']['authDuration']); // 1 Year
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $id . '/auth/duration', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'duration' => 60, // Set session duration to 2 minutes
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals('Project Test 2', $response['body']['name']);
$this->assertArrayHasKey('platforms', $response['body']);
$this->assertArrayHasKey('webhooks', $response['body']);
$this->assertArrayHasKey('keys', $response['body']);
$this->assertEquals(60, $response['body']['authDuration']);
$projectId = $response['body']['$id'];
// Create New User
$response = $this->client->call(Client::METHOD_POST, '/account', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
], $this->getHeaders()), [
'userId' => 'unique()',
'email' => 'test' . rand(0, 9999) . '@example.com',
'password' => 'password',
'name' => 'Test User',
]);
$this->assertEquals(201, $response['headers']['status-code']);
$userEmail = $response['body']['email'];
// Create New User Session
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
]), [
'email' => $userEmail,
'password' => 'password',
]);
$this->assertEquals(201, $response['headers']['status-code']);
$sessionCookie = $response['headers']['set-cookie'];
// Test for SUCCESS
$response = $this->client->call(Client::METHOD_GET, '/account', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
'Cookie' => $sessionCookie,
]));
$this->assertEquals(200, $response['headers']['status-code']);
// Check session doesn't expire too soon.
sleep(30);
// Get User
$response = $this->client->call(Client::METHOD_GET, '/account', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
'Cookie' => $sessionCookie,
]));
$this->assertEquals(200, $response['headers']['status-code']);
// Wait just over a minute
sleep(35);
// Get User
$response = $this->client->call(Client::METHOD_GET, '/account', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
'Cookie' => $sessionCookie,
]));
$this->assertEquals(401, $response['headers']['status-code']);
// Return project back to normal
$response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $id . '/auth/duration', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG,
]);
$this->assertEquals(200, $response['headers']['status-code']);
$projectId = $response['body']['$id'];
// Check project is back to normal
$response = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => 'console',
], $this->getHeaders()));
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(Auth::TOKEN_EXPIRATION_LOGIN_LONG, $response['body']['authDuration']); // 1 Year
return ['projectId' => $projectId];
}
/**
* @depends testGetProjectUsage
*/

157
tests/e2e/Services/Storage/StorageCustomClientTest.php
View file

@ -25,10 +25,16 @@ class StorageCustomClientTest extends Scope
use SideClient;
use StoragePermissionsScope;
public function testBucketAnyPermissions(): void
public function testCachedFilePreview(): void
{
/**
* Test for SUCCESS
Create a bucket with File Level Security with no permissions.
Add a file with no permissions.
Login as UserA from SDK
Call File Preview from SDK all good userA can't see preview.
Add read permission to UserA, all good userA can now see preview.
Remove read permission for UserA.
Call File Preview from SDK and now userA can't see the preview.
*/
$bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
@ -37,22 +43,19 @@ class StorageCustomClientTest extends Scope
], [
'bucketId' => ID::unique(),
'name' => 'Test Bucket',
'permissions' => [
Permission::read(Role::any()),
Permission::create(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'fileSecurity' => true,
'permissions' => [],
]);
$bucketId = $bucket['body']['$id'];
$this->assertEquals(201, $bucket['headers']['status-code']);
$this->assertNotEmpty($bucketId);
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', [
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], [
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'fileId' => ID::unique(),
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/logo.png'), 'image/png', 'permissions.png'),
]);
@ -65,46 +68,142 @@ class StorageCustomClientTest extends Scope
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/preview', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals(404, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_PUT, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'name' => 'permissions.png',
'permissions' => [
Permission::read(Role::user($this->getUser()['$id'])),
],
]);
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/preview', [
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/preview', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]);
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]);
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/view', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]);
], $this->getHeaders()));
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_PUT, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'name' => 'permissions.png',
'permissions' => [],
]);
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/preview', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals(404, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
$this->assertEquals(204, $file['headers']['status-code']);
$this->assertEmpty($file['body']);
}
public function testBucketAnyPermissions(): void
{
/**
* Test for SUCCESS
*/
$bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'bucketId' => ID::unique(),
'name' => 'Test Bucket',
'permissions' => [
Permission::read(Role::any()),
Permission::create(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
]);
$bucketId = $bucket['body']['$id'];
$this->assertEquals(201, $bucket['headers']['status-code']);
$this->assertNotEmpty($bucketId);
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], [
'fileId' => ID::unique(),
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/logo.png'), 'image/png', 'permissions.png'),
]);
$fileId = $file['body']['$id'];
$this->assertEquals($file['headers']['status-code'], 201);
$this->assertNotEmpty($fileId);
$this->assertEquals(true, DateTime::isValid($file['body']['$createdAt']));
$this->assertEquals('permissions.png', $file['body']['name']);
$this->assertEquals('image/png', $file['body']['mimeType']);
$this->assertEquals(47218, $file['body']['sizeOriginal']);
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]);
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/preview', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]);
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]);
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/view', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]);
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_PUT, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], [
'name' => 'permissions.png',
]);
$this->assertEquals(200, $file['headers']['status-code']);
$file = $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]);
$this->assertEquals(204, $file['headers']['status-code']);

16
tests/unit/Auth/AuthTest.php
View file

@ -204,46 +204,46 @@ class AuthTest extends TestCase
public function testSessionVerify(): void
{
$expireTime1 = 60 * 60 * 24;
$secret = 'secret1';
$hash = Auth::hash($secret);
$tokens1 = [
new Document([
'$id' => ID::custom('token1'),
'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), 60 * 60 * 24)),
'secret' => $hash,
'provider' => Auth::SESSION_PROVIDER_EMAIL,
'providerUid' => 'test@example.com',
]),
new Document([
'$id' => ID::custom('token2'),
'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -60 * 60 * 24)),
'secret' => 'secret2',
'provider' => Auth::SESSION_PROVIDER_EMAIL,
'providerUid' => 'test@example.com',
]),
];
$expireTime2 = -60 * 60 * 24;
$tokens2 = [
new Document([ // Correct secret and type time, wrong expire time
'$id' => ID::custom('token1'),
'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -60 * 60 * 24)),
'secret' => $hash,
'provider' => Auth::SESSION_PROVIDER_EMAIL,
'providerUid' => 'test@example.com',
]),
new Document([
'$id' => ID::custom('token2'),
'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -60 * 60 * 24)),
'secret' => 'secret2',
'provider' => Auth::SESSION_PROVIDER_EMAIL,
'providerUid' => 'test@example.com',
]),
];
$this->assertEquals(Auth::sessionVerify($tokens1, $secret), 'token1');
$this->assertEquals(Auth::sessionVerify($tokens1, 'false-secret'), false);
$this->assertEquals(Auth::sessionVerify($tokens2, $secret), false);
$this->assertEquals(Auth::sessionVerify($tokens2, 'false-secret'), false);
$this->assertEquals(Auth::sessionVerify($tokens1, $secret, $expireTime1), 'token1');
$this->assertEquals(Auth::sessionVerify($tokens1, 'false-secret', $expireTime1), false);
$this->assertEquals(Auth::sessionVerify($tokens2, $secret, $expireTime2), false);
$this->assertEquals(Auth::sessionVerify($tokens2, 'false-secret', $expireTime2), false);
}
public function testTokenVerify(): void

2
tests/unit/Usage/StatsTest.php
View file

@ -38,10 +38,12 @@ class StatsTest extends TestCase
{
$this->object
->setParam('projectId', 'appwrite_test')
->setParam('projectInternalId', 1)
->setParam('networkRequestSize', 100)
;
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
$this->assertEquals(1, $this->object->getParam('projectInternalId'));
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
$this->object->submit();